diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5696647f06..ba16730649 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,7 +33,10 @@ defaults: # job's slowest successful leg on 2026-11-09, rounded up to a multiple of 5 # minutes: 1.5 times it for engine-free (109 minutes) and engine-us (73), twice # for engine-uk (45) and integration-uk (8), and for each shorter job at least -# 5 minutes and at least twice its slowest run. +# 5 minutes and at least twice its slowest run. The engine-uk-after-merge cap +# is about 1.6x its expected leg: microcosm#931's clone-first H2 test runs +# about 2.6 times as long as the H2 parity test (10 minutes in engine-uk), so +# about 27 minutes, plus the sync. # Every job names its runner image. `ubuntu-latest` is whichever image GitHub # assigns: on 2026-11-10 it gave one engine-free job ubuntu-26.04 and another @@ -148,6 +151,27 @@ jobs: - name: Run UK engine tests run: bash tools/run_engine_uk_tests.sh + engine-uk-after-merge: + # UK engine tests too slow for every pull request run on pushes to main + # only, after merge (microcosm#931). ci needs this job: on a push it must + # pass, and on a pull request its skip passes. + if: github.event_name == 'push' + runs-on: ubuntu-24.04 + timeout-minutes: 50 + strategy: + fail-fast: false + matrix: + python-version: ["3.13", "3.14"] + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + with: + python-version: ${{ matrix.python-version }} + - name: Sync workspace with the UK engine + run: uv sync --all-packages --locked --extra uk + - name: Run UK after-merge engine tests + run: bash tools/run_engine_uk_tests.sh engine-uk-after-merge + integration-uk: needs: select-countries if: needs['select-countries'].outputs.run_uk == 'true' @@ -204,7 +228,8 @@ jobs: ci: # ci gives the main ruleset one check that covers the country jobs. It # fails unless every other job passed, or skipped because select-countries - # succeeded and reported `false` for its country. A skipped matrix job + # succeeded and reported `false` for its country, or skipped because it + # runs after merge and the run is not a push. A skipped matrix job # reports under its unexpanded name, so a ruleset cannot require engine-us # or engine-uk by name. GitHub reports a skipped required check as passing, # so ci runs with `if: always()`. verify-workflow refuses any other shape @@ -216,6 +241,7 @@ jobs: - engine-free - engine-us - engine-uk + - engine-uk-after-merge - integration-uk - wheels - installed-emitter diff --git a/.gitignore b/.gitignore index 44ce9ecc74..2182a90c35 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,13 @@ out/ *.h5 # Licensed run outputs: pool tables and audit directories (UKDS CD137 cl. 8). *.npz +# Except the H2 parity fixture's three toy conditioned atomic-area supports, +# which tools/graph_uk_spine_fixture.py writes from test support's invented +# table over public area codes, as toy_.npz: a real support keeps +# its own name and stays ignored (microcosm#931). +!packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/toy_uk_ew_output_area_2021_conditioned_support.npz +!packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/toy_uk_scotland_output_area_2022_conditioned_support.npz +!packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/toy_uk_ni_data_zone_2021_conditioned_support.npz audit-*/ !packages/*/build/ # microcosm.build is a real namespace package, not a build-artifact dir @@ -27,3 +34,8 @@ tools/route_a/route_a.env # file holds licensed-data review values and is never committed PRIVATE_review.csv signoff-draft/ + +# Spine stage profiles (--stage-profile-dir, microcosm#931): a record keeps the +# message of a failure its node raised, so profiles stay private +stage_profile.json +.stage_profile.json.lock diff --git a/changelog.d/1112-national-home-values.added.md b/changelog.d/1112-national-home-values.added.md index 27e29fcee0..9393c1fcf8 100644 --- a/changelog.d/1112-national-home-values.added.md +++ b/changelog.d/1112-national-home-values.added.md @@ -1,4 +1,4 @@ -The national release now carries Land Registry home values (microcosm#1112). A new spine stage, `regional_home_values`, runs after the high-value home support copies and gives every England-and-Wales owner household with a value the one its region's uprated Land Registry distribution holds at the household's weighted mid-rank among the owner households of its FRS region, at the spine's weights. It ranks as the dense line's value node does (ties broken by a keyed draw; the support copies spread over the top stratum by their own draws), reads the region level of the same knots resource (`uk/high_value_home_value_knots.json`; Wales is `W92000004`), and moves `property_wealth` by the value's change. Scotland, Northern Ireland, renters and owners without a value keep theirs, and the council-tax band is untouched. +The national release now carries Land Registry home values (microcosm#1112). A new spine stage, `regional_home_values`, runs after the high-value home support copies and gives every England-and-Wales owner household with a value the one its region's uprated Land Registry distribution holds at the household's weighted mid-rank among the owner households of its FRS region, at the spine's weights. It ranks as the dense line's value node does (ties broken by a keyed draw; the support copies spread over the top stratum by their own draws), reads the region level of the same knots resource (`uk/high_value_home_value_knots.json`; Wales is `W92000004`), and moves `property_wealth` by the value's change. Scotland, Northern Ireland, renters and owners without a value keep theirs, and the council-tax band is untouched. (Later in #931 the stage retires: the WAS chain reads each England and Wales owner's main residence off its assigned constituency instead, `931-spine-localises-home-values`.) Before it, the national line solved the bound spine with the WAS-imputed values after the regional uprating, not the regions' Land Registry values. The dense line is unchanged: the stage keeps the value it replaced in `main_residence_value_rank_basis`, the dense value node receives that column through its slice and ranks on it (refusing if it is missing), and the release boundary drops it on both lines. diff --git a/changelog.d/1262-uk-brma-from-area.added.md b/changelog.d/1262-uk-brma-from-area.added.md index 6da29f92fd..023d086238 100644 --- a/changelog.d/1262-uk-brma-from-area.added.md +++ b/changelog.d/1262-uk-brma-from-area.added.md @@ -1 +1 @@ -A placed household's Broad Rental Market Area now follows its assigned area (microcosm#1262, B1). `tools/build_uk_brma_area_layer.py` places every 2021 LSOA of England and Wales and every 2022 output area of Scotland in one BRMA from Chronicle's raw registrations (PolicyEngine/chronicle#314): England and Wales by population-weighted centroid in the VOA May 2020 and Rent Officers Wales May 2012 polygons with uk-data#516's six overrides, Scotland by centroid in the Scottish Government's polygons corrected by Rent Service Scotland's postcode lookup with microcosm#1092's two overrides. The layer, `uk/brma_area_layer.json` (192 BRMAs, OGL v3.0), reproduces the region-by-BRMA support of uk-data#516's census table and microcosm#1092's 63 Scottish BRMA-by-council pairs exactly; its provenance records both checks. The post-geography node `uk.full.geography.brma` (`--brma-localisation area`, the default on the atomic assignment; `none` for measurement builds, refused by a release candidate) reads each placed Great Britain clone's BRMA off its LSOA or output area, so its LHA rate describes the same place as its local authority and constituency. Northern Ireland and the national line keep the spine's `frs_brma` draw. A terminal release-blocking gate, `uk_local_brma_enum_domain`, rechecks the rewritten column against the engine's enum; the local gate battery has seven entries. The dense and full-battery contract digests, the declared gate count and the certification exclusions move with it, so a national or dense release certified before this change fails `validate_release` after it. +A placed household's Broad Rental Market Area now follows its assigned area (microcosm#1262, B1). `tools/build_uk_brma_area_layer.py` places every 2021 LSOA of England and Wales and every 2022 output area of Scotland in one BRMA from Chronicle's raw registrations (PolicyEngine/chronicle#314): England and Wales by population-weighted centroid in the VOA May 2020 and Rent Officers Wales May 2012 polygons with uk-data#516's six overrides, Scotland by centroid in the Scottish Government's polygons corrected by Rent Service Scotland's postcode lookup with microcosm#1092's two overrides. The layer, `uk/brma_area_layer.json` (192 BRMAs, OGL v3.0), reproduces the region-by-BRMA support of uk-data#516's census table and microcosm#1092's 63 Scottish BRMA-by-council pairs exactly; its provenance records both checks. The post-geography node `uk.full.geography.brma` (`--brma-localisation area`, the default on the atomic assignment; `none` for measurement builds, refused by a release candidate) reads each placed Great Britain clone's BRMA off its LSOA or output area, so its LHA rate describes the same place as its local authority and constituency. Northern Ireland and the national line keep the spine's `frs_brma` draw. A terminal release-blocking gate, `uk_local_brma_enum_domain`, rechecks the rewritten column against the engine's enum; the local gate battery has seven entries. (Later in #931 the dense line takes its BRMAs from the spine, `uk_local_brma_enum_domain` retires, the local battery has six entries, and `uk_brma_enum_domain` checks the column at the spine's transferred boundary: `931-spine-localises-home-values`.) The dense and full-battery contract digests, the declared gate count and the certification exclusions move with it, so a national or dense release certified before this change fails `validate_release` after it. diff --git a/changelog.d/1308-uk-new-stage-health-gates.added.md b/changelog.d/1308-uk-new-stage-health-gates.added.md index f8b03405fa..45e91b20b4 100644 --- a/changelog.d/1308-uk-new-stage-health-gates.added.md +++ b/changelog.d/1308-uk-new-stage-health-gates.added.md @@ -1 +1 @@ -The three stages #1308 left ungated have stage-health gates (María's ruling of 2026-10-17, taken up with the next policyengine-uk bump), each release-blocking and blocking on an absent receipt. `uk_stage_property_purchases_fit` holds each purchase kind to HMRC's Stamp Tax transactions within 5% and to its value less a recorded shortfall of at most one fifth, the floor the stage itself refuses below. `uk_stage_regional_home_values_ranking` requires rows re-ranked in every English region and Wales on the 2024 knots, at the weighted midpoint within each FRS region, with no source at the value cap. `uk_stage_uc_legacy_closure_take_up_contract` checks what the Move to UC receipt prints, since it withholds what a thin combination realizes: the design-weight Great Britain basis, a probability and a declared source for every combination's rate, a printed support band, and a well-formed unreachable flag. The property purchases and regional home-values gates check the frame against published facts, so a synthetic smoke build records their failures without blocking on them (`population_fact_check`), as it does for the other fact gates; the legacy closure gate blocks every posture. The mirrored battery digests move. +The three stages #1308 left ungated have stage-health gates (María's ruling of 2026-10-17, taken up with the next policyengine-uk bump), each release-blocking and blocking on an absent receipt. `uk_stage_property_purchases_fit` holds each purchase kind to HMRC's Stamp Tax transactions within 5% and to its value less a recorded shortfall of at most one fifth, the floor the stage itself refuses below. `uk_stage_regional_home_values_ranking` requires rows re-ranked in every English region and Wales on the 2024 knots, at the weighted midpoint within each FRS region, with no source at the value cap. (Later in #931 it becomes `uk_stage_area_home_values_ranking`, on the WAS stage's area home values at the constituency level, with its maximum re-derived for the constituency knots: `931-spine-localises-home-values`.) `uk_stage_uc_legacy_closure_take_up_contract` checks what the Move to UC receipt prints, since it withholds what a thin combination realizes: the design-weight Great Britain basis, a probability and a declared source for every combination's rate, a printed support band, and a well-formed unreachable flag. The property purchases and regional home-values gates check the frame against published facts, so a synthetic smoke build records their failures without blocking on them (`population_fact_check`), as it does for the other fact gates; the legacy closure gate blocks every posture. The mirrored battery digests move. diff --git a/changelog.d/931-atomic-assignment-tool-on-the-spine.changed.md b/changelog.d/931-atomic-assignment-tool-on-the-spine.changed.md new file mode 100644 index 0000000000..4a0c2977e3 --- /dev/null +++ b/changelog.d/931-atomic-assignment-tool-on-the-spine.changed.md @@ -0,0 +1,23 @@ +`tools/measure_uk_atomic_assignment.py` measures the clone-first spine's own area draw (#931): G2, the draw of the spine's `atomic_area_assign` stage (`spine_geography`), on a bound spine checkpoint, in process, with no engine and no graph. Since the dense line stopped drawing areas earlier in #931, the tool refused; it now measures in process, and its graph route and the readers of the retired dense nodes' manifests go. It records the assignment-only receipt the arms read and the Northern Ireland district check of decision 9. No build runs it, and its outputs stay with the run. + +- **What it reads.** `--input-h5` is a spine checkpoint, hashed (and held to `--input-sha256` where one is given), loaded and authenticated by `calibration_run.load_bound_spine_checkpoint` against its build sidecar and spine gate report (`--input-sidecar` and `--input-spine-gates`, the H5's `.build.json` and `.spine_gates.json` by default), as the dense and national lines bind a spine. The tool keeps its base rows: the households on no copy branch (geographic clone 0), read through the shared copy-path reader on the UK copy lineage, with each one's true source, region, IMD decile, council-tax band and band flag. A household's weight is its mass in its areas: the spine's household weights as the checkpoint holds them (importance weights, once the SPI support channel has moved part of each stratum's mass onto its copies), summed over every row that keeps its areas, its geographic clones and the copies later stages make of them; SPI-channel rows (`atomic_area_conditioning.uk_support_channel_copies`), which draw areas of their own, are left out. It refuses, naming columns only, a table without those inputs, the source columns or the clone index, a source with two base rows, geographic clones that disagree on an input and more than one source year. `--supports-dir` holds the three conditioned supports, `.npz` each as `tools/build_uk_conditioned_area_supports.py` writes them; each must be the conditioned register's bytes and the support it extends the register's (`country_adapter.uk_atomic_support_register`), unless `--no-pin-supports` (synthetic supports only). Supports that place a constituency or local authority in two regions are refused. +- **How it draws.** For each clone count in `--n-clones` (default `1,15`), the base rows go through the spine's geographic expansion (`expand_uk_geographic_clones`, G1) and geography identity (`key_uk_households_by_clone_path`, G1b) on a frame of the base households alone, so every clone draws on the key the spine gives it; each clone carries 1/K of its household's weight. Two laws (`--laws`, both by default) replace `legacy` and `keyed`: + - `region_only`, today's v1 definition (`atomic_area_support.uk_atomic_assignment_definition`, by census households within the FRS region) at `--region-only-seed` (the dense role's seed, 42, by default), over the registered supports the conditioned ones extend, drawn, derived and validated by the shared operator on the key and the region, as the dense line drew it; + - `conditioned`, G2's own draw (`spine_geography.draw_uk_spine_areas` with `uk_conditioned_assignment_definition` at the spine's seed, 0), its fallback levels included. + Each resolves the engine's local authority as the spine's local-authority stage does (`uk_local_authorities`). +- **What it records**, per cell {law} x {K}, at the spine's weights with no solve: + - expected rows per constituency and local authority, and their variance, read off the definition the law's draw runs (`expected_area_support`): each row draws an area among its support's rows that match every constraint input it carries, its region and, where its level keeps one, its decile, in proportion to the weight column its stage gives it, its band's where its level keeps the band and the declared default otherwise, each on its own key. Under the region-only law this is the old single-stage expectation, now on the supports' census households: the region's rows times the area's census household share within it; + - realized rows, Kish ESS and distinct source households on the supports' roster of areas, the 50-floor breach tables, z-scores against each law's own spread (the sum of p(1 - p)), the ladder gate's verdict and London's weighted household share; + - Northern Ireland's local government districts: each one's census household share beside its expected rows, rows, sources, z-score and weighted share, with the largest published |z| and share gap; + - identity stability: the same draw on reversed rows and on the clone-0 subset, unique keys and, where the checkpoint carries G2's columns, whether the spine drew the same areas; whether every clone's area at each lower K stays at the highest; and, for the conditioned law, the source households at each fallback level, rounded; + - the spine's file, digest, K, rounded source households and sampling rung (fraction, seed and rung token, never its household counts), the supports' and definitions' digests and the definitions' streams. +- **Disclosure.** The evidence and the receipt carry no unit record, say `"publishable": false`, and the tool refuses an output path inside a checkout (`receipt_counts.outside_work_trees`) before it reads any input. Neither is published or quoted: an area's rows, and the region-only law's expected rows, sum to each region's exact base households times K, so the rounded source counts protect only the receipt and the terminal. The evidence declares `disclosure_rule: 2`: + - an area's rows, ESS and sources read " 1 the quotient is snapped to the nearest integer when it is within K ulps of it, so a weight that is an exact multiple of 60 keeps its count at any K; at K = 7 the rounding of the clone weight would otherwise add a copy for some exact multiples. The snap gives up the converse case: a quotient within K ulps above an integer counts one copy fewer than at K = 1. At K = 1 nothing is snapped, and the count is `ceil(w / 60)` bit for bit, as before. + +- **Receipt and gate.** The stage receipt's parameters gain `clone_count`, and nothing else is new. The stage-health gate compares K times the heaviest copy, and K times the heaviest selected household, with the maximum copy weight, so a receipt at K > 1 no longer fails falsely. At K = 1 both checks and their messages are unchanged. A receipt without a positive integer `clone_count` is refused. The identity tool's recompute (`tools/verify_uk_identity_stability.py`) reads K from the pre-split table, as the stage reads it from its input, and its receipt keys are unchanged. Folding a family's copies back re-sums them, which can land ulps from the family's weight: for a weight a few ulps above a multiple of 60 (at K > 1, by more than the snap absorbs), or where an exact-total correction or the #970 anchor's and the residential arms' roundings moved a member. So neither the count check nor the family test recounts the fold. At any K, a family of more than three members is held exactly where at least four of its members show their two clone halves at equal weights and every member that does shares one pair sum c, whatever the others carry: its stored count n must be the count of a weight w with fl(w / n) = c. A member that took the split's own fallback correction keeps equal halves at another weight, since the incidence clone halves it exactly, so its family keeps the bracket. The split's own exact-total correction, the incidence clone's and the residential split's each move one row, so one of those four or more is uncorrected, and equal halves mean the anchor and the arms left its pair sum at exactly w / n, so c is exactly w / n. The stage counts n on each clone's own weight and gives every member fl(w / n), and the snapped count still rises with w, so this holds at K > 1 as at K = 1. Where every member shares c but fewer than four show equal halves, the hold reaches one float either side of c, the most the anchor's and the arms' roundings move an uncorrected member where the floats either side of c are as wide, and two floats where c sits within two floats of a power of two, below which they are half as wide; c lies in (45 / K, 60 / K] to within the snap, which can leave it a few floats above 60 / K (at most 16 for K up to 15, by design arithmetic over weights at and up to 40 floats above each multiple m × 60 / K, m from 4 to 3,999), and that range holds 16 at K = 3, 8 at K = 6 and 7, and 4 at K = 12 to 15. Every other family keeps the bracket: the stored count must lie between the counts at the folded weight times 1 − 1e-9 and 1 + 1e-9. The family test, which decides whether a household is a family at all, is held the same way at the K-aware count: a household the rule selects is possibly a family where its count at the folded weight times 1 + 1e-9 exceeds one and possibly whole where its count at the folded weight times 1 − 1e-9 is one, its stored state accepted in between, and the hold's counts decide where it applies. That bracket opens for a household whose source-scaled weight K × w lies within a relative 1e-9 of a multiple of 60 (at K = 1, its own weight), where it accepts n or n + 1 copies, a whole household or a family of two among them. A stored count moved there on its own is still reported, by the flag and family-weight checks, but a household the stage split consistently into one copy more there, every member at w / (n + 1), is not. Where the hold applies, that blind spot narrows from 4.5 to 9 million floats of w on each side of a multiple to a few floats next to the count's step, for some counts n. At K = 1 the step sits a float above the multiple, which leaves at most the multiple itself where the family is held exactly, or the multiple and the float either side of it where the hold reaches one float. At K from 2 to 15 the snap puts the step 3 to 17 floats above the multiple of 60 / K, which leaves at most the float on either side of the step where the family is held exactly, or up to three floats from it in the one-float tier, so a family on the multiple itself split into one copy more is reported. The receipt's `copies_recomputed` stays the exact recount from the re-summed fold, and its tolerance policy says all this. +- **Declarations.** The `copy_rule` text in the code dictionary, `sources.yaml` and `source_stages.json` now reads "copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies, except where none of the 64 heaviest households left whole, tried heaviest first, tied weights in row order, can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight; of those 64, a selected household left at one copy can carry the correction only where its corrected weight keeps its one copy and the walk's selection, a check the fallback does not make". The high-value home support split's (`high_value_home_support`) reads "every selected household becomes copies rows at weight / copies", followed by the same exception and no more, since it makes no one-copy check. Both stages' notes say where the correction lands, and the CGT stage's add that the shared fallback does not make the one-copy check, so it can put the correction on a one-copy household the check passed over, and that the identity tool's selection recompute can then count that household as a family the stage never made, which the bracketed family test does not report, or miss one it made, which the tool reports. Such a household, lifted across 60 / K, moves only `copies_recomputed`, since the bracketed family test cannot tell it from a whole household (see Receipt and gate). The CGT stage notes give the cap as 60 divided by the geographic clone count before the incidence clone, and name the incidence clone stage (`cgt_incidence_clone`) where they mean it. +- **Exact total.** The exact-total correction lands on a household the split left whole: a positive-weight household it did not divide, unselected or selected at one copy, whose weight the split left as it was. The stage tries the 64 heaviest of them (`CGT_SUPPORT_CORRECTION_CANDIDATES`), heaviest first with tied weights in row order, and the first that can carry the correction takes it, so every root and copy weighs bitwise w / n. A selected household left at one copy can carry it only where its corrected weight keeps its one copy and the walk's selection, the one-copy check `copy_rule` names, kept as a reading of the whole-households-first rule, so wherever one of the 64 carries it, the rule rerun on the stage's output, as the identity tool reruns it, divides the same households; on the synthetic probe frames, 11 corrections would otherwise have lifted a one-copy household to a weight that counts two copies, and the rule rerun on the stage's output would have counted a family the stage never made. Only where none of them can does spi_support's shared fallback place it, unchanged, and that row can be a root or a copy, then off w / n by about one ulp of the frame's total weight: thousands of ulps of its own weight, a relative error below 1e-12 on the synthetic frames. The fallback does not consult that check, so its row can also be a whole household, among them a one-copy household the check passed over; the rerun can then divide a different set. That can happen on synthetic frames, as it could under the rule before, with the weights the rule before gave: where the one-copy household sits one to three ulps below its band's support mass, the identity tool then reports a missing family, and where it weighs 60 at K = 1 and outweighs every copy, the rule rerun counts a family the stage never made, which the tool's family test, held to its bracket, does not report, since the corrected weight lies within a relative 1e-9 of 60. On the synthetic nesting scenario at K = 7 and 13 every whole household sits in the half of NumPy's pairwise sum whose other half carries exactly half an ulp of the total, so no change to its weight rounds the total onto the target, and the fallback still puts the correction on a copy. Before, the stage tried only the heaviest unselected household. That is still the first candidate unless a selected household left at one copy outweighs it, or ties it from an earlier row, which needs every unselected household to weigh at most 60 / K (to within the K-ulp snap at K > 1). The high-value home support split (stage 26) corrects through the same helper and cap. Every household it selects becomes six rows, so its whole households are its unselected positive ones, and its first candidate is always the household it corrected before. The cap bounds the cost where no whole household can carry the correction: on a synthetic vector of 15 × 78,188 rows, the size of a K = 15 spine, 64 such households and the fallback take about 0.4 s, against 0.02 s for the one candidate before. A selected household left at one copy that can carry it also costs a rerun of the selection walk, about 0.41 s at that size, so 64 that the check passes over would take about 26 s; a national frame does not reach that, since its 64 heaviest whole households are unselected ones, far heavier than 60 / K. +- **What moves.** The copy count moves no value at K = 1. Neither does the exact-total rule wherever the heaviest unselected household is still the first candidate and can carry the correction. Where it cannot and a later one of the 64 can, at stages 42 and 26 and at any K, a rounding-sized correction moves from the row the shared fallback picked, often a root or copy, to a whole household, so those two rows' weights move by rounding amounts and later stages can move with them; where none of the 64 can, nothing moves. At stage 42, and at K = 1 too, a selected household left at one copy that outweighs the heaviest unselected household, or ties it from an earlier row, comes first, which needs every unselected household to be as light; where it can carry the correction and keep the split, it takes it even where the heaviest unselected household could have, so those two rows' weights move by rounding amounts. On the synthetic frames the copy count left the stage's output unchanged: ids, copy counts, weights bit for bit, mass records and frame content identity. The identity tool's receipt keys are unchanged, and its tolerance policy names the hold, the count bracket and the bracketed family test. Against the exact recount it replaces, it no longer reports a count problem for a family whose re-summed fold lands across a multiple of 60 from its stored count, nor a family missing, or extra, where a household's re-summed fold lands across the one-copy boundary (60 / K) from its stored state: on the synthetic frames, the one false missing family the full-chain probes found, a two-copy family a float above 60 that folds onto 60, and one false extra family at K = 2. It still reports a family of more than three members that the stage split consistently into one copy more on a multiple, as that recount did, where at least four of its members show equal halves and every member that does shares one weight, whatever the others carry: at K = 1 for all but 31 of the 9,995 counts n from 4 to 9,998, and at K from 2 to 15 for every count. Where fewer show equal halves but every member shares one weight, the hold reaches one float, which at K = 1 takes in the multiple itself for 6,393 of those counts, so such a family goes unreported there, as the full-chain test's 900 family does at one clone; at K > 1 it is reported. It no longer reports a family of three or fewer members, or one whose members fold to different weights without four equal halves among them, nor a household split into two copies at 60 / K, which the family test's bracket cannot tell from a whole one. The stage evidence gains `clone_count`. The stage contracts of `cgt_support_split` and `high_value_home_support` move with their texts. Their implementation hashes move, `high_value_home_support`'s because it binds `cgt_support`, and node keys chain on their inputs' keys, so every spine node from stage 26 on, and every node downstream of the spine, re-keys with them. The spine gates hash `stage_health`, which moved. Every node keyed on the UK country spec's or solve's fingerprint moves too, since the coverage manifest, `sources.yaml` and `source_stages.json` moved: the spine gate; the full, preflight and holdout gates; and the target compilation, target selection, problem, measures, national targets, national problem and national gate kernels. With the spine gate, every spine stage after the assembled gate moves. The coverage manifest's source digests and the H2 fixture's declarations and contract hashes are regenerated. diff --git a/changelog.d/931-cgt-keyed-offsets.changed.md b/changelog.d/931-cgt-keyed-offsets.changed.md new file mode 100644 index 0000000000..e79c2bc541 --- /dev/null +++ b/changelog.d/931-cgt-keyed-offsets.changed.md @@ -0,0 +1,12 @@ +The CGT gains redraw (`hmrc_cgt_gains_spine`) and the BADR claims in the asset-type stage (`hmrc_cgt_asset_type_spine`) key each systematic walk's offset, and each plan's quantile jitter, on its own label instead of taking the next draw of one Generator (#931; María's ruling 6 of 7 Nov). The redraw built one Generator from its seed (552) and the build period and handed it to every Table 3 cell walk, every income band's pooled walk and every (gain band, income band) plan in run order, so the n-th uniform belonged to whichever walk or plan ran n-th: a cell that gained or lost its gainers, or skipped on its carried target, moved the uniform of every walk and plan after it. Each is now an identity-keyed uniform under the declared seed, read once: + +- a cell walk is keyed `cell:::` and an income band's pooled walk `pool:`, both salted `hmrc_cgt_gains_spine:offset:`; +- a plan is keyed `plan::`, salted `hmrc_cgt_gains_spine:jitter:`; +- a BADR band is keyed on its bounds, `|` (`open` for the top band), salted `assign_badr_qualifying_gains:offset:`, under the BADR seed (555). The band offsets never depended on rows, but each was its band's position in one draw of every band's offset, so a facts revision that added a band moved the offset of every band after it. + +Bands and age groups are named by their lower bounds, region groups by their labels. The period moved from the Generator's seed into each salt, so the declarations still read "seed combined with the build period". + +- **Shared scalar.** `microcosm.build.stochastic_assignment.stable_identity_uniform(key, *, seed, salt)` returns the draw `stable_identity_uniforms([key], ...)` gives for one label, and refuses a sequence. The duck-typed `random()` adapter the walks read (`_KeyedOffset`) moved from `isa_subscriptions` into `cgt_imputation`, beside `_walk_bands`; the ISA subscriptions take the adapter from `cgt_imputation` and the scalar from `stochastic_assignment`, and their walk offsets and band jitters are the uniforms they were, bit for bit. `property_components`' one receipts-walk offset stays as it is (ruling 6). +- **Declarations.** The redraw's `seed_mixing` and the BADR `realization` texts say the above, and the sub-AEA remainder's `mapping` text no longer names a walk and plan draw order, since there is none. Each is identical in the code dictionary, `sources.yaml` and `source_stages.json`. `hmrc_cgt_gains_spine` and `hmrc_cgt_asset_type_spine` now compute through `stochastic_assignment`, so both bind it, as the derived binding rule requires. +- **No nesting promised.** The offsets and jitters are the same at every geographic clone count, and whichever other cells walk or skip. The walks themselves stay pooled: the rounding carry runs through an income band's cells, the pooled walk takes the band's net shortfall over its unassigned gainers, each plan stratifies members pooled from every cell, and each BADR band walks its whole pool. A change in one cell can still move the assignments and amounts of the other cells in its income band, and a plan's amounts whenever its members change. Through the rake it can move cells in other income bands too: the rake seeds every cell from the frame's own (age group, region group | income band) gainer shares and fits the seed to the joint and to the Table 6 age and Table 5 region margins, which span every income band, so a change in one cell's gainers moves the raked targets of cells in other income bands, and with them possibly those cells' assignments and amounts. On a synthetic frame, adding one cell in the lowest income band moved all 20 positive raked targets of a two-cell higher band, by up to 0.8 per cent. +- **What moves.** Every Table 3 walk's offset and every plan's jitter changes, so band membership near each walk boundary and the within-band quantiles move. On three synthetic frames, 9 of 1,002, 9 of 3,000 and 126 of 1,512 gainers changed gain band, and 338, 2,920 and 1,300 changed amount; no non-gainer moved. On the third, 70 of the 1,512 gainers crossed the annual exempt amount, between a Table 3 plan and the sub-AEA remainder (none on the other two), so the liable set moves, and with it the households stage 46 splits into arms and the row count from there. Every BADR band's offset changes, so each band's walk selects its claimants at new positions. On three synthetic frames of equal weights no claimant row of the old walk claims in the new one, while each frame's claimant count moved by one or two; the main asset types moved with the claims. ISA amounts do not move: on three synthetic frames every ISA output column is unchanged. The stage contracts of both stages, the H2 fixture's declarations and the 22 values of its synthetic asset-type facts (which the fixture tool sizes to the frame after the redraw), the coverage manifest's 25 source digests and `source_stages.json` are regenerated. Every spine node re-keys, since `graph_kernels` and `stochastic_assignment` changed and the spine's root binds `stochastic_assignment`, and on a bound spine the national line re-keys 5 of its 6 nodes, all but the bound checkpoint, since the national target, problem and gate kernels hash the country spec, which this change edits; values move from stage 44 on, and `isa_subscriptions` re-keys through its `cgt_imputation` binding and its own module with no change in its output. diff --git a/changelog.d/931-council-tax-band-d.added.md b/changelog.d/931-council-tax-band-d.added.md new file mode 100644 index 0000000000..e7e5f79dff --- /dev/null +++ b/changelog.d/931-council-tax-band-d.added.md @@ -0,0 +1,19 @@ +2024-25 Band D council tax for every Great Britain local authority, and the statutory band formula, in `microcosm.build.uk_runtime.council_tax_band_d` (#931). No stage reads them yet; council-tax localisation will. + +- **England and Wales.** `uk/council_tax_band_d_england_wales_2024_25.json`, built by the new `tools/build_uk_council_tax_band_d.py` from two raw-only Chronicle registrations (chronicle#413). Each file is held to the sha256, size and manifest its registration records at the Chronicle commit the UK feed pins. + - England: MHCLG's 2024-25 Table 8 (area council tax for a Band D dwelling occupied by two adults), read with the standard-library ODS reader `ods_tables.read_ods_tables`. Each amount must lie within 1e-9 of a whole number of pence and is kept as that pence. + - Wales: StatsWales' composition of average band D council tax (the 2024-25 total for each billing authority area), read with `csv` and kept as published. Each authority is joined to its April 2023 code by its exact name on `local_authority_names.json`, and the all-Wales row is a declared exclusion. + - The codes read must be exactly the roster's English and Welsh codes. `--built-on` is explicit, and `--check` rebuilds the resource and compares it with the committed one, `built_on` aside. +- **Scotland.** `uk/scotgov_band_d_council_tax_2024_25.json`, vendored by `tools/vendor_uk_ledger_facts.py` from the pinned feed: CTAS 2026's 2024-25 average Band D for the 32 council areas (`scotgov.ctas2026.band_d.fy2024`; the `local_authority` level leaves out the Scotland row). Its register entry in `ledger_fact_vendor_selections.json` names `uk_runtime.council_tax_band_d` as its consumer and carries the OGL v3.0 attribution. +- **The loader.** `load_uk_council_tax_band_d()` merges the two over the roster's Great Britain codes as exact rationals, each read as a `Fraction` through `Decimal`. Scottish rows are read through `vendored_rows`, so a copy taken from another feed is refused before any row is read, and each Scottish amount is kept as the pence it lies within 1e-9 of. + - It refuses a Northern Ireland code, a code off the roster or of a nation the resource does not cover, a missing or repeated authority, and a value that is not a positive finite decimal. + - It also refuses a wrong schema version, kind, resource, fiscal year or roster vintage, and a Scottish row of another record set, concept, unit, period, level or vintage. + - Refusals name codes, fields and record ids, never an amount. Test payloads replace the packaged resources as in `load_hmrc_isa_statistics`. + - `check_scotland_band_d_declaration` is the runtime arm of the `verify_vendored_fact_resource` operation a stage will declare for the Scottish resource. +- **The statute.** `UK_COUNCIL_TAX_BAND_RATIOS` gives each nation's band proportions to Band D as exact fractions: England and Wales 6, 7, 8, 9, 11, 13, 15 and 18 ninths for A to H, and Wales 21 ninths for I; Scotland 240, 280, 320, 360, 473, 585, 705 and 882 over 360 for A to H, in force from 2017-18. `UK_COUNCIL_TAX_SINGLE_ADULT_DISCOUNT` is a quarter, and `uk_statutory_council_tax(band_d, nation, band, *, single_adult)` returns the exact bill and refuses Northern Ireland, a band the nation does not use and a float Band D. Nations and band sets are read from `council_tax_constants`, which is unchanged. +- **Shared with the tool.** The tool takes the England-and-Wales resource's name, kind, schema version, method, fiscal year, pence tolerance and its amount checks (`band_d_pence`, `band_d_decimal`) from the runtime module, so the two cannot drift. +- **Keys.** The two packaged resources, the register entry and `country_package.json` move the UK country fingerprint and the solve fingerprint. The kernels that hash them (the spine gates, and the full line's target compilation and selection, measures, problem, preflight, gates, holdout and national nodes) move, and the nodes after them re-key. No stage declares the new resources, so no stage contract moves; no H2 file or value moves, and no other country's key moves. +- **Tests.** + - `tests/engine_free/uk/test_uk_council_tax_band_d.py`: the committed resources load over exactly the roster's Great Britain codes; England and Scotland are at pence and Wales has at most six decimals and is not rounded to pence; the England-and-Wales `sources` equal the tool's pins; the Scottish resource carries the feed pin. It also covers the loader's refusals on invented amounts, the runtime arm, and every band of every nation under the statute, which restates the proportions so a typo or Scotland's pre-2017 proportions fail. + - `tests/engine_free/uk/test_uk_council_tax_band_d_tool.py`: the tool on synthetic inputs, and its `--check` on the committed resource when the registrations are mounted. + - `tests/engine/uk/test_uk_council_tax_band_d_engine.py`: the proportions equal policyengine-uk's at 2024 within 1e-9 and the single-adult discount exactly, and every Great Britain code's engine key, and no Northern Ireland one, is a key of the engine's Band D amounts. diff --git a/changelog.d/931-council-tax-localisation-code.added.md b/changelog.d/931-council-tax-localisation-code.added.md new file mode 100644 index 0000000000..335a079b5f --- /dev/null +++ b/changelog.d/931-council-tax-localisation-code.added.md @@ -0,0 +1,21 @@ +Council-tax localisation for the clone-first spine, G6, in `microcosm.build.uk_runtime.spine_council_tax` (#931), unwired: no source manifest declares its stage, `council_tax_localise`, and nothing imports the module, so no key or value moves. Later in #931 group (d), the stage is declared between `private_rent` and `was_wealth`. + +- **The kept bill.** Before it rewrites anything, the stage writes `council_tax_frs_bill`, `max(council_tax - council_tax_rebate, 0)` on every row, float64 and nonnegative, by the expression `was_wealth.recipient_predictors` computes its council-tax predictor with. +- **The reference.** `uk_council_tax_references` recomputes each household's draw cell from the inputs the area draw (`atomic_area_assign`) read, with the area draw's definition built at seed 0 over the three conditioned supports (`uk_draw_inputs`, `uk_draw_levels`, `uk_draw_frame`). A Great Britain household's reference is the exact expected 2024-25 Band D of that cell under the draw's own weights, from the shared `draw_cell_expectations`. Its ratio is its own authority's Band D over that reference: one correctly rounded division of the two exact rationals. The rows the SPI support channel copied, SPI copies and income band donors, have both draw inputs missing by rule, so their cell is their region by households, the L3 cell the SPI-copy area stage drew them from. +- **The draw-cell guard.** Each Great Britain row's drawn `atomic_area_code` must be an area of its recomputed cell with mass in the weight it samples by, and its `atomic_area_system` and `local_authority_code` must be that area's. +- **The paths**, by nation first and channel second. + - Northern Ireland's rows keep their bill, rebate and reduction bit for bit, with no cap. + - Great Britain's SPI-channel rows take the ratio, whatever bill provenance they inherit. + - Great Britain's FRS-channel rows follow stage 5's `council_tax_bill_provenance`. `observed_gross` takes the ratio. `missing` and `unknown_reduction` take the 2024-25 statute where `council_tax_band_observed` holds (`uk_statutory_council_tax` at the household's authority, nation and band, less the single-adult discount where `council_tax_single_adult_raw` is 1), and the ratio on stage 5's bill where it does not. `not_levied` is refused there. +- **Reductions.** On every Great Britain row, the rebate and the household's positive `council_tax_benefit_reported` become `min(ratio x amount, new bill)`, so the rebate equals the reduction after the stage wherever it did before. +- **Checks and refusals.** After writing, the stage asserts, through the shared `amounts_within_bound` at tolerance 0, that Great Britain's reductions and rebates lie within their bills, finite and nonnegative. It also asserts that the rebate still equals the reduction wherever it did, and that Northern Ireland is unchanged. Its refusals name columns, dtypes and declared constants only. It refuses: + - a missing column, or a bill, rebate or reduction that is not float64, finite and nonnegative; + - more than one positive reduction in a household, or a person whose household is absent; + - a provenance outside stage 5's four values, `not_levied` in Great Britain, or another value on a Northern Ireland FRS-channel row; + - an authority without a 2024-25 Band D, on a row or in a row's cell, or a statute band its nation does not price; + - the guard, a frame that already carries the kept bill, and a frame period other than 2024. +- **The transform.** `UKCouncilTaxLocaliseStageTransform(stage, supports, band_d=None)` holds its stage to the reviewed declaration `council_tax_localise_operation_parameters()` states. That declaration has four operations, in order: `verify_vendored_fact_resource` for the Scottish Band D, `derive` for the kept bill, `localise_council_tax` and `record_mass_conservation_receipt`. The stage must also declare its two Band D artifacts with `runtime_sha256_required`, the kept bill as its one output and the three rewrites. The transform loads the committed Band D unless a test supplies one. Running it runs the Scottish resource's runtime arm with the frame's period, localises, and appends the stage's household mass record. +- **Evidence.** It starts with `stage`. Then come the method, the clone count, the Band D and support digests and distinct true source households by path: FRS-channel rows over every clone through `rounded_count`, support-channel rows in geographic clone 0. The households whose reductions were capped, on a ratio or a statute path, and the Northern Ireland households whose reductions exceed their bill are counted in clone 0 through `extreme_count`. The five checks follow as booleans, whether Northern Ireland's reductions lie within their bills read in clone 0 as their count is, then the outputs and the rewrites. Every field but `clone_count` is the same at every clone count, and no ratio, bill, authority, cell or level is printed. +- **Tests**, on invented households and an invented Band D. + - `test_support/microcosm_build/uk_council_tax_fixtures.py`: invented council-tax cases on the toy conditioned supports, covering every path, one statute row in a cell of two authorities among them. The spine's own steps clone, key and place them. The support channel's stand-in copies them, and the copies are re-keyed and re-placed at L3 as the SPI-copy area stage does it. Each toy authority has a Band D in thirds of a pound. A sweep holds one household for each Great Britain draw cell of the toys. + - `tests/engine_free/uk/test_uk_spine_council_tax.py`, at K = 1 and 3. Each level is the area draw's, and every SPI copy and band donor is at L3. Each reference equals the cell's expected Band D worked again from the supports, and the ratio averages exactly one over every Great Britain cell of the toys. Ratio rows' bill, rebate and reduction are their ratio times the input, bit for bit. Statute rows follow each nation's proportions at their own authority's Band D, restated in the test, with the discount, and their rebate and reduction take the ratio, in a cell of two authorities too. The cap holds on ratio and statute rows. Northern Ireland is left bit for bit, a reduction above its zero bill included and counted. The kept bill is the paid bill from the columns before the rewrite, netting the rebate where it differs from the reduction. Clone 0 at K = 3 is K = 1's, and the result does not depend on row order. The evidence matches its counts, each its own where every FRS path holds a different number of sources. It counts the support channel's rows in clone 0 even where later clones copy other households, and the capped households and Northern Ireland's reductions above the bill, and whether they lie within it, in clone 0 even where later clones differ. It is byte-equal at K = 1 and 3 but for `clone_count`. Every refusal names no key, code or count of a household, and the declaration refuses a drifted stage. diff --git a/changelog.d/931-council-tax-provenance-and-check.added.md b/changelog.d/931-council-tax-provenance-and-check.added.md new file mode 100644 index 0000000000..c29a6ba067 --- /dev/null +++ b/changelog.d/931-council-tax-provenance-and-check.added.md @@ -0,0 +1,20 @@ +The council-tax bill's provenance at stage 5 (`frs_council_tax`) and a stage-health check that holds council-tax reductions within the bill (#931), unwired: no stage writes the provenance and no gate declares the check, so no value moves. Later in #931 group (d), stage 5 writes the provenance, council-tax localisation reads it, and a transferred gate declares the check. + +- **The bill's provenance** (`microcosm.build.uk_runtime.frs_council_tax`). `derive_council_tax_provenance(household, household_raw)` names the branch `derive_council_tax` takes for each household's bill outside Northern Ireland, and `not_levied` there, as a `StringDtype("python")` series on the bill's index, for the column `FRS_COUNCIL_TAX_BILL_PROVENANCE_COLUMN` (`council_tax_bill_provenance`) that stage 5 will write. The first value that holds names the row: + - `not_levied` where raw GVTREGNO is 13 (`NORTHERN_IRELAND_GVTREGNO`, beside `SCOTLAND_GVTREGNO`): Northern Ireland pays domestic rates, not council tax; + - `missing` where CTANNUAL is negative or blank, or zero where CTPAID is 1 and no reduction is reported, so the bill takes the cell mean; + - `observed_gross` where CTREB is 1 and a positive CTREBAMT is added back to the netted CTANNUAL; + - `unknown_reduction` where CTREB is 1 without a positive CTREBAMT, whose bill is the larger of the netted CTANNUAL and the cell mean; + - otherwise `observed_gross`, the netted CTANNUAL, which stays zero where CTANNUAL is zero and CTPAID is not 1. + + The four values are `FRS_COUNCIL_TAX_BILL_PROVENANCE_VALUES`. One branch function, `_council_tax_branches`, chooses both the bill and its provenance from the same masks, and `derive_council_tax` returns its bill, bit for bit as before. `frs_council_tax_provenance_operation_parameters()` is the `derive` declaration (`output` and `formula`, the precedence above in raw codes) that the stage will declare. `council_tax_provenance_evidence(frame, provenance)` is the stage's receipt form: `{"stage": "frs_council_tax", "bill_provenance_sources": {value: rounded_count}}` for all four values, zeros included, counting distinct source households before the copies through `record_source_households`. It refuses an input that is not one-dimensional, has the wrong length or holds a value outside the four, by texts that name the column and the four values, never a value or count read. +- **The check** (`stage_health`, check `council_tax_reduction_within_bill`). It reads the phase's frame and no receipt key beyond the stage match every stage-health check makes. The households whose `region` is one of `excluded_regions` leave first, with their persons. Then each remaining household's `household_reduction_columns`, and each of its `person_reduction_columns` summed over its persons through `person_household_id`, must be at most its `bill_column` plus `absolute_tolerance`, through the shared `microcosm.build.bounded_amounts.amounts_within_bound`. Every value read of the bill and of those columns must also be finite and non-negative. The household ids must name each household once over the whole table before any leave. + - Its details are booleans: `households_checked`, one per reduction column under `columns`, and `finite_nonnegative`. A frame with no household left to check passes, with `households_checked` false. + - Its failure texts name the stage, the check and columns or parameters, never an id, a value or a count. A missing frame, a malformed declaration, a missing or repeated column, a non-numeric column, a repeated household id and a person whose household is in no table each fail by name. +- **Bindings** (`battery_bindings`). The stage-health binding hands the phase's frame to the new check, as to the band donor and copy area checks, and routes its four keys: `bill_column`, `household_reduction_columns`, `person_reduction_columns` and `excluded_regions`. Its tolerance is the routed `absolute_tolerance`. `UK_GATE_REGISTRY_MODULES` gains `microcosm.build.bounded_amounts`, which the whole registry now reaches. +- **Counting stage.** `frs_council_tax` joins the counting stages in `graph_kernels`, since its module now counts through `receipt_counts`. The counting test's control moves to `frs_employment`. +- **Keys.** UK only. `uk.stage.frs_council_tax@1`, every spine kernel through `graph_kernels`, the spine gates (`stage_health`, `battery_bindings`), the full battery's preflight and calibrated gates and the national gates (whose registry modules now include `bounded_amounts`) change hash, and the nodes after them re-key. No stage contract, phase code, country or solve fingerprint, declaration, generated file, `gates.json` or H2 file changes. No NZ, BE, AM or US key moves: `microcosm.build.gates`, `country_spec` and `source_manifest` are untouched, and only the UK stage-health check calls `bounded_amounts`. +- **Tests**, on invented rows and tables only. + - `tests/engine_free/uk/test_uk_frs_council_tax.py`: each provenance value and the precedence on hand-worked raw rows, with each bill equal to a literal worked by hand; a seeded property that re-derives the branch and the bill one row at a time; alignment by household id; the declaration's value order; the two GVTREGNO constants against `frs_spine.REGION_MAP`; the receipt's rounded counts, with copies counted once; and the refusals. + - `tests/engine_free/uk/test_uk_stage_health.py`: reductions at the bill and within the tolerance pass, in float and integer columns; a rebate or a reported reduction above the bill fails its column alone, as does a two-person sum above it and a reduction on a zero bill; NaN, infinite, negative and missing values fail `finite_nonnegative`, a NaN or negative bill failing every column; a Northern Ireland household above its zero bill fails nothing when its region is excluded and every column when it is not, and its persons leave with it; the declaration and table refusals; no digit in any text; details that are booleans. + - `tests/engine_free/uk/test_uk_battery_bindings.py`: the binding hands the check the phase's frame (evidence absent without it), routes its four keys, each of which changes the verdict, and refuses a misspelt key through `validate_gate_parameters`. diff --git a/changelog.d/931-council-tax-reduction-gate.changed.md b/changelog.d/931-council-tax-reduction-gate.changed.md new file mode 100644 index 0000000000..a7fddec89a --- /dev/null +++ b/changelog.d/931-council-tax-reduction-gate.changed.md @@ -0,0 +1,26 @@ +Gate each household's council-tax reductions within its bill on the UK spine, and hold the copies made after the council-tax localisation to their parent's council tax (#931). In a second gates digest break for #931's new spine gates (the plan allowed one; this extends it), the spine battery gains `uk_council_tax_reduction_within_bill` at the transferred boundary, and the copy area agreement holds `council_tax` and `council_tax_rebate`. No value moves. Every spine built before this change no longer binds, those built since `council_tax_localise` was declared included: the strict loader holds a stored report's gate digests to the current declarations. + +- **The gate** (`gates.json`, `uk_council_tax_reduction_within_bill`, right after `uk_copy_area_agreement`). It declares the stage-health check `council_tax_reduction_within_bill` on `council_tax_localise` at the transferred boundary. It is release-blocking: a spine without the `council_tax_localise` receipt fails it closed, and absent evidence (no frame, or no stage evidence) blocks it in every posture (`evidence_absent_blocks`). + - The households whose `region` is in `excluded_regions` (`NORTHERN_IRELAND`) leave first, with their persons. Then each household's `council_tax_rebate` (`household_reduction_columns`), and its persons' `council_tax_benefit_reported` summed (`person_reduction_columns`), must be at most its `council_tax` (`bill_column`) plus `absolute_tolerance`, and every value read must be finite and non-negative. + - The check reads the transferred frame, and no receipt key beyond the stage match every stage-health check makes. Its details are booleans, and its failure texts name the stage, the check and columns. + - It sits at the transferred boundary because `council_tax_localise` is the last spine stage to write the bill or a reduction, and the copies made after it inherit both. The localisation caps every Great Britain reduction and rebate at the new bill and leaves Northern Ireland's rows as they are, uncapped, so the gate leaves Northern Ireland out. + - No gate kind is added. The gate is a stage-health entry whose check reads the frame, as the copy area agreement is, so `country_spec` and `gates`, which NZ transport hashes, are untouched. +- **The copy area agreement** (`uk_copy_area_agreement`). Its later layers (`high_value_support`, `cgt_support_split`, `cgt_incidence_clone` and `cgt_residential_split`), whose copies are made after `council_tax_localise`, also hold each copy to its immediate parent on `council_tax` and `council_tax_rebate`, the household columns the localisation rewrites. The support channel's layer stays on region and the clone index alone, since the localisation prices those copies at their own cell. The kept bill (`council_tax_frs_bill`) is not held, and neither is the person reduction, since the copy check reads households. +- **The WAS support gate's notes reworded to name the clipped and the exempt columns (María, 14 Nov).** Text only: `uk_stage_was_wealth_support`'s parameters are unchanged. +- **Scopes and phase codes.** `UK_SPINE_GATE_SCOPE` gains the gate, so the closed-world partition gives it to the spine's battery, and the calibration seam and the local battery leave it to the spine. The spine's transferred gate code also hashes `microcosm.build.bounded_amounts`, which the check computes through, and its docstring's count of the modules that no stage up to `area_brma` hashes is recounted from the walk. +- **`microcosm.data.contract`** follows, re-pinned from the live producer: the entry's gate and phase, the evidence ids (the stage-health binding records an evidence digest for every entry), the spine part's scope, the battery's policy, manifest and fingerprint digests, and the spine part's digests. The calibration seam's and the release cut's digests do not move. +- **Keys.** UK only. `gates.json` is in the UK country fingerprint and in the solve fingerprint, so both move, and so do the gate kernels and the kernels keyed on the solve fingerprint. `gate_scopes`, and for certification readiness `microcosm.data.contract`, move the kernels that hash them. Of the gate phase codes only the spine's transferred code moves. Both spine gate phases re-key, and with them every node after `area_brma`: the rest of the spine, and every dense and national node. No stage kernel or stage contract moves, and no H2 or generated file changes. Outside the UK no key moves: no transport or US kernel, no NZ node with its donors held fixed, no NZ battery source, no US graph node, and no AM, BE, NZ or US fingerprint, spec-engine wire or bundle lock. +- **Tests**, on invented frames, the toy supports and the packaged spec. + - `tests/engine_free/uk/test_uk_stage_health.py`: + - the copy area gate's later layers hold the localisation's household rewrites, beside the SPI-copy area stage's; + - the packaged gate holds the bill and the reductions the localisation rewrites, with its excluded regions, at the transferred boundary, release-blocking, and the stage-health binding routes each of its keys and refuses each one misspelt; + - `council_tax_localise` is the last packaged stage that writes the bill or a reduction, with the support channel before it and the later copy stages after it; + - on the frame and the receipt the localisation's packaged stage writes, at one and three clones, the registered gate passes through the battery's registry. An invented Northern Ireland household's reduction above its zero bill, which the stage's receipt records, is left out, and held in it fails both columns; + - a Great Britain household's rebate, or its first person's reported reduction, set above its bill fails the gate on that column, and a Northern Ireland household's does not; + - stage evidence without the localisation's receipt fails the registered gate closed and blocks the build, and absent evidence (no stage evidence, or no frame) blocks it in a development build too; + - through the battery's registry, the packaged copy area gate fails a copy of each later layer moved on `council_tax` or `council_tax_rebate`, on that layer and column alone, and passes a support channel copy moved on either. + - `tests/engine_free/shared/test_gate_battery_contract_pins.py`: the mirrored evidence ids are the entries whose binding attests evidence, every stage-health entry among them. + - `tests/engine_free/uk/test_uk_battery_bindings.py`: the stage-health binding reads the frame for three packaged gates, all at the transferred boundary. + - `tests/engine_free/shared/test_country_spec.py`: the battery's entries and the evidence-absent entries in order, the new gate right after the copy area agreement. + - microcosm-data's `tests/engine_free/shared/test_contract.py`: the mirrored entry, its stage and the battery's digests. + - `tests/integration/uk/test_uk_staging_integration.py`: the smoke spine's battery passes the new gate. diff --git a/changelog.d/931-dense-k-pin.changed.md b/changelog.d/931-dense-k-pin.changed.md new file mode 100644 index 0000000000..e8df3efeca --- /dev/null +++ b/changelog.d/931-dense-k-pin.changed.md @@ -0,0 +1,38 @@ +Hold dense engine blocks, release candidates and size evaluation to the spine's clone count (#931). The dense line takes its geographic clone count K from the spine it binds; now every reader of K holds to it. The dense line still refuses every spine whose roster is not geography-first, so no dense build runs yet; what changes is which spines and runs are refused, and how size evaluation reads a dataset. + +- **Engine blocks.** `full_measure`'s engine blocks take the clone count their caller binds (`clones`, through `resolve_uk_full_measures` and `resolve_uk_full_national_problem`) and refuse a frame whose geographic clone count is not K, at every block count, one included: a one-block call on some of the pool's clones (one of them, say) used to pass. The check reads the clone count alone, so a part of the pool that keeps every clone index is not refused there. Blocks other than one or K are refused too. The refusal names the two clone counts and nothing else. The dense measures node carries the spine's K as its `geographic_clones` parameter, which its kernel hands to the engine blocks beside `engine_blocks`, and `append_uk_target_nodes` takes it, refusing engine blocks other than one or K and a K that is not a positive integer. The national rows (`materialize_uk_national_rows`) resolve at one block of one clone. Without `clones` (callers outside the build graph) nothing changes. +- **`--engine-blocks`.** Preparation refuses a count other than one or the spine's K once it has bound the spine, naming K. Validation still compares it with an asserted `--n-clones` only. +- **Release candidates.** On top of the sampled-spine refusal, preparation refuses a `--release-candidate` whose spine fails any of three checks, naming each failure and K: + - its clone count is not the doctrine's 15; + - it was not built as a release candidate (its sidecar's `release_posture`, or the one a spine request's flags give); + - its geography binding is not a release's: the new `country_adapter.uk_spine_geography_binding_failures` wants the atomic law (`assignment` `"atomic"`), the geography key as `identity`, a SHA-256 `definition_sha256`, support pins equal to the conditioned supports `sources.yaml` registers (`uk_atomic_support_register(conditioned=True)`) and no system the register does not list, and, where it is asked for one, that K; sizes and K must be integers, so a recorded `15.0` fails. + + The in-graph spine of a `--spine-request` records no geography block, so its binding carries K alone: a release candidate on one is refused as recording no geography binding until the request route composes a block. (Later in #931 it composes one with the sidecar writer's helper, and the check also holds the block's stream and register: `931-spine-assigns-areas`.) The `uk_atomic_support_register` docstring now says a release builds on the conditioned supports, and that the base ones are what `tools/build_uk_conditioned_area_supports.py` builds them from. +- **The spine's own `--release-candidate`** still takes one clone. It now also takes the dense release's 15, but only where the packaged UK spec's roster is geography-first (`spine_posture.roster_failures`); otherwise the refusal names what the roster lacks. Every other clone count is refused, in the parser and in the preparation the full build hands a spine request, in a sentence that says 15 needs a geography-first roster. Once later commits of #931 declare the area stages and the WAS chain's localiser, the check passes without a code change. +- **The release pre-flight** (`tools/preflight_uk_local_release_candidate.py`) holds the manifest's `geography.assignment` to the same helper, so to the conditioned register rather than the base one, and requires its `geographic_clone_count` to be 15, as `parameters.n_clones` must be. Its failures name the field and what it must be, never the value: `geography.assignment.assignment is not 'atomic'.` replaces the message that printed the law it found. +- **Size evaluation** (`size_evaluation.weight_tables`) reads a dense dataset as its clone-first spine's rows: + - Every row's ordinal at the copy-id table's `geographic_support` field must equal its `clone_index`. Each row's spine household is then itself, and its design weight is that spine row's own weight, with no division by K. + - Where a file keeps no `source_household_id`, distinct sources fold the geographic clones back to clone 0's id (`copy_id_table.parent`), copies apart. + - The old dense layout's multiplier decode (`_source_multiplier`, `_spine_ids` and their `id_multiplier_for_values` import) is removed, as are `weight_tables`' `n_clones` and `id_multiplier` arguments. A dataset in another layout is refused in one fixed sentence, which names no count, id or value and points at the tools of the commit that built it. + - The result's `source_id_multiplier` and `source_id_multiplier_basis` give way to `source_household_basis` (`source_household_id` or `geographic_support_parent`), the spine-coverage refusal no longer prints a count, and spine weights that list a household twice are refused. + - The copy-id guards now list `size_evaluation` among the table's readers, not among the users of the retired rule. +- **What moves.** Keys and records only; no dataset value, and no spine node. No spine kernel or gate phase code hashes an edited module, and no spine stage calls edited code, so every spine node keeps its key and H2 is not regenerated. + - 12 of the 86 UK kernels change hash, none of the spine registry's 60: + - the dense and gate kernels that hash `full_measure` or `graph_targets` (`uk.full.target_compilation@1`, `uk.full.target_selection@1`, `uk.full.measures@1`, `uk.full.problem@1`, `uk.full-gates.preflight@1`, `uk.full-gates@1` and `uk.full.rotated-holdout@1`); + - the national kernels that hash `graph_national` (`uk.full.national_targets@1`, `uk.full.national_problem@1`, `uk.full.national-gates@1` and `uk.full.national-readback@1`); + - `uk.full.bound_spine@1`, which hashes `graph_build`. + - So every node of the dense graph re-keys, on a composed spine (after its 86 spine nodes, whose keys hold) and on a bound one, and so does every node of the national graph. The measures node's key also takes its new `geographic_clones` parameter. + - `full_build_cli`, `rowwise_cli`, `spine_build`, `country_adapter`, `size_evaluation` and the pre-flight tool are hashed by no kernel, and the gate phase codes do not move. + - The NZ transport kernels' hashes, the US kernels and post-transfer graph keys, the spec engine's compiled bundles and the packaged specs are unchanged. + - No declaration, generated file, `gates.json` or schema changes. +- **Tests** (invented data): + - The engine blocks on the clone-first stand-in: the whole pool resolves in one block or K, while a frame of some of its clones, or of more clones, is refused at either count, and before any engine loads in both resolvers. Blocks other than one or K are refused. + - The measures' `geographic_clones` parameter, the measures kernel handing it to the engine blocks beside `engine_blocks`, and the target nodes' refusals. + - `--engine-blocks` held to the spine's K on both routes. + - A release candidate prepared on a stand-in spine at 15 clones, built as a release candidate, with the stand-in's own binding. + - A release candidate on a bound spine at another K, in another posture, under another law or off the register, refused naming each failure; the same spines build candidates. The request route's refusal, naming no geography binding. + - The spine CLI refusing 2, 7 and 16, and refusing 15 on a roster before the area stages while accepting it on a geography-first one. + - The binding helper field by field, a float K and float sizes included. + - The pre-flight refusing base-register pins, another K and each binding field. + - Size evaluation decoding a table-minted clone-first dataset by the identity above 2^62 without dividing by K, and refusing the old layout and rows off the layout without a figure. + - The stand-in sidecar's geography block is now the stand-in spine's own: the toy definition's digest and stream, and the toy supports' pins (`uk_full_population_graph.toy_geography_binding`). `patch_support_register` registers those pins as the conditioned supports and refuses a reader that asks for the base register. diff --git a/changelog.d/931-dense-localisation-nodes-retired.removed.md b/changelog.d/931-dense-localisation-nodes-retired.removed.md new file mode 100644 index 0000000000..09a5579684 --- /dev/null +++ b/changelog.d/931-dense-localisation-nodes-retired.removed.md @@ -0,0 +1,8 @@ +Retire the dense line's own property, BRMA, private-rent and fuel-relief nodes, with their flags, configuration fields and release-candidate refusals (#931). The dense line takes home values, BRMAs, private rents and Rural Fuel Duty Relief flags from the spine, whose own stages set them in later commits of #931 group (c), and keeps the spine's council-tax bands. It already refuses, before it pins or reads any other input, a spine whose roster lacks those stages (`spine_posture`), so no dense build runs until they land. + +- **Nodes and kernels.** `uk.full.property.value@1`, `uk.full.property.band@1`, `uk.full.geography.brma@1`, `uk.full.geography.rfdr@1` and `uk.full.private_rent@1` leave `microcosm.build.uk_runtime.graph_population`, with the helpers that composed them, the property quantile artifact type (`microcosm.uk.property-quantile-draw`) and the pool checkpoint's typed edge to it. `append_uk_population_nodes` no longer takes `property_localisation_mode`, `brma_localisation_mode`, `private_rent_localisation_mode`, `rfdr_localisation_mode` or `full_scale`, and `uk_pool_validation_inputs` no longer takes a property mode. On the atomic assignment the dense graph's geography is now the identity, the shared support imports, assignment and derivation, the local-authority resolve, the shared gate and the UK distribution gate; the next change of #931 retires those too, when the dense line takes its areas from the spine. +- **Configuration.** `UKFullBuildConfig` loses `property_localisation`, `brma_localisation`, `private_rent_localisation` and `rfdr_localisation`, and with them its refusal of BRMA target rows without area BRMAs. The dense line no longer leaves its BRMA-grain rows out on either assignment: they bind on the spine's `brma` column, which a spine the dense line accepts reads off its areas. So `append_uk_target_nodes` loses its `brma_rows` switch and the target selection's payload its `brma_rows_left_out` field, which nothing could set once the configuration field went. +- **Flags.** `microcosm-build-uk` (`tools/build_uk_full.py`) no longer takes `--property-localisation`, `--price-paid-quantiles`, `--voa-band-mix`, `--scotgov-band-mix`, `--brma-localisation`, `--brma-area-layer`, `--private-rent-localisation`, `--brma-rent-quantiles`, `--rfdr-localisation` or `--rfdr-area-layer`, nor the six resource pins `--price-paid-quantiles-sha256`, `--voa-band-mix-sha256`, `--scotgov-band-mix-sha256`, `--brma-area-layer-sha256`, `--brma-rent-quantiles-sha256` and `--rfdr-area-layer-sha256`. A script that still passes one is refused as an unrecognized argument. The release-candidate refusals of the four `none` modes go with them: the spine-roster check holds every dense build, release candidates included, to a spine with the stages that set those columns. The build request's bindings lose their four localisation blocks, and the size-checkpoint identity loses the four blocks that bound those resources' pins. +- **Kept.** Every function the nodes called stays, with its tests: `owner_uniforms`, `localise_values`, `localise_bands` and `spine_household_keys` in `property_localisation`, `area_brmas` in `brma_area_layer`, `area_flags` in `rfdr_area_layer` and `localise_rents` in `private_rent_localisation`. The spine's geography and private-rent transforms (`spine_geography`, `spine_private_rent`), which no stage declares yet, call `area_brmas`, `area_flags` and `localise_rents`, and the `regional_home_values` stage calls `owner_uniforms`. (Later in #931 that stage retires, and the WAS chain's area home values call it: `931-spine-localises-home-values`.) (Later in #931 the packaged spine declares their stages: `931-spine-assigns-areas`.) The committed resources the nodes read and the terminal `uk_local_brma_enum_domain` gate stay too, though no dataset build reads the VOA or Scottish band mix now. (Later in #931 that gate retires, and the spine's `uk_brma_enum_domain` checks the column at its transferred boundary: `931-spine-localises-home-values`.) The unread mode tuples `BRMA_LOCALISATION_MODES`, `RFDR_LOCALISATION_MODES` and `PRIVATE_RENT_LOCALISATION_MODES` go; `PROPERTY_LOCALISATION_MODES` stays for now, since spine stages hash `property_localisation`. `private_rent_localisation`'s docstring now names the spine's private-rent stage as the caller that supplies the row keys and tie-break uniforms. `tools/uk_private_rent_pool.py` and `tools/uk_private_rent_receipt.py` now say they read only dense runs built before this change. +- **What moves.** Keys only, no value: the dense line already refuses every spine, the national line every clone-first one, no spine module changes, and a national build on a spine without clone indices computes what it did before. Of the 98 kernels the UK registries register (103 before), 30 change hash, all `uk.full`: the nine `graph_population` defines (sample, normalisation, expand, identity, locations, mapping, local authority, pool and the distribution gate); the fifteen that hash it (the dense measures and problem, the dense solve, the calibration install and the five size kernels, the rotated holdout, both export preparations, and the national problem, gates and read-back); the bound spine's checkpoint kernel (`uk.full.bound_spine@1`), which hashes `graph_build`; and the five that hash `graph_targets` but not `graph_population` (the target compilation and selection, the national targets, and the full battery's preflight and calibrated gate kernels). Each kernel that hashes `size_checkpoint` also hashes `graph_population`, and no kernel hashes `full_build_cli`, `rowwise_cli`, `orrery_contract`, `brma_area_layer`, `rfdr_area_layer` or `private_rent_localisation`. Of the gate phase codes only the full battery's terminal code moves, since it lists `graph_population`, so the calibrated gates' `terminal_code` parameter changes too. On a spine the dense build composes, every spine stage and both spine gates keep their keys and every dense node re-keys; on a bound spine the checkpoint re-keys, and every dense and national node after it. A dense run's input pins no longer carry the seven resources, so its input-pins digest, candidate identity digest and size-checkpoint identity change. No declaration, generated file, `gates.json` or schema changes. +- **Tests.** The node and configuration tests go: `test_uk_brma_area_graph.py` and `test_uk_rfdr_area_graph.py` are removed, as are the graph sections of `test_uk_private_rent_localisation.py`, `test_uk_property_localisation.py` and `test_uk_regional_home_values.py` and the CLI's localisation flag tests. Their function tests stay. New tests check that each retired flag is refused as an unrecognized argument, and that the atomic dense graph composes none of the five nodes, neither declares nor reads their resources, and the Orrery roster names none of them. The toy resource writers that only the nodes read leave `test_support/microcosm_build/uk_atomic_support_fixtures.py`. diff --git a/changelog.d/931-dense-needs-a-geography-first-spine.changed.md b/changelog.d/931-dense-needs-a-geography-first-spine.changed.md new file mode 100644 index 0000000000..ae7c19c683 --- /dev/null +++ b/changelog.d/931-dense-needs-a-geography-first-spine.changed.md @@ -0,0 +1,14 @@ +The dense line refuses a spine that does not assign areas, before it pins or reads any other input (#931; María's ruling of 10 Nov, decision 4). Later commits of #931 group (c) move the dense line onto the spine's own areas; refusing first means none of them can build a dense dataset on a half-moved line. `full_build_cli.prepare_full_build` now checks the spine's roster first: after the engine freeze, and before it pins an input, reads the spine, authenticates the checkpoint or composes a graph. It checks again the roster the build then binds. No spine built from this code passes yet. The check passes once the spine declares its area stages and the main-residence localiser, and a bound spine's sidecar records the geography block. (Later in #931 the packaged spine declares both: `931-spine-localises-home-values`.) + +- **The geography-first names** (`microcosm.build.uk_runtime.geography_first_roster`, new). A names-only leaf that imports nothing first-party: `GEOGRAPHY_FIRST_STAGES` (`geographic_expand`, `geographic_identity`, `atomic_area_assign`, `local_authority`, `rfdr`, `area_brma`, `spi_copy_areas` and `private_rent`), `GEOGRAPHY_FIRST_OPERATIONS` (`was_wealth`'s `localise_main_residence_value`) and `SIDECAR_GEOGRAPHY_KEYS`, the keys of the build sidecar's `geography` block (`assignment`, `identity`, `definition_sha256`, `support_pins`, `stream` and `register`). Its stage names are the ones the stages' transforms declare, and a test holds them together. (Later in #931 the transforms read their names from the leaf.) +- **The check** (`microcosm.build.uk_runtime.spine_posture`, new). `roster_failures(stages, operations, geography=None, *, from_sidecar=True)` names, in the leaf's order, every stage, operation and block key a roster lacks. A roster that is not names fails as a whole (`stage roster`, `operation roster`), an operation counts only in a stage the roster ran, and extra names never fail. Only the block's keys are read, never their values. `require_geography_first_spine` refuses a roster that lacks anything, in one sentence that names what it lacks and nothing else: no count, id or value. `stage_roster` reads declared stages' names and operation kinds, and `spec_roster` a spec's. +- **Both routes.** + - With `--input-h5` the check reads the build sidecar's `stages`, `operations` and `geography`, from `--input-sidecar` where it is named, before the ladder or the spine is pinned. An absent or unreadable sidecar is refused in `load_bound_spine_checkpoint`'s own words. The checkpoint is still authenticated after the check, and the sidecar it authenticates, the one the build binds, is checked again, so one changed since the first read is refused too. + - With `--spine-request` it reads the stage names and operation kinds of the packaged UK spec, whose whole roster every request runs, before the request is parsed or its spine composed. Its spine runs inside the dense build's own graph, so it has no block to read. The spine the request composes is checked again, on its own stages, before the build reads it. + - A dry run, with or without `--candidate-clone-counts`, is refused alike. The national line and the row-wise dataset tool (`tools/build_uk_rowwise_dataset.py`) are unchanged. +- **What moves.** No line builds differently: the dense expansion already refused, at any clone count, a spine that carries the geographic clone indices, as every spine built since #931 group (b) does. The check also refuses a spine without them, one built before group (b), which the dense line could still bind until now. The expansion refused only once the graph reached it, so a dense dry run could still plan on a clone-first spine; dry runs and builds now both stop at preparation. A refused build records its attempt as failed, with the refusal as its message. `tools/measure_uk_atomic_assignment.py`'s graph route runs the dense preparation, so it is refused the same way; later in #931 the tool measures the spine's own draw with no graph. No node hashes `full_build_cli` or the two new modules, so no node key moves. No declaration, generated file, `gates.json` or schema changes. +- **Tests** (invented rosters, rosters read off the packaged spec, and the spine build's own sidecar writer over a synthetic frame; no built spine's sidecar). + - The packaged roster, and the sidecar the spine build's writer records for it, are refused naming the six area stages and the localiser, and the sidecar also the block. (Later in #931 the localiser alone, and then nothing: `931-spine-localises-home-values`.) + - Each missing name fails alone, in the leaf's order; unreadable rosters fail by name; a geography-first stand-in passes, with extra names too; no refusal carries a figure. + - Through `prepare_full_build` on both routes, recording stand-ins, under which the real request-roster reader runs, show the refusal comes before any input is pinned or read and before any graph is composed, and that a geography-first stand-in goes on to pin its first input. A sidecar block that lacks any one key is refused naming that key; an unreadable sidecar is refused word for word as the bound loader refuses it. A roster that passes the first check but not on the authenticated sidecar or the composed spine is refused before the spine is used. Through `main`, a refused build records a failed attempt without running a graph. + - The stand-in sidecars of the preparation tests and of the dense dry-run test carry a geography-first roster (`geography_first_sidecar_fields` in `test_support/microcosm_build/uk_full_build_cli.py`). diff --git a/changelog.d/931-dense-on-the-clone-first-spine.changed.md b/changelog.d/931-dense-on-the-clone-first-spine.changed.md new file mode 100644 index 0000000000..60553dc758 --- /dev/null +++ b/changelog.d/931-dense-on-the-clone-first-spine.changed.md @@ -0,0 +1,18 @@ +Build the dense line on the bound spine's clones, areas and sampling (#931). The dense line no longer samples, expands, keys or places anything of its own: its targets, measures and problem run on the spine's version as it stands, its pool is that version whole, and its clone count K, its sampling rung and its geography binding are the spine's. It still refuses every spine whose roster is not geography-first (`spine_posture`), so no dense build runs until the spine declares its area stages. (Later in #931 it declares them, and the dense line waits on the WAS chain's home-value localiser alone: `931-spine-assigns-areas`; then the chain declares it, and the posture check passes on the packaged spine: `931-spine-localises-home-values`.) + +- **Graph.** `uk.full.sample`, `uk.full.normalize`, `uk.full.expand` (with `uk.full.expand.owned`), `uk.full.identity`, `uk.full.locations`, `uk.full.geography_mapping`, the shared support imports, assignment, derivation and gate (`uk.full.geography.*`), `uk.full.geography.local_authority` and the UK distribution gate `uk.full.geography_gate` leave the dense graph, with their eight kernels and the population receipt, location draw and geography gate artifact types. `microcosm.build.uk_runtime.graph_population` keeps `context_frame`, `population_columns`, `population_slices` and the pool checkpoint, `uk.full.pool@1`, now a keep-all FILTER on `uk.full.spine_checkpoint` (`--input-h5`) or the spine's endpoint (`--spine-request`) with no typed edge, declared bitwise and hashing its own module and `national_frame`. `append_uk_population_nodes`, `uk_pool_validation_inputs`, `GEOGRAPHY_ASSIGNMENTS` and `UK_GEOGRAPHY_IDENTITY_INPUTS` go. The UK registry no longer registers the shared atomic-geography kernels; NZ's transport registers its own. The dense geography gate's ladder and installed-engine local-authority checks (#953) and the pool's refusal of a failed shared geography validation leave with their nodes; the spine's own geography gates take them over when they are registered later in #931, and until then the roster refusal holds every dense build. +- **Targets.** `append_uk_target_nodes` takes `population` with no default, and the full build passes the spine's version. The measures lose their edge to the dense geography gate and their check of it, and take the spine's transferred-gate admission the sample node held, where the graph declares the spine's gates: they refuse to read the version unless that stored report admits it, before they read anything else. +- **Configuration.** `UKFullBuildConfig` loses `sample_fraction`, `sample_seed`, `geography_assignment`, `source_vintage`, `constituency_vintage` and `source_lineage_modulus`. `n_clones` (default 1, the declared spine's) is the bound spine's K and `effective_sample_fraction` is `source_sample_fraction`, the spine's rung. `uk_full_graph` no longer takes `spine_weight_kind` or `atomic_geography_definition`, and refuses a composed spine whose `geographic_expand` node declares another K than the configuration's, naming both. `build_uk_country_graph` no longer takes `atomic_geography_definition`. +- **K, sampling and geography from the spine.** `prepare_full_build` reads them from the authenticated sidecar (`geographic_clone_count`, one where a sidecar predates the record; `sampling`; `release_posture`; the `geography` block) or from the spine request's `--n-clones`, `--sample-fraction` and `--sample-seed`, and sets the run's `n_clones`, `sample_fraction` and, where the spine was sampled, `sample_seed` to them before the run's parameters, its candidate identity digest, its size-checkpoint identity, its staging rung and both Logbook rungs record them. A spine whose sampling fraction is not a scale-ladder rung is refused there, naming the rungs. The attempt's build id, minted before preparation reads the spine, takes the spine's rung read ahead off the bound spine's sidecar or the spine request's `--sample-fraction` (f100 where that read fails, and preparation then reports the real error); preparation refuses an attempt whose build id carries another rung than the spine it binds, naming both rung tokens, and a refusal before the spine is bound records the build id's rung on its Logbook row. The request bindings gain a `spine` block (clone count, sampling and release posture), and their `geography` block is the spine's binding with its clone count beside it. The rowwise manifest reads its `sampling`, `identity.sampling`, rung and release verdict's full-rung leg from that spine block and its `geography.ladder_gate` from the export descriptor's geography check on the calibrated frame; its weight-kind chain names `geographic_expand` for the clones. +- **Bound-spine checks.** `_assert_spine_sidecar_binds_frame` also holds the sidecar's `geographic_clone_count` to the frame's clone count, naming both, and the bound spine's checkpoint kernel hashes `geographic_clones` for it. `spine_provenance_from_sidecar` adds the spine's clone count, `geography`, `sampling` and `release_posture`. The national build record, the national calibration diagnostics' build block and the dense manifest's `identity.spine.spine_provenance` carry that provenance, so they gain the four fields; a national build on a spine without clone indices records K = 1 and computes the same dataset. +- **Flags.** `microcosm-build-uk` no longer takes `--geography-assignment`, `--atomic-support-ew`, `--atomic-support-scotland`, `--atomic-support-ni`, `--atomic-support-sha256-ew`, `--atomic-support-sha256-scotland`, `--atomic-support-sha256-ni` or `--source-lineage-modulus`; a script that still passes one is refused as an unrecognized argument, on either role. The dense role refuses a `--sample-fraction` other than 1.0, any `--sample-seed`, `--candidate-clone-counts` with `--input-h5`, and `--n-clones` with `--candidate-clone-counts`, by name, before any input is read. `--n-clones` is an assertion of the spine's K: the dense role no longer fills it from the doctrine, and preparation refuses one that differs from the spine's, naming both, and on a spine request pointing at the request's own `--n-clones`. `--candidate-clone-counts` plans the spine request once per count, each as the request's own `--n-clones`. The engine blocks are held to one or the spine's K by the configuration; on validation only an asserted `--n-clones` is compared. `--expected-constituency-vintage` is still taken and recorded in the run's parameters, but its one reader, the legacy location draw, is gone, so no dense node checks the ladder against it; its help says so. (Later in #931 the flag retires, as María ruled on 13 Nov: `931-spine-localises-home-values`.) +- **Release-candidate refusals.** A sampled spine is refused as a release candidate at preparation, where its rung is read, on either route. The release-candidate refusal of `--n-clones` other than the doctrine's leaves validation, as do the support pins' requirement and the legacy assignment's refusal; `--sample-fraction` other than 1.0 is refused on every dense build (above). The spine's clone count is held to the doctrine's, and its geography binding to the conditioned register, by later commits of #931. The ladder, Ledger and input pins, the target rule, the epochs, the measure exclusions, the local target levels and the signing key are refused as before. +- **The dense sampler.** `national_sampling` keeps the rung tokens, the default seed and the spine root's sampler; `sample_uk_spine_frame` and `uk_spine_source_family_units`, which drew whole source families for the dense line, go, with their re-exports. +- **Texts.** The row-wise lane's clone refusals (`rowwise_dataset`) point at the dense graph on a geography-first spine and promise no later group; `tools/build_uk_rowwise_dataset.py` says it serves spines without clone indices. `geographic_identity`, `household_lineage`, `geographic_clones` and `diagnostics` no longer describe the dense identity node as a key's writer or a column's reader: `household_lineage`'s three lineage refusals no longer open "Geography identity", its `UK_OPTIONAL_LINEAGE_COLUMNS`, which nothing reads now, goes, and `geographic_clones`' docstring names the national gates among the kernels that hash it. `tools/measure_uk_atomic_assignment.py`'s graph route, which ran the dense line's geography nodes, refuses with a pointer to the spine's own draw, its command before it reads any input, and the helpers that composed its requests go; later in #931 the tool measures the spine's own draw in process. The Orrery roster loses its geography group and the retired population nodes. +- **What moves.** Keys and records only; no dataset value. No spine stage's code changes: `household_lineage`, `geographic_clones` and `geographic_identity` change in docstrings, comments and refusal texts (and `household_lineage` loses a constant nothing reads), and `national_sampling` loses only the dense sampler, which no spine stage calls; the dense line still builds on no spine. On the synthetic H2 fixture the legacy oracle's final frame, stage records and stage evidence are identical before and after. + - Of the kernels the UK registries register, 86 remain (98 before: the eight dense population kernels and the four shared atomic-geography ones go), and 68 change hash: 43 of the spine registry's 60 (the root, `uk.create@1`; the 39 stages that hash `household_lineage`, `geographic_identity` among them; and `cgt_support_split`, `was_wealth` and `was_lisa`, which hash `geographic_clones` alone), and the 25 `uk.full` kernels that hash an edited module. `uk.spine-gates@1`, `uk.claim@1`, `uk.identity@1` and 15 stages keep theirs. A spine build's root also hashes `national_sampling` through its sampler. So every spine node re-keys, and every dense and national node after it. + - The pool was the last platform-bitwise UK kernel, so no dense node key carries the platform now (on the declared spine 16 of the dense graph's 23 nodes did); a shared graph store can serve them across platforms on their own bitwise declarations. + - Of the gate phase codes only the full battery's terminal code moves. + - A dense run's input pins no longer carry the three atomic-area supports, its recorded parameters drop `source_lineage_modulus`, and its size-checkpoint identity drops `source_lineage_modulus` and the `geography` block, so its input-pins digest, candidate identity digest and size-checkpoint identity change, and a size checkpoint cut before this change does not resume. The bound spine's provenance gains four fields (above). + - The coverage manifest's `--check` passes; no declaration, generated file, `gates.json` or schema changes. +- **Tests.** `test_uk_full_population_graph.py` and `test_uk_atomic_identity_graph.py` are removed with the nodes they ran, and `test_uk_national_sampling.py` now tests the root's sampler. Their tests of live code move: the lineage reader's branch paths, residential arms and refusals to the new `test_uk_household_lineage.py`, which runs `uk_household_clone_paths` itself, and the row-wise lane's clone and source-lineage refusals to `test_uk_rowwise_dataset.py`. The full-graph stand-in (`test_support/microcosm_build/uk_full_population_graph.py`) is a clone-first, geography-first bound spine: the toy households at K clones in the copy-id table's geographic field, keyed as `geographic_identity` keys them and placed on the toy supports with the shared operators, with a BRMA and a fuel-relief flag; a bound checkpoint of it takes a stand-in sidecar whose geography block is under the leaf's keys. New tests cover each retired flag, the dense role's spine-bound refusals on both routes, K, sampling and seed taken from the spine on both routes (a seven-clone spine without `--n-clones` records seven, in the parameters, the identity digest and a written manifest), a disagreeing `--n-clones`, a sidecar whose K is not its frame's, the engine blocks held to the spine's K, `--candidate-clone-counts` planning a spine request once per count, a release candidate on a sampled spine on both routes, a spine off the scale ladder, the build id's rung read ahead and held to the spine's, the measures' admission, the pool on the spine's version and its keep-all mask, the manifest's spine-bound sampling and ladder gate, the dense allow-list pin (every column a geography-first spine adds to the export surface is in `UK_ALLOWED_EXTRA_EXPORT_COLUMNS`), the local area support gate's row, effective-sample-size and source floors in the K-sensitive constants table, and the measurement tool's refusal. The `geographic_identity` stage's keys are held to `household_draw_key` itself. diff --git a/changelog.d/931-engine-batches-read-year-window.changed.md b/changelog.d/931-engine-batches-read-year-window.changed.md new file mode 100644 index 0000000000..1f56f84362 --- /dev/null +++ b/changelog.d/931-engine-batches-read-year-window.changed.md @@ -0,0 +1 @@ +The UK spine's per-clone engine batches go through the read-year window of microcosm#1615 (#931). Each clone's simulation is built by the same construction as a call's one simulation: `PolicyEngineUKEngine._read_year_simulation` projects the clone's dataset to the call's year on the adapter's own parameters, stops the Universal Credit rebalancing loop at that year, and constructs the simulation through `_simulation`, which filters policyengine-uk's `EntityIdWarning`. An engine block's annual-income and counterfactual simulations are built by the block's wrapped factory inside the measure resolver's cut. An engine block computes at no dated measure's year after the resolver's: its scratch engine holds no later year and the resolver refuses such a measure, so the block neither checks the engine's growth nor sets the pool's shares there. Every dated target in the contract is a 2024 one, so a block at 2024 or later keeps its periods. policyengine-uk builds each year only from earlier ones, so no value at the call's year moves. Tests (engine-free, made-up data): both adapter routes project, open the cut and construct in that order, with `_simulation` constructing while the cut is open, an engine block's later simulation is built inside the resolver's cut, and a contract target dated after the resolver's year is none of its block's periods. diff --git a/changelog.d/931-engine-simulation-rank-ids.fixed.md b/changelog.d/931-engine-simulation-rank-ids.fixed.md new file mode 100644 index 0000000000..09e228d840 --- /dev/null +++ b/changelog.d/931-engine-simulation-rank-ids.fixed.md @@ -0,0 +1,14 @@ +Each policyengine-uk simulation the UK adapter builds now holds ids that policyengine-core's 32-bit store keeps apart (#931; decision 12, María's rulings of 11 Nov). policyengine-core stores policyengine-uk's five id variables (`person_id`, `benunit_id`, `household_id`, `person_benunit_id` and `person_household_id`) as int32, converting them with numpy's `astype`, which wraps an id outside the type without an error. It builds the entities and memberships from the int64 columns it is given, so two ids that wrap to one value merge no entity: they become one stored key for every formula that reads the id. policyengine-uk 2.174.1 hashes its Warm Home Discount draw on `household_id`, its two Universal Credit deduction draws on `benunit_id` and its birth-month spread on `person_id`. Anything rebuilt from the stored values would see one entity. Within one clone's batch every id carries the same clone offset, so a repeat there needs a copy-layer ordinal on at least one of the two ids. + +- **The rule** (`microcosm.frame.adapters.engine_identifiers`, shared, for Max's review). `EngineIdentifiers(stored_dtype)` declares the integer type an engine stores a schema's id and membership columns as. `engine_identifier_tables(schema, tables, identifiers)` hands a simulation the tables it is given when no entity's ids repeat once converted as the engine converts them (`stored_identifier_repeats`). An entity whose stored ids would repeat takes 1 plus each id's rank among that simulation's ids, in id order, and each person's membership of it takes the new id of the group it names, never the 0-based rank (`rank_identifier_tables`, the step that ranks and maps and decides nothing). Ranks are distinct, fit the type and keep id order, so the engine holds every entity's rows in the frame's order, and results still return by position or by the frame's own ids. The frame is never changed. The module also names the entities whose ids the conversion changes (`inexact_identifier_entities`), refuses a request for such an entity's id or membership variable (`refuse_inexact_identifier_requests`, which reads no table unless an id or membership variable is requested), and holds a declaration to an installed engine's types (`require_stored_dtypes`). It reads a schema and entity tables, never a Frame, and names no country. It refuses to renumber an entity that a declared link's table among the tables names, since it maps only the person table's memberships. Its refusals name entities, links, columns, variables and types, never an id or a count. national_frame's release rule keeps its own ranker for now; moving it onto `rank_identifier_tables`, and deriving `UK_ENGINE_IDENTIFIER_MAX` from the declaration, are owed (decision 7). +- **The UK adapter.** `UK_ENGINE_IDENTIFIERS` declares int32, and `_require_identifier_dtypes` checks it against the installed engine's five id variables once per adapter; a test holds national_frame's release range (`UK_ENGINE_IDENTIFIER_MAX`) to the same width. `_dataset_tables` applies the rule to `engine_tables`' tables, and both construction sites build through it, so a call run clone by clone ranks each batch within its own rows. `materialize` refuses, before any engine is built, a request for an id or membership variable whose entity holds an id the engine cannot store exactly, since the engine would return wrapped ids or ranks for it. Each simulation is built with policyengine-uk's `EntityIdWarning` filtered, since the adapter now checks the stored ids itself, and the adapter refuses to build one if policyengine-uk no longer defines that class at `utils.entity_ids`. Building a dataset now builds the adapter's `CountryTaxBenefitSystem` first, which is inert. The stages that call the engine bind the new module beside `microcosm.frame.materialize` (`_ENGINE_COMPUTE_MODULES`), as the walk from the engine the builds inject reaches it. +- **What moves.** + - Nothing on a frame whose stored ids do not repeat: `UKSingleYearDataset` is built from the same tables, byte for byte, so every engine output is the same. + - Elsewhere, only that simulation's values keyed on the renumbered entity's ids. For households that is the Warm Home Discount draw, which runs for 2025 to 2030 and adds only to HBAI income, which spine stages request only at 2024. A batched call's id-keyed values can differ from one simulation's over the whole frame wherever either renumbers, since each batch ranks its own rows. + - The adapter's simulations no longer write `EntityIdWarning` to stderr, and an id request for an entity the engine cannot store exactly is refused; no stage makes one. + - Every UK graph node re-keys: the kernels that hash the adapter directly, and every other node through its population's version. No other country's keys move. The US annual static-aging manifest's `source_sha256`, which hashes every module under `microcosm.frame`, gains the new file and changes the adapter's entry; it is provenance, not a node key. No declaration, generated file, `gates.json`, schema, receipt or sidecar field changes. +- **Tests** (invented data only). + - The shared module (`tests/engine_free/shared/test_engine_identifiers.py`): pass-through, the same table objects, on ids inside int32 and on copy-layout ids past 2**31 that convert out of int64 order; a stored repeat renumbered by rank, with 1-based memberships; a seeded sweep of a few hundred frames under int8 and int16 declarations, over NumPy and nullable, signed and unsigned id dtypes, against the tests' own reading of the conversion; inexact entities and request refusals; the rank step's refusals, a link table naming an entity to renumber among them; and refusals checked by exact string to quote no id, wrapped value or row count. + - The adapter, engine-free: the declaration, its once-per-adapter check, and the dataset builder holding it before it builds anything; on ids past 2**31 that stay apart, `_dataset_tables` hands on the very tables `engine_tables` made; each batch ranks its own rows, through `materialize`'s batched site and the dataset builder too; the request refusals; and the warning filter fails closed at both construction sites. The binding walk gains the module, and the two widths agree. + - Through the engine (`tests/engine/uk/test_policyengine_uk_identifiers.py`): the declaration is the installed engine's. On copy-layout ids every value, the id-keyed draws included, is a direct Microsimulation's over the frame's engine tables at 2024, at 2025 and at 2025 projected from 2024, with each household's member count and an echoed household input in row order. Households 7 and 7 + 2**32 take different Warm Home Discount draws, and every value is the same call's over the frame renumbered by hand: at the dataset's year, projected through policyengine-uk's council-tax and student-loan joins on the raw memberships, and clone by clone at two clones. These three fail where the adapter passes the frame's ids through unchanged, and a 0-based membership map fails the first two. The warning is filtered at both construction sites, while a direct Microsimulation over the same tables still records it. + - The H2 parity test records, as two booleans, that its simulations reach the rule and that none of them renumbers. diff --git a/changelog.d/931-engine-uk-after-merge-ci.added.md b/changelog.d/931-engine-uk-after-merge-ci.added.md new file mode 100644 index 0000000000..6a378fc288 --- /dev/null +++ b/changelog.d/931-engine-uk-after-merge-ci.added.md @@ -0,0 +1 @@ +A new CI job, `engine-uk-after-merge`, runs on pushes to main only, after merge (#931). It runs the new `engine_after_merge/uk/` test category, UK engine tests too slow for every pull request. Its first module is the clone-first H2 test, which runs the whole UK spine at one and at four geographic clones through the engine, about 27 minutes per leg. `tools/run_engine_uk_tests.sh` takes the job whose files it runs, `engine-uk` by default. `tools/ci_test_plan.py` marks the category's group `after_merge`, so the plan gives the job the push condition as its `if` and no selection need; the `ci` aggregate needs it, so a push must run it and pass, and `require-results` passes its skip on a pull request (`GITHUB_EVENT_NAME`). `verify-workflow` refuses a job that needs it unless that job runs after merge too. diff --git a/changelog.d/931-fold-lowest-id-sorted-keys.changed.md b/changelog.d/931-fold-lowest-id-sorted-keys.changed.md new file mode 100644 index 0000000000..0b2adc20e5 --- /dev/null +++ b/changelog.d/931-fold-lowest-id-sorted-keys.changed.md @@ -0,0 +1,21 @@ +The UK fits now train each variant of a survey record on its lowest-id clone instead of its first row, through a shared fold that takes the order from its caller and reports what the clone receipts will need; `integer_record_keys` finds repeated ids by sorting (#931; the sort as María ruled on 10 Nov). No output moves: every spine is one clone today, so every fit keeps its rows and weights value for value, and every id check on integer ids gives the verdict it gave. + +- **`fold_replicas`** (`microcosm.build.source_lineage`) returns a `ReplicaFold` with one value per record in each field: + - `kept`: the rows the fit trains on; + - `weights`: on each kept row, its variant's selected weights summed exactly (`math.fsum`); every other weight unchanged; + - `representative`: the row each selected record's variant trains on, -1 off the selection; + - `varied`: whether a selected record's key holds more than one variant; + - `key_group`: a code of each selected record's replica key, -1 off the selection. + + A variant's representative is its selected record with the least `order`, ties going to the earlier row. `order` is one numpy integer per record, and a malformed one, a timedelta included, is refused with counts only. Without `order` the representative is the variant's first selected record in row order, as before. A selected record with a null key, which `replica_keys` never gives, is now refused rather than folded with other records. +- **Early returns decided on the key.** When no two selected records share a key, the selection and weights come back value for value and nothing varies. When keys repeat but no two selected records share a variant, the selection and weights still come back value for value, and the records of every key with more than one variant vary. Deciding on the key and the inputs together would report no variety exactly when every clone differs. +- **`collapse_replicas`** is a wrapper that returns `fold_replicas`'s kept rows and weights. It takes the same keywords as `fold_replicas`, `order` included. Without `order` a variant keeps its first selected row, as before, so every caller and fixture keeps its values. +- **The UK entry.** `household_lineage.collapse_geographic_clones` keeps its signature and return, and orders the fold by the entity's id column. The new `fold_geographic_clones` returns the whole `ReplicaFold`. A key's clones differ in id only by their clone's offset, so each variant trains on its lowest clone among the selected rows: clone 0 whenever clone 0 is selected and agrees with it, whatever the row order. Before, it trained on whichever of its clones came first. +- **`integer_record_keys`** (`microcosm.build.stochastic_assignment`) compares neighbours in a sorted copy of the ids instead of building `np.unique`'s array. On integer ids its verdicts, errors and return are unchanged. Timedelta ids, which numpy types as integers and the check used to accept, are now refused as ids that are not integers. +- **What moves.** No output, at one clone or on any spine. On frames whose clones are out of clone order, which only tests build today, a variant now trains on its lowest-id record. Node keys move: + - every UK spine node, since the spine's root binds `stochastic_assignment`, `source_lineage` and `household_lineage` and keys chain; with them the spine gate phases (the transferred phase's own code also hashes `stochastic_assignment`) and, in the in-graph build, every node after the spine; + - on a bound spine, whose root hashes none of the three, the full build's `uk.full.identity`, `uk.full.property.value` and `uk.full.private_rent`, which hash `source_lineage` and `household_lineage`, and every node keyed on them; + - the national line's gates, whose registry modules include `stochastic_assignment`. + + The spine checkpoint's gate digests do not move. The derived binding tables are unchanged, since the new names live in modules already bound wherever they are reached. No generated file or `gates.json` changes: `uk_spine.json` and the H2 fixture hold stage contract hashes, not code hashes. Of the transport kernels only the gate battery (`gates.battery@1`) re-keys: its key binds the sources its gate bindings compute through, US keyed draws among them, and since microcosm#1585 those import `stochastic_assignment`. So NZ's terminal gate node re-keys, and so do the export, readback and package nodes that chain on it. No other NZ node and no US or BE node hashes an edited module or chains on one, so their node keys stay. The US seed protocol attests `stochastic_assignment` (microcosm#1585), so its and the seed map's digests move, with the US bundle's spec hash (later in #931 these pins are re-derived), and the US PUF-support builder's whole-tree resume fingerprint moves, as it does on any edit to packaged source. +- **Tests.** The fold is checked against a record-by-record statement of its contract on random frames; under every row order of a three-clone frame; per variant; on a frame without replicas; and on a two-clone frame whose clones all differ, which folds nothing and varies everywhere. Without `order` it keeps the earlier tests' rows and weights. The UK entry keeps clone 0 when every table lists its clones in reverse, and the id check gives `np.unique`'s verdict on random integer arrays of several dtypes and refuses timedeltas. The test that holds every folding stage to a live lineage now finds a stage through either UK entry. diff --git a/changelog.d/931-isa-tied-balance-rank.fixed.md b/changelog.d/931-isa-tied-balance-rank.fixed.md new file mode 100644 index 0000000000..a466420d9e --- /dev/null +++ b/changelog.d/931-isa-tied-balance-rank.fixed.md @@ -0,0 +1,9 @@ +The UK ISA subscriptions stage (`isa_subscriptions`, stage 54) gives every holder tied on a balance one weighted mid-rank (#931; María's ruling 7 of 7 Nov). Half of each subscriber's ISA amount key, and of each Lifetime ISA payer's payment key, is the person's balance rank, blended with a keyed uniform. The rank sorted balances stably and gave each row its own mid-rank, so tied balances ranked by row position, although its docstring said ties share a rank. Ties are common: the copies and clone halves made after the WAS wealth draw carry their root's ISA totals, so a single-adult one ties its root, and QRF draws repeat a few atoms. A root sits before its copies in the table, so it ranked below them, as clone 0 would below its clones once the spine clones households at its start. The stage was also not invariant to row order where balances tie: on a synthetic walk of 400 holders with balances on five atoms, permuting the rows moved 372 amounts, and none with distinct balances. + +- **The rank.** `_weighted_rank(values, weights, ids)` orders the rows by (value, id). Every member of a tie group takes the group's mid-rank, `(F(x-) + F(x)) / 2` over the total weight. A value held by one row keeps its own, `(c - w / 2) / total`, computed as before. Both callers pass the person ids, so the keyed uniform alone orders tied holders, exchangeably, and each rank is the same bit for bit whatever the row order. +- **No seed; the notes say it.** The rank draws nothing, so no seed, salt or operation key changes. Stage 53's notes now say that tied balances share one weighted mid-rank, so the keyed uniform alone orders them, and that the Lifetime ISA payments, whose operation declares no rank weight, reuse the amounts operation's `balance_rank_weight` (María's ruling of 10 Nov: in the notes, not a declared field). The notes change in `sources.yaml`, `source_stages.json` and the H2 fixture's declarations; notes enter no stage contract, so the stage's contract hash does not move. The band walk and the Lifetime ISA strata stay pooled. +- **What moves.** Nothing where no balance ties: on three synthetic frames of distinct balances every output is unchanged, bit for bit. Elsewhere: + - Tied holders' ranks move. A singleton's rank is unchanged bit for bit wherever the rows arrive in id order, as the spine's tables do, but its amount can still move, since the band walk and the within-band order are pooled over the holders a tie moves past. + - On three synthetic frames with balances on five atoms, every tied holder's rank moved; 248 to 260 of the 363 to 376 tied subscribers and 329 to 344 of the 420 to 430 other subscribers moved amount; and 46 to 56 of the 52 to 68 Lifetime ISA payers, all of them tied, moved payment. No balance moved. + - With three geographic clones per holder on five atoms, clone 0 now gets the least of its three in about a third of the holders, against about 43 per cent (ISA amounts) and 40 per cent (Lifetime ISA payments) before. + - Of the 101 UK kernel hashes only stage 54's moves. Node keys move from stage 54 on: 4 of the gated spine's 83 nodes and 36 of the in-graph build's 115. The national line on a bound spine keeps its keys. diff --git a/changelog.d/931-keyed-tie-breaks.changed.md b/changelog.d/931-keyed-tie-breaks.changed.md new file mode 100644 index 0000000000..aa5262a866 --- /dev/null +++ b/changelog.d/931-keyed-tie-breaks.changed.md @@ -0,0 +1,19 @@ +Six walks on the UK spine that ordered exactly tied rows by id now order them by an identity-keyed uniform, then by id (#931; accepted as designed on 7 Nov). Ties broken by id alone put the lowest id first every time, and the copies and geographic clones of one household differ only in their copy ordinal, so one copy ordinal won every tie: the first copy or clone at stages 28, 42 and 44, and the last at stage 26, which takes the top of an ascending order. On synthetic clone pairs the lower clone index ranked first in every tied pair, and in about half of them once ties are keyed. Values move only where an exact tie touches a cut or a position schedule. + +- **Shared helper.** `microcosm.build.stochastic_assignment.keyed_tie_order(primary, ids, *, seed, salt)` returns `np.lexsort((ids, u, primary))`, with `u` the records' `keyed_tie_uniforms(ids, *, seed, salt)`: `stable_identity_uniforms` of `integer_record_keys(ids, label=salt)`. It is a secondary sort key, the form stage 27 already uses, so on distinct values the order is the id order's bit for bit; it draws the uniforms of tied records only. Each record is keyed on its own id, never on `spine_household_keys`, which strips the geographic branch and would tie the clones again, and the rank-inside-a-tie form is not used, since it moves records that have no tie off their cut. +- **Stage 42** (`cgt_support_split`): households tied on investable wealth, seed 0, salt `cgt_support_split:wealth_tie`. High-value families tie exactly on the four wealth columns, since no stage from 26 to 42 writes them. +- **Stage 26** (`high_value_home_support`): owners tied on home value before the weighted mid-rank, which is unchanged; seed 0, salt `high_value_home_support:value_tie`. +- **Stage 28** (`property_components`): landlords tied on profit in the receipts walk, under the walk seed and a new `tie_salt`, `allocate_property_rental_income:tie`. The stage refuses an empty tie salt and one equal to a salt its finance-cost draws key the same person ids on. The walk's one offset stays as it is (ruling 6). +- **Stage 44** (`hmrc_cgt_gains_spine`): one uniform per person under the stage's seed (552), salted `hmrc_cgt_gains_spine:rank_tie:`, drawn once per stage call and read by every cell walk, pooled walk, plan and the sub-AEA remainder, which take it as `_ranked`'s keyword-only `tie` (without it, person_id alone breaks ties, as before). The plans and the remainder walk the rank ascending, so a tie's member with the higher uniform takes the lower stratum. +- **Stage 47** (`hmrc_cgt_asset_type_spine`): the BADR walk and the main asset type walk, under the BADR seed (555), salted `assign_badr_qualifying_gains:tie:` and `assign_main_asset_type:tie:`. +- **Declarations.** The two support splits declare their tie-break as new `seed` and `salt` keys and the receipts walk its `tie_salt`, in the code dictionaries, the strict schema (three operations), `sources.yaml` and `source_stages.json`; the selection, stratum rule, order, ordering, mapping, realization and values texts, the stage notes and the receipts walk's and asset types' receipt order texts say the above, and "no draw, no seed, no salt" and "deterministic, no seed" are gone. The build sidecar's declared seeds register the splits under their salts and the receipts walk's seed under its tie salt. +- **Stage 42's rule B.** The split's walk, `keeps_the_split`'s rerun of it and the identity tool's recompute all pass the declared pair (`cgt_support_tie_break()`); the selection takes no default, so neither the rerun nor the tool can walk ties in another order. The exact-total correction's candidate order (the heaviest households left whole first, tied weights in row order) is unchanged: it picks the row of a rounding-sized correction rather than drawing anything. `copy_rule` names the cap of 64 candidates and their order, heaviest first, tied weights in row order, so of a whole household's geographic clones, which weigh the same, the one in the earliest row is tried first. +- **No nesting promised.** All six walks are pooled. The uniform is a function of each row's own id, so clone 0's equals the `K = 1` row's and every clone draws its own. +- **What moves.** Nothing where no exact tie touches a cut or a position schedule. Elsewhere, counted on synthetic frames: + - At stage 42, a band's crossing household when it sits in a tie group: with three clones per household, 2 to 8 of about 850 households per frame changed copy count, and clone 0, which carried the most families in all six frames, carries the most in one. At one clone none moved on those frames, whose synthetic tied families are light, most members at or under 60, the weight above which a selected household takes a second copy. Where a band's crossing falls in a tie group whose members weigh more, as a high-value family's can, the keyed order changes which members are copied at one clone too. + - At stage 26, the tie group across a region's cut: 0 to 5 of 3,000 owners per frame. + - At stage 28, no untied landlord changes band, since the same landlords sit above it in either order. Where a tie group shares one weight, as the copies of one household do, only its own landlords move: on three frames of six-row families, about two thirds of the tied landlords' receipts moved, none outside the tie groups, and the band counts held. Where a tie group of unequal weights straddles a band boundary, as different households reporting one profit do, the keyed order puts another tied landlord at the boundary. Tied landlords then cross it, the bands' record counts and masses change, and the receipts of the other landlords in the bands on either side move with them; inside a band whose members hold, a reordered tie group can still move later landlords' receipts by rounding. On three frames of 2,400 landlords, half of them on 60 round profits at weights that differ, 24 to 49 tied landlords changed band and the record counts of 4 to 7 of the 11 bands changed; no untied landlord changed band, and the receipts of 398 to 941 of the about 1,200 untied landlords moved, by at most about 0.3 per cent. + - At stage 44, often, since the blended rank key ties gainers even on distinct inputs: 43 to 99 of about 1,800 gainers moved amount where wealth varies, and none where it does not. + - At stage 47, nothing, since plan amounts do not tie (three frames bit for bit). + - Node keys: every spine node re-keys, since `stochastic_assignment` changed and the spine's root binds it; values move from stage 26 on. On a bound spine the national line re-keys 5 of its 6 nodes, all but the bound checkpoint, since the national target, problem and gate kernels hash the country spec, which this change edits. + - Regenerated: `source_stages.json`, the coverage manifest's 25 source digests, the H2 fixture's declarations and the five stages' contract hashes in `uk_spine.json`, and the 39 values of the synthetic facts its tool sizes to the frames stages 28, 29 and 46 read. diff --git a/changelog.d/931-national-one-clone-areas.changed.md b/changelog.d/931-national-one-clone-areas.changed.md new file mode 100644 index 0000000000..31aa0a1b03 --- /dev/null +++ b/changelog.d/931-national-one-clone-areas.changed.md @@ -0,0 +1,28 @@ +The national line builds on a geography-first spine of one geographic clone (#931). Every spine since #931 group (b) carries the geographic clone index, and the national line refused all of them; it now takes the spine #931 group (c) builds, one that assigns every household its area, at one clone, and refuses any other. It also refuses every build until the degenerate-surface register's `household.atomic_area_basis` exclusion is signed for the national line and in force on the build's review date. That entry lists the local line alone at this change, so no national build runs; the register decides, and signing the entry for the national line, with the contract re-pin, lifts the refusal with no code change. The #931 commit that follows signs that entry for the national line. A national build writes its release without the spine's geographic clone indices, which are constant at one clone, and with the area columns the spine assigns, the three area codes under the consumers' names; the gates that read what the release ships read that export. The release certifier holds the spine's geography binding to a release's, as the dense release pre-flight does, before any gate runs, and refuses a candidate that carries a clone index. + +- **The national problem** (`graph_national`). `refuse_clone_first_national_spine`, which refused every spine carrying the geographic clone index, retires. The national problem, the first national node that reads the bound frame, now reads the bound spine's build sidecar (`uk_spine_evidence`, which the bound checkpoint authenticates against its frame, and which the problem node now declares as a source), and before it reads its frame or resolves anything it refuses (`require_national_bound_spine`) a spine whose sidecar records a geographic clone count other than the int one, or whose roster lacks a geography-first name: a clone or area stage, the WAS chain's main-residence localiser, the geography block or one of its keys (`spine_posture.roster_failures`, the dense line's check). A sidecar that records no clone count describes one clone, as the checkpoint authentication reads it. The refusal is one sentence naming K and what the roster lacks, with no row count, id or value; a sidecar that is not a JSON object, unparseable bytes among them, is refused in one fixed sentence. `uk_national_graph` refuses a spine population that does not read its build sidecar. +- **The checks** (`spine_posture`). + - `require_national_spine(sidecar)` holds a bound spine's build sidecar to one clone and the geography-first roster, as above. `NATIONAL_LINE`, `NATIONAL_CLONE_COUNT` and `NATIONAL_AREA_EXCLUSION` name the line, its clone count and the entry it needs signed. + - `require_national_exclusion_signed(review_date, *, source=None)` loads the degenerate-surface register (the committed one, or `source`) through `load_uk_reviewed_exclusion_register` and requires `household.atomic_area_basis` to apply on the national line (`UKReviewedExclusion.applies_on`: an entry that names no line applies on every one) and to be in force on the review date, neither expired (after its `expires_on`) nor premature (before its `approved_on`), as the gates read the register. It takes the review date as a `datetime.date` and reads no clock of its own. Its refusals name the exclusion, the national line and the entry's window, and nothing else: a missing entry, an entry not signed for the national line, an expired one and one not yet in force each have their own sentence. +- **The national preparation** (`full_build_cli.prepare_national_build`). After the engine freeze it refuses, on `--review-date`, the build's own review date (the date the national graph's target and gate nodes evaluate their registers on), an entry that is not signed for the national line or not in force, before any input is read. No gate of the national build reads this register; the release cut reads it again, on the day it certifies, and fails there on an entry that has lapsed since. The preparation then reads the bound spine's build sidecar, before the input is pinned, and refuses a spine of any clone count but one, naming K, or one whose roster is not geography-first; it checks the sidecar the checkpoint authentication returns again, so one changed since the first read is refused too. Every refusal comes before any engine work. A dry run, which takes its graph inventory from the same preparation, is refused there too, after its own reads of the input and the targets. A refused build records its attempt as failed, with the refusal as its message. +- **The national export** (`national_frame`). `UK_NATIONAL_EXPORT_DROPPED_COLUMNS` names each entity's geographic clone index (`person_clone_index`, `benunit_clone_index`, `household_clone_index`), and `uk_release_export_dropped_columns(line)` gives what the release boundary drops on a line: the reviewed export exclusions on every line and, on the national line, the clone indices after them. `uk_release_export_frame` takes that line as a keyword, a gate context's `release_line` (`"national"`, `"local"`, or none); without a line, or on the local line, it drops what it did before, and it refuses a line the gates do not know. The three area codes leave under the consumers' names on the national line as on the dense one. A frame that already left on a line now passes that line's boundary again unchanged when it carries the consumer names, as a national release with areas does (it raised before, as a table carrying a stale consumer name); a household table carrying both a ladder and a consumer name is still refused. The national writer (`full_build_cli`) and the graph's readback of the written H5 (`graph_national`) write and expect the national line's export. The comment on the release boundary's drop of `main_residence_value_rank_basis` now describes the WAS chain's kept draw. +- **The gates** (`battery_bindings`). The degenerate-surface gate skips what the context's line drops at export, so a national frame fed to it before export does not fail on its constant clone indices; the national line runs that gate only at the release cut, on the exported candidate, which carries none. A gate that checks a declared stage output reads the columns the context's line drops off the certifier's spine frame (`_export_dropped_columns_from_spine`), so the release cut's nonnegative gate checks the national candidate's clone indices there. The release input coverage and input-mass parity gates read the export of the context's line; the coverage gate, which also runs the preflight's manifest check, reads the line as declared and leaves its check to the export. +- **The certifier** (`release_certification`, `tools/certify_uk_release_cut.py`). The certification composer refuses a spine whose build sidecar does not record one geographic clone (a sidecar that records none describes one) or whose geography block is not a release's, by the helper the dense release pre-flight uses (`country_adapter.uk_spine_geography_binding_failures`, here at K = 1): the atomic law, the geography key, a SHA-256 definition digest, exactly the conditioned supports `sources.yaml` registers, the spine's area stream and the register's name. A sidecar without a geography block is refused as recording none. The certifier tool refuses a spine build sidecar whose bytes are not the ones the build record's `spine_provenance.sidecar.sha256` names, then runs the composer's geography check on it, both before any gate reads it, so a spine the check refuses costs no release-cut battery and leaves no signed release-cut report; the composer checks it again. The tool also refuses a candidate that carries any entity's geographic clone index, which the national line's export drops and which the release cut's degenerate-surface gate, skipping what the national line drops, would not report. The refusals name fields, columns and K only. +- **Texts.** `rfdr_area_layer`'s module docstring no longer says the national line keeps the engine's fuel-relief default, and `geography_first_roster`'s names the national line beside the dense one. Three test texts that described the national line as it was before this change follow: the K-sensitive table's zero-weight strata reason, a rules-engine provenance comment and a terminal-graph comment. +- **What moves.** Until the entry is signed for the national line, no national build runs. Before this change none ran on a spine from #931 group (b) on; a spine from before group (b), which carries no clone index and which the national line still built on, is now refused as not geography-first. Once the entry is signed, values move in a national build only: its release carries the spine's area columns and no clone index, so its council tax reduction, LHA, private rents, home values and fuel duty for households in a relief area follow the spine's areas, as ruled (decision 4 of 4 Nov; item 4 of 6 Nov), and the fuel-relief flag replaces the engine default of false the national line kept while it had no areas (María's ruling of 28 Oct). No national build has measured them. No value moves on the dense line: the release export without a line gives the frame it gave before on every invented frame checked, and the dense line's gates read the local line's export, which is the same. Keys, measured engine-free on code and invented data against a snapshot of the parent commit: + - UK kernels: 84 of 88 move, and none hashes a module it did not hash before. `national_frame`, which all 62 kernels of the spine registry and 21 other UK kernels hash, moves those 83: the spine registry's 62, `uk.spine-gates@1`, `uk.full-gates.preflight@1`, `uk.full-gates@1`, `uk.full.bound_spine@1`, `uk.full.calibrated@1`, `uk.full.dense@1`, `uk.full.measures@1`, `uk.full.pool@1`, `uk.full.problem@1`, `uk.full.rotated-holdout@1`, `uk.full.selected@1`, the four size kernels, `uk.full-export.prepare@1`, `uk.full-export.prepare-inspection@1`, `uk.full-export.readback@1`, `uk.full.national_problem@1`, `uk.full.national-gates@1` and `uk.full.national-readback@1`. Among them, `battery_bindings` moves the four gate kernels as well, `release_certification` the bound spine checkpoint, `geography_first_roster` the eight geography-first stage kernels and `rfdr_area_layer` two of those (`uk.stage.rfdr@1`, `uk.stage.spi_copy_areas@1`). `graph_national` moves the four national kernels, `uk.full.national_targets@1`, which hashes no other module this change edits, among them. `uk.full-export.package@1`, `uk.full.certification-readiness@1`, `uk.full.target_compilation@1` and `uk.full.target_selection@1` do not move. + - Graphs: every node re-keys, in the spine (90 of 90; split at four clones, 90 of 90; the bundle, 88 of 88), dense on the in-graph spine (100 of 100), dense on a bound spine (11 of 11; 15 of 15 through the export, package and certification-readiness nodes, the last two by chain alone) and national on a bound spine (7 of 7 with its readback; the national problem also by the sidecar source it now declares). Each spine node re-keys by its own kernel's hash, since every spine stage kernel hashes `national_frame`. With `national_frame` held at the parent's bytes, 75 of the spine's 90 nodes and 85 of the dense composed graph's 100 would still re-key, from `geographic_expand` on (`geography_first_roster`, and `battery_bindings` at the spine gates); `national_frame` alone re-keys the 15 nodes before it, `create_uk_frs` through `frs_annual_income_spells`. `release_certification` re-keys the bound checkpoint and so every node on a bound spine, and the calibrated gate node moves by its `terminal_code` parameter too. No value moves, and the gate declarations, which the strict loader holds a stored gate report to, do not move. Gate phase codes: the full battery's terminal code moves (it hashes `release_certification`); the preflight code and the spine's assembled and transferred codes do not. + - Not moving: the stage contracts, the UK country and solve fingerprints, the UK spec-engine wire and bundle lock, and every generated file and contract pin (the H2 fixture tool's `--check` regenerates byte for byte, and the coverage manifest and `source_stages.json` read current). Outside the UK no key moves: the 48 transport kernels and the 31-kernel shared transport registry, NZ's 27- and 31-node graphs (with the toy donor digests held fixed, since freshly written toy donors move them between two runs on the same parent) and the NZ battery's sources, the six US kernels and the eight-node US post-transfer graph, the F0 registry, and the AM, BE, NZ and US fingerprints, resource hashes, spec-engine wires and bundle locks. +- **Tests** (invented data, the seam suite's stand-in frame, its clone-first copies at one and three clones, the stand-in geography-first sidecar, and test registers with invented dates; one test reads the committed register as it stands, on a fixed date inside its entry's window). + - The national problem refuses a bound spine of three clones naming K, and a one-clone spine whose roster is group (b)'s naming the six area stages it lacks, the localiser and the geography block; it encodes its problem on a one-clone geography-first spine that carries the clone index; its node declares the sidecar it reads, and a spine population that reads none is refused. Its helper refuses a sidecar that is not a JSON object, unparseable bytes among them, in one fixed sentence, with no position from the parser's message and no figure but the issue's number. + - `require_national_spine` passes a geography-first one-clone sidecar, with or without a recorded clone count, and refuses another K, a K that is not an int, a roster from before the area stages and a sidecar that is not a JSON object, naming K and missing names only. + - `require_national_exclusion_signed` passes an entry signed for the national line, or naming no line, on its window's first and last days and inside it; refuses an entry on the local line alone, an expired and a premature entry and a register without the entry; refuses a review date that is a `datetime`, a string or None; and its refusals name the entry, the line and the window only. The committed register as it stands is refused on a fixed date inside its entry's window; the commit that signs the entry for the national line flips that test. + - Through `prepare_national_build`: an unsigned entry refused before the input exists; an expired and a premature entry refused; a spine of three clones and a spine from before the area stages refused before the input is pinned; the authenticated sidecar checked again; and a signed entry, on its window's first and last days and inside it, preparing a one-clone geography-first spine, the check evaluated on `--review-date` and the problem node reading the sidecar. No national preparation test reads the wall clock. + - The national role end to end: on a one-clone clone-first spine whose every entity carries its clone index, the build solves, its H5 carries none of the three, and the graph's readback passes; on a test register whose entry lists the local line alone, a build is refused with the refusal as its failure message, a failed row and a failed run and no build record, and a dry run is refused with no plan printed. The national role's suite now runs on a fixed review date, the latest approval across the committed registers (`committed_registers_review_date`, moved to `test_support/microcosm_build/uk_full_build_cli.py` from the national graph's tests), never the wall clock. + - The national graph's and the national role's stand-in sidecars record the stand-in geography-first roster and block at the frame's clone count, and the national role's end-to-end tests stand the exclusion check in as signed (`stand_in_signed_national_exclusion`) except where they test its refusal; `write_national_exclusion_register` and `use_national_exclusion_register` in `test_support/microcosm_build/uk_full_build_cli.py` write a test register and hold the preparation's check to it. + - The derived binding rule lists `require_national_bound_spine` as refusal-only in place of the retired helper. + - The national export (the stand-in geography-first spine at one and three clones): the drop list names each entity's clone index and the export takes the gate contexts' lines, refusing any other; the national release of a one-clone spine carries no clone index and keeps every lineage and area column value for value, the three area codes under the consumers' names; without a line and on the local line the export is one frame that keeps the clone indices, and the national one differs from it by the three indices alone; a released frame with areas passes the boundary again on any line, before and after the H5 round trip, and a household carrying both a ladder and a consumer name is refused; the graph's readback passes a calibrated one-clone spine's H5 written as the national line's export, and fails one written with the clone indices kept. + - The national allow-list pin: every lineage and area column the national release of a one-clone spine keeps, and every column it adds to the export surface, is in `UK_ALLOWED_EXTRA_EXPORT_COLUMNS`, with no list change; the clone indices stay listed for the dense line. + - The battery, on an invented one-clone frame before export: the degenerate-surface gate passes with the line-aware drop on the national line, and fails all three clone indices on the local line, with no line, and with the shared list alone on the national line; the spine frame answers the clone indices on the national line only, so the nonnegative gate checks a national release's indices there and fails them as missing without the national line or the spine; the coverage and input-mass gates read the export of the context's line; the coverage gate refuses a line the gates do not know. + - The certifier: one refusal per failure class of the binding check on the sidecar's block (the law, the identity, the definition digest, a support pin's digest and its size's type, a missing and an extra system, the stream's seed and its type, the register), a clone count that is not the int one, a block that is absent, null, empty or not a mapping (with K named where it is not one), and a binding held to a register whose supports are not the spine's; a sidecar without a clone count reads as one clone; no refused certification is written. The green certification fixture's sidecar carries the stand-in spine's one-clone binding under the register `patch_support_register` puts in place. + - The certifier tool refuses a sidecar rewritten since the calibration recorded it, and one the build record records none for; refuses a spine of three clones, and one that records no geography binding, by the composer's sentence; does all of this before the Ledger read, so no release-cut report is written; goes on to the Ledger read with the recorded sidecar and its stand-in binding; and refuses a candidate that carries any one entity's clone index, naming the column, and takes one that carries none. diff --git a/changelog.d/931-retire-legacy-national-sampler.removed.md b/changelog.d/931-retire-legacy-national-sampler.removed.md new file mode 100644 index 0000000000..32b98f2083 --- /dev/null +++ b/changelog.d/931-retire-legacy-national-sampler.removed.md @@ -0,0 +1 @@ +Retire the UK national compact sampler, `sample_uk_national_frame`, and its source-family decoder, `uk_source_family_units`, from `microcosm.build.uk_runtime.national_sampling` (#931). The sampler drew rungs from the certified national compact. The decoder recovered each derivative row's source from retired id arithmetic: a clone multiplier of ten to the digit count of the largest canonical id, and SPI and capital-gains offsets of the largest raw and pre-capital-gains household ids plus one. No build or tool has called either since 8adc836b9 (#762) moved the row-wise candidate onto the spine sampler. Their private helpers go with them: the post-sample SPI replacement quota check, the required-column list and the clone-index column name. The module also leaves the copy-id tests' inline-rule allowlist, which now admits only `rowwise_geography` (the dense line's multiplier) and `copy_id_table` (its field spans). The spine sampler (`sample_uk_spine_frame`, `uk_spine_source_family_units`), the rung tokens and the default sampling seed are unchanged here; that sampler, which drew the dense line's source families, retires later in #931, when the dense line takes its sampling from the spine it binds. The tests of the retired functions are removed, and the numeric-region refusal they covered is now tested on the spine sampler. No value moves. The full build's population nodes that hash the module (`uk.full.sample`, `uk.full.normalize`, `uk.full.expand`, `uk.full.locations`, `uk.full.geography_mapping`, `uk.full.pool` and `uk.full.geography_gate`) re-key, and with them every node downstream of `uk.full.sample`. No spine node re-keys, and no generated file moves. diff --git a/changelog.d/931-shared-cell-expectations-and-bounds.added.md b/changelog.d/931-shared-cell-expectations-and-bounds.added.md new file mode 100644 index 0000000000..0153c27160 --- /dev/null +++ b/changelog.d/931-shared-cell-expectations-and-bounds.added.md @@ -0,0 +1,30 @@ +Two country-neutral helpers, shared for Max's review and imported by nothing yet (#931): the exact expectation of an area code's value under a conditioned draw cell, in `microcosm.build.atomic_support_expectations`, and amounts held within a bound each record carries, in `microcosm.build.bounded_amounts`. The UK's council-tax localisation will call both: the first for each household's reference over the cell its area was drawn from, the second for the check that a reduction stays within its bill. + +- **`draw_cell_expectations(support, households, *, constraint_columns, weight_by, default_weight, value_column, values)`** takes `choose_fallback_levels`' arguments but the ladder, with each household's inputs already at the level it is drawn at, and returns one exact `Fraction` per household, in row order: the mean of `values` over the household's cell, weighted by the column it samples by. `values` are ints or Fractions keyed by the codes of `value_column`, a support code column. + - It reads the inputs as the shared draw does. A present constraint input is an exact-match constraint on its support code column, and a missing one is skipped, as the draw skips an input declared `required: false`. The household samples by the weight column the rule maps its code to where the code is present, and by `default_weight` where it is missing or no rule is given. + - The integer weights are summed by cell and `value_column` code, each sum is multiplied by its code's value, and the total is divided by the cell's mass once. A code without mass in the cell is not read. It is vectorised, with one grouped mass table per set of constraints present and one expectation per distinct cell and weight column. + - Refusals name declared columns only, never a count, a household, a code or a value. It refuses `constraint_columns`, `weight_by` and `default_weight` as the chooser does; a `value_column` that is not a support code column; values that are not ints or Fractions, a float or a bool among them; a frame that lacks or repeats an input column; a present input that is not a nonempty string without surrounding whitespace; a present code the rule does not map, which the draw also refuses; a household whose cell has no support row, or no mass in the weight it samples by; and a code with mass in a household's cell but no value. +- **`amounts_within_bound(tables, *, entity, id_column, bound_column, same_entity_columns, member_columns, membership_columns, absolute_tolerance)`** returns one boolean per named amount column and `finite_nonnegative`. + - Each column's boolean says whether every record's amount is at most its bound plus the tolerance. The amount sits on the record itself, or on its members, summed to the record through their membership column. + - `finite_nonnegative` says whether every value read of the bound and of the amounts, member rows included, is finite and nonnegative, so any NaN, infinite or negative value fails it. A missing value reads as NaN: a NaN amount fails its column, and a NaN bound fails every column. An amount of plus infinity fails its column unless its bound is plus infinity too; minus infinity, like any amount at most its bound, passes it. + - Exclusions are the caller's: a check drops the records it leaves out, with their members, before calling. Refusals name tables and columns only, never an id, a value or a count. +- **Nothing calls them yet, and no key moves.** Neither module is imported by a stage, kernel, gate, tool or other module, so no node binds either. +- **Tests**, on invented supports and tables only. + - `tests/engine_free/shared/test_atomic_support_expectations.py`: + - expectations worked by hand, which the region's mean, the cell's unweighted mean or the mean by another weight column would miss; + - a missing input skipped and the default weight used, whether the frame holds it as None, NaN or pd.NA, in object, StringDtype (either storage), pandas' `str` or category columns; a missing input never matches a code spelled like a missing value; + - inputs follow `constraint_columns`, the default weight is the declared column, two codes sharing a weight column both read it, and a code without mass in the cell needs no value; + - at the decoder's whole weight range the expectation stays exact, equal to no float and to no fraction with a smaller denominator recovered from one; + - two Hypothesis properties: the expectation equals a loop over the support's areas, and it equals the expectation built from the shared draw's own integer cumulative table for the household's cell and weight column, over which the drawn area's value divided by a nonzero expectation averages exactly one. Where the loop or the draw refuses a household, the helper refuses it; + - a draw that takes the value code first, by the same weight column, has the same expectation, read from the shared draw's own table at the code's level; + - the vectorised result equals one household at a time, and shuffling the households or the support's areas changes no household's expectation; + - each refusal, with no digit in its text. + - `tests/engine_free/shared/test_bounded_amounts.py`: + - amounts at the bound pass, and just above it fail; + - the tolerance is honoured, and a two-member sum above the bound fails; + - members count toward their own record only; + - NaN, infinite and negative values fail `finite_nonnegative` wherever they are read; a NaN or plus-infinite amount fails its own column, and a negative one, minus infinity included, does not; a missing value reads as NaN; + - integer columns and tables without rows are read, and exclusions are the caller's; + - the verdicts do not depend on row order, and several member tables report in the order named; + - each refusal, with no digit in its text. + - The expectation test reads the shared draw's private cumulative table, so a change to `atomic_geography`'s draw can fail it. diff --git a/changelog.d/931-shared-clone-expansion-core.added.md b/changelog.d/931-shared-clone-expansion-core.added.md new file mode 100644 index 0000000000..22cd6e0456 --- /dev/null +++ b/changelog.d/931-shared-clone-expansion-core.added.md @@ -0,0 +1,28 @@ +A country-neutral clone expansion core, `microcosm.build.clone_expansion` (#931; shared now, as María ruled on 10 Nov). `expand_clones(frame, clones=K, declaration=...)` turns a frame into K clones of every record, clone-major. Clone 0 is the frame's own rows with their ids, and clone `k` repeats them with every id minted by the country's rule. The module names no country's columns, tables or constants: it reads each entity, id column, membership column and weighted entity from the frame, and the country passes the rest in a `CloneDeclaration`. + +- **Declarations.** + - A `CloneIdRule`: the most clones the rule numbers (`bound`), `mint(ids, k)` for clone `k`'s ids in the ids' own dtype, and `require_below(ids, what)`, which refuses incoming ids the mint has no room for. + - A clone-index column for every entity. + - The weight kinds the expansion accepts. + - The mass records' reason, declared once, so it is the same at every K. + - Optionally, a column per entity for each row's id before the expansion. + + A declaration compares and hashes by identity. +- **What the clones carry.** + - Each entity's ids are minted for clones 1 to K - 1, and a person's membership in clone `k` is its unit's clone-`k` id. Every other column is copied unchanged. Ids and memberships keep their dtype at every K. + - Each row carries its clone ordinal, with exactly K values on every entity, and, where declared, its clone-0 id. + - Every weighted entity's weights become `numpy.tile(w / K, K)`: one float per row of the frame, the same in each of its clones, of the entity's kind. Strata repeat once per clone and metadata is kept. + - The mass log gains one record per weighted entity, in the frame's entity order, with factor 1.0 and the declared reason. Each entity's clones' total must equal the frame's within a relative 1e-9. + - At K = 1 the rows, ids and weights are the frame's own; only the declared columns and the records are added. +- **Refusals.** Each names entities, columns, dtypes, weight kinds and declared constants only, never an id or a weight. The core refuses: + - a clone count that is not an integer in `[1, bound]`; + - a declaration that names an entity the frame lacks, or gives one of its entities no clone-index column; + - a frame that already carries a declared column, or carries a link table; + - weights, on any entity, of a kind the declaration does not accept; + - ids or memberships that are not numpy integers, memberships in another dtype than their unit's ids, and ids the rule's `require_below` refuses; + - a mint that does not return one integer per id in the ids' own dtype, does not keep the ids' order, or does not put every clone's ids above the clone before's. +- **Not covered yet.** Link tables and a different number of clones per record. These wait for a caller that needs them. +- **Tests.** + - `tests/engine_free/shared/test_clone_expansion.py` runs the core under neutral and US-shaped declarations on invented frames. The US-shaped frame weights households and people, each split with a record of its own, and its clones, labelled from their clone index, pass the shared two-channel invariant `outer_stage_runtime.assert_clone_expansion`. Narrow ids keep their dtype at every K, and each refusal, the dtype ones included, is checked to print no id or weight. + - `tests/engine_free/uk/test_uk_clone_expansion_core.py` runs it under the UK's declarations: the copy-id table's `geographic_support` mint and `require_below`, the ladder's clone-index columns, `source_household_id`, and design weights on households. At K = 1, 2, 3, 7 and 15 it reproduces the clone-first stand-in `clone_first_frame`, which is written without the core, bit for bit on the ids, the household clone index, the source household id and the weights. +- **First caller.** The UK's geographic expansion step (`microcosm.build.uk_runtime.geographic_expand`, G1 of the clone-first spine) calls the core with the UK's declaration, `UK_GEOGRAPHIC_CLONES`. Its mint and its `require_below` are named functions in that module that call the copy-id table, so the derived binding rule binds the table to the key of the stage that runs the step. The UK test above now takes that declaration from the step instead of keeping a copy of its own. The spine's `geographic_expand` stage runs the step, at one clone until the spine build takes a clone count, and binds the core into its key (`931-uk-spine-geographic-expand`). diff --git a/changelog.d/931-shared-fallback-level-chooser.added.md b/changelog.d/931-shared-fallback-level-chooser.added.md new file mode 100644 index 0000000000..054a0106ae --- /dev/null +++ b/changelog.d/931-shared-fallback-level-chooser.added.md @@ -0,0 +1,16 @@ +A country-neutral fallback-level chooser for conditioned area draws: `FallbackLevel` and `choose_fallback_levels` in `microcosm.build.atomic_support_conditioning` (#931; shared, for Max's review). The shared draw (`assign_atomic`) refuses a household whose constraint cell has no sampling mass in the weight it samples by. A country that conditions its draw, as the UK geography-first draw will on the deprivation decile and the council-tax band, declares a fallback ladder instead, and the chooser picks each household's level before the draw. + +- **Declarations.** `FallbackLevel(name, constraints, weight_by)` names the household inputs a level keeps as exact-match constraints, and says whether the level samples by the weight chosen for each household (`weight_by=True`) or by the default weight. `choose_fallback_levels(support, households, *, constraint_columns, weight_by, default_weight, levels)` takes the decoded support, a frame of household inputs, the support code column each input must match, the per-household rule `{"input": ..., "map": {code: weight column}}` (the draw's `weight_by` without its default) or `None`, the default weight column, and the ladder in order. +- **The rule.** Each household takes the first level whose inputs are present (every constraint the level keeps and, where it samples by the household's own weight, a code the rule maps) and whose cell, the support's areas matching the household on every kept constraint, has positive summed mass in the weight the household samples by there. It returns each household's level name, in row order. A draw that keeps a household's level's inputs, reads every other input as missing and samples each stage by the same rule then meets no empty cell. The chooser is vectorised, with one grouped mass table per level, read for the households no earlier level took. +- **Refusals.** + - A household no level takes is refused with one fixed text that names no count, household or value. + - A present input that is not a nonempty string without surrounding whitespace is refused, as the draw refuses it. A households frame that repeats an input column is refused with a fixed text. + - A declaration is refused where a level keeps an input `constraint_columns` does not map, a level samples by the household's own weight but no rule is given, an input maps to the area or to a column that is not a support code column, two inputs map to one column, a weight is not a support weight column, two levels share a name, or the rule's input is also a key of `constraint_columns`. The draw samples by a present code's weight at every level, so a level could not keep such an input and sample by the default weight, nor drop it and keep the household's own weight. +- **No caller yet, and nothing moves.** The chooser names no country, and nothing calls it until the UK geography-first draw. (Later in #931 the spine's `atomic_area_assign` calls it: `931-spine-assigns-areas`.) No node binds the module: no transport kernel (the shared kernels New Zealand's graph runs on, the atomic geography and rules kernels among them), no UK graph kernel or gate battery phase, and no transport binding's source closure hashes it, and no US module imports it. So no node key moves. +- **Tests.** `tests/engine_free/shared/test_atomic_support_conditioning.py`, on invented supports only: + - a household whose band has mass in its cell takes the first level, one whose cell has households but no banded mass takes the second, and a missing decile falls to the region-only levels, whether the frame holds it as None, NaN or pd.NA, in object, StringDtype, pandas' `str` or category columns; + - two codes that share one weight column both read it, and of two identical levels the first is taken; + - the default weight is the declared column, each input matches the support column `constraint_columns` names for it rather than a column of its own name, and a missing input never matches a support code spelled like a missing value; + - a household no level takes, and a frame that repeats an input column, are refused with count-free texts; + - on 3,000 invented households the vectorised choice equals a per-row loop over the support's arrays, and again under a shuffled row order; + - two Hypothesis properties on small invented supports and ladders, whose rules may share a weight column between codes and whose ladders may repeat a level: the chooser equals the per-row loop, and the shared draw itself finds mass at the chosen level and is refused at every earlier level whose inputs are present (and at every level, for a household the chooser refuses). diff --git a/changelog.d/931-spi-housing-council-tax-code.added.md b/changelog.d/931-spi-housing-council-tax-code.added.md new file mode 100644 index 0000000000..12ff6c0945 --- /dev/null +++ b/changelog.d/931-spi-housing-council-tax-code.added.md @@ -0,0 +1,20 @@ +The housing shell's council tax on the rows the SPI support channel copied, at each copy's reference Band D, in `microcosm.build.uk_runtime.spi_housing_council_tax` (#931), unwired: no stage uses it yet and nothing imports the module, so no key or value moves. Later in #931 group (e), the housing shell (`spi_housing_shell`) prices its bills with it after its chained draws and declares what the module states. + +- **The reference.** A copy's draw inputs are missing by rule, so the SPI-copy area stage (`spi_copy_areas`) draws its area at L3, its region by households. `uk_spi_housing_council_tax_inputs` takes each Great Britain copy's reference Band D as the exact expected 2024-25 Band D of that cell under the draw's own weights, from the shared `draw_cell_expectations` with the arguments the council-tax localisation (`council_tax_localise`) passes for the same rows. + - It reads each row's region, support channel and copy lineage, on a narrow frame of the region alone whose two draw inputs are present and missing (`uk_spi_copy_draw_frame`). It reads no area column, geography key, IMD decile or band flag. + - Its rows are those whose `household_support_channel` is `spi`, the housing shell's rule, and they must be exactly the rows the support channel's lineage marks as its copies, SPI copies and income band donors. The supports' digests are keyed by the supports given. +- **The bill.** `uk_spi_statutory_council_tax` prices a Great Britain row by the 2024-25 statute (`council_tax_band_d.uk_statutory_council_tax`) at its reference, for its nation and drawn band, less the single-adult discount where `council_tax_single_adult_raw` is 1: one correctly rounded conversion of the exact bill. `uk_spi_council_tax_bills` returns that on Great Britain rows and the chained draw, bit for bit, on Northern Ireland rows, by María's ruling of 14 November. The branch is the nation of the row's region, never a value of the draw. + - The bill reads neither the drawn area nor its local authority. The localisation multiplies a copy's bill by its drawn authority's Band D over the same reference, so it stays the one step that localises the bill, and the paid bill it keeps for the WAS chain carries no drawn area. +- **Checks and evidence.** `uk_spi_housing_council_tax_checks` returns booleans: every Great Britain bill at the statute of its reference, every Northern Ireland bill as chained, every reduction within its bill, and every rebate equal to its reduction. `uk_spi_housing_council_tax_evidence` returns the block the housing shell's receipt is to carry under `council_tax`: the method, the reference, the copy level, the England-and-Wales and Scottish Band D resources with their digests and the Scottish feed's source commit, the supports' digests and the checks. It has no number leaf, never copies the feed, and its keys are the same whatever systems, nations or clones a frame holds. +- **The declaration pieces** the housing shell's stage is to declare: the localisation's Band D artifacts and its Scottish `verify_vendored_fact_resource` parameters, and a `derive` of `council_tax` whose formula states the bill. +- **Refusals** name columns, dtypes and declared constants only, never a household's id, area code, authority code, band, value or count. A Northern Ireland row sent to the statute, and a band its nation does not price, are refused by the module's own messages before the statute is called. +- **Tests**, `tests/engine_free/uk/test_uk_spi_housing_council_tax.py`, on test_support's invented council-tax cases on the toy conditioned supports, at one clone and at three, and on the housing shell's invented tables, with an invented Band D: + - each copy's reference equals the localisation's own reference for the row, worked again from the toy supports, one for each system and region, and the rows are the lineage's copies in table order; + - the references and bills are the same with every area column, the geography key, the IMD decile and the band flag dropped, on the household columns the housing shell consumes, and for a copy moved to another authority of its region; + - the narrow frame is the area draw's frame of a copy at its level, and the shared expectation refuses one without either draw input; + - each Great Britain bill is the statute at its reference bit for bit, over every band of every Great Britain nation, with the single-adult flag given as `council_tax_single_adult_raw` or as the housing shell's zero-or-one flag; Northern Ireland's rows keep the chained draw bit for bit, zero or positive, whatever their band; the bills do not depend on row order; + - the checks hold where the bills are written, each reduction capped at its written bill and the rebate set equal to it, and each check fails on its own breach: the statute check on a Great Britain row, the Northern Ireland check on a Northern Ireland row, a zero of the other sign included, and the reduction and rebate checks on a row of each; + - with the localisation after the bills, each Great Britain copy's bill is the statute at its own authority within four units in the last place, its kept bill is the statute at its reference less its rebate, the localisation's cap never binds on a copy, the FRS rows are as the localisation leaves them without the bills, and Northern Ireland's rows are left as written; + - the bills of each wrong design are caught: the reference at the drawn authority, the region's unweighted mean, Northern Ireland priced by the statute, the chained bill kept in Great Britain, and the reduction capped at the localised bill; + - the evidence has no number leaf on the invented Band D, on one whose Scottish feed carries an integer, and on the committed one, and it is the same across frames and clone counts; + - the declaration pieces equal the localisation's, and every refusal names no key, code or count of a household. diff --git a/changelog.d/931-spi-housing-council-tax.changed.md b/changelog.d/931-spi-housing-council-tax.changed.md new file mode 100644 index 0000000000..d056a9ee21 --- /dev/null +++ b/changelog.d/931-spi-housing-council-tax.changed.md @@ -0,0 +1,41 @@ +Price Great Britain SPI copies' council tax by the statute after the housing shell's chained draws, and bind the shell to the conditioned supports (#931). The housing shell (`spi_housing_shell`) now prices its bills with `uk_runtime.spi_housing_council_tax`, which #931 group (e) added unwired, and declares what that module states. + +- **The shell** (`uk_runtime.spi_housing_shell`): + - `council_tax` stays a target of the chained QRF on every row, so the fit, the band draw and every other chained amount are as before. After the learned zeros and the tenure rules (`apply_support_rules`), each Great Britain row the SPI support channel copied, SPI copies and income band donors, takes the 2024-25 statute at its reference Band D (`uk_spi_council_tax_bills`): the expected 2024-25 Band D of the cell `spi_copy_areas` draws its area from, its region by households, for its nation and drawn band, less the single-adult discount where `council_tax_single_adult_raw` is 1. The bill reads neither the drawn area nor its local authority, so `council_tax_localise` stays the one step that localises it. + - By María's ruling of 14 November, Northern Ireland's rows keep the chained draw. The branch is the nation of the row's region. + - The drawn council tax benefit is then capped at the bill written (`cap_council_tax_benefit`); the reference person's `council_tax_benefit_reported` carries it and the household's `council_tax_rebate` equals it, as before. `apply_structural_rules` is now the composition of the two steps, with the same rule firings in the same order, so `tools/validate_uk_housing_imputation.py` is unchanged. + - By María's ruling of 14 November, the band is drawn from the income classifier alone, as before, and the band tilt moves to SPI seating. The shell reads no household area column, so the pin that holds its consumes free of the area stands. + - `impute_spi_housing_shell` takes the required keyword `council_tax_inputs` (`uk_spi_housing_council_tax_inputs` on its household table) and refuses inputs that do not hold its SPI-channel rows, with their regions, in table order. The bill's four checks are asserted on what the stage writes, each failing with a fixed message that names no household. + - The receipt keeps its keys and their forms, and gains the `council_tax` block: the method, the reference, the copy level, the Band D resources with their digests and the Scottish feed's source commit, the supports' digests and the checks, with no number leaf. The cap's firing count now counts the reductions above the bill written. + - `UKSPIHousingShellStageTransform` takes the conditioned supports' bytes (`supports`) and an optional Band D, the committed one by default, as the council-tax localisation's transform does. Loading the committed Band D holds Scotland's vendored resource to the pinned feed. The transform checks its stage when it is built and when it runs, and before it reads a row checks that its Scottish declaration maps the facts to the frame's period. +- **Stages** (`sources.yaml`, then `source_stages.json` by the by-name merge). The shell declares, in order, Scotland's Band D `verify_vendored_fact_resource` as `council_tax_localise` declares it, its imputation with its parameters unchanged, and a `derive` of `council_tax` whose formula states the bill. Its artifacts are the localisation's two Band D artifacts (roles `council_tax_band_d` and `scotland_band_d_facts`), each with `runtime_sha256_required`. Its source and notes describe this behaviour. Its outputs and rewrites are unchanged, and so is the stage order. +- **The graph and the spine build.** + - The shell reads the three conditioned supports as graph sources, as the area stages and the localisation do (`_SPLIT_STAGE_SOURCES`). Its household consumes are unchanged. + - In `graph_kernels`, the shell binds the helpers its transform now computes through, among them `spi_housing_council_tax`, `atomic_support_expectations`, `atomic_area_conditioning` and `council_tax_band_d`. The H2 fixture's registry builds it on the toy supports at the committed Band D. + - `spine_build` builds the shell on the supports the area stages read, holding them to the spine's support pins first, so a support replaced after the spine was prepared is refused with the area stages' message before the shell builds anything. The support flags' help names the shell. + - `tools/graph_uk_spine_fixture.py`'s oracle builds the shell on the toy supports. +- **The geography-first roster.** `GEOGRAPHY_FIRST_OPERATIONS` names the shell's `derive`, before the WAS chain's main-residence localiser. So the dense line's posture check, the spine CLI's release candidate and the national line refuse every spine built before this change, naming `operation derive in spi_housing_shell`. +- **Generated files.** `source_stages.json` regenerates by the by-name merge, and the shell's imputation keeps its committed key order. The H2 fixture regenerates through the engine slot: `uk_spine.json` carries the shell's new stage contract and `sources/fixture.json` its new descriptor, and `sources/cgt_asset_type_facts.json` and `sources/property_purchase_targets.json`, which the fixture tool sizes to its oracle's frames before `cgt_residential_split` and `property_purchases`, move with the values from `was_wealth` on. `release_input_coverage_manifest.json`, which pins `source_stages.json`'s digest, regenerates, and only that digest moves. +- **What moves.** + - Values, at one clone. Every column through `aps_occupation` is as before, and so are the shell's FRS rows, every band and other category it draws, and every chained amount but the bill and the reduction capped at it. + - Great Britain SPI copies and income band donors take the statute bill at their reference. Their reduction is the drawn reduction capped at that bill, and their rebate equals it, which differs from before only where a cap binds. + - After `council_tax_localise`, their bill is the statute at their drawn authority within four units in the last place. The paid bill the localisation keeps for the WAS chain's bill predictor is, on those rows, the statute at the reference less the rebate, still free of the drawn area. FRS rows through the localisation, and its evidence, are as before. + - From `was_wealth` on, SPI rows move directly, and FRS rows through the WAS chain's pooled ranks. + - The shell's receipt gains its block, and its cap's firing count can move. + - Keys. UK only. Every spine node re-keys, and so does every dense and national node after the spine. Of the stage contracts only the shell's moves. The UK country and solve fingerprints move, and so do the UK spec-engine bundle lock and wire. No gate phase code moves. Outside the UK no key moves: no transport or US kernel, no NZ node with its donors held fixed, no NZ battery source, no US graph node, and no AM, BE, NZ or US fingerprint, spec-engine wire or bundle lock. No gates entry, contract pin, operation kind or schema changes. +- **Tests**, on test_support's invented tables, the toy supports, an invented Band D, the packaged spec and H2. + - `tests/engine_free/uk/test_uk_spi_housing_shell.py`: + - every imputation takes the council-tax inputs of its own table, a permuted table and the clone pools included; + - only the Great Britain copies' bill path moves, on the invented tables and on a variant whose invented Northern Ireland training households carry positive bills and reductions: every band, category, other chained amount and Housing Benefit is the fit, draw and rules' own bit for bit, Northern Ireland's bill, reduction and rebate are the chained draw's, and each Great Britain bill is the statute at its reference, worked again from the toys; + - the fitted chain's targets, written out in the test, keep `council_tax` first; + - the bill is written after the learned zeros, the capped reduction is the reference person's and the rebate, and `apply_structural_rules` is the support rules then the cap, bit for bit; + - the receipt's block has no number leaf and is the same at one clone and at three, each check's fixed message names no household, and a chained Great Britain bill fails the statute check; + - the inputs are required and must be the SPI rows, with their regions, in table order, the transform checks Scotland's Band D period before it reads a row and checks the housing invariants once, a drifted stage is refused when the transform is built and when it runs, and the shell imports neither the localisation's module nor stage 5's; + - the committed smoke fixture's descriptor passes the stage's check; + - the H2 fixture's registry and `tools/graph_uk_spine_fixture.py`'s oracle both build the shell on the toy supports. + - `tests/engine_free/uk/test_uk_source_stages.py`: the shell's operations, in order, and its artifacts, the localisation's. + - `tests/engine_free/uk/test_uk_graph.py`: the support readers gain the shell, and the binding rules hold. + - `tests/engine_free/uk/test_uk_frs_spine.py`: the shell is built on the supports' graph sources and refuses a support replaced after the spine was prepared, and the release-candidate roster refusal names its `derive`. + - `tests/engine_free/uk/test_uk_spine_posture.py`, `test_uk_full_build_cli.py`, `test_uk_national_graph.py` and `test_uk_area_home_values.py`: the roster's operations, in the spine's order and held to the stages' own names, and the dense and national refusals naming the shell's `derive`. + - Engine tier: the adapter's support readers gain the shell. + - After merge, H2's clone-first run at one clone and at four: every SPI-channel row's bill after the localisation is the statute at its own authority within four units in the last place. diff --git a/changelog.d/931-spi-support-keyed-share.changed.md b/changelog.d/931-spi-support-keyed-share.changed.md new file mode 100644 index 0000000000..c8d8b4f8c9 --- /dev/null +++ b/changelog.d/931-spi-support-keyed-share.changed.md @@ -0,0 +1,7 @@ +The SPI support channel (`spi_support_channel`) copies its zero-weight SPI households as a keyed Bernoulli share instead of a fixed count drawn by row position (#931). A household is copied when its uniform, keyed on its own `household_id` under the stage's seed (42) and the new salt `spi_support_channel:inclusion`, falls below π = min(1, 10,000 × the sample fraction / the distinct `source_household_id` values entering the stage); at π = 1 every household is copied without reading a uniform. The manifest's 10,000 is now the share's numerator, so a one-clone spine's copy count is binomial, with mean min(10,000 × the sample fraction, the source households), instead of exact: about 1,000 at f010, by design arithmetic (10,000 × 0.1). Each household's draw reads only its own id, the seed, the salt and π, so row order does not matter. π counts true sources, so it is the same at every geographic clone count, and clone 0 of a cloned spine, whose ids are the one-clone ids, draws exactly the one-clone copies, while the other clones draw on their own ids (María's rulings of 7 Nov). + +- **Gate.** `e7_spi_synthetic_preclone` is renamed `e7_spi_synthetic_share`. Its declaration carries `maximum_zero_weight_rows_rule: {rule: binomial_share_band, sigma: 4}` instead of the integer 10,000. After the draw the stage resolves it to min(n, ceil(n × (π + band(π, n)))) over the n household rows entering the stage, and runs the unchanged zero-weight gate. The band is the repo's binomial band, moved unchanged from `stage_health.latent_attribute_tolerance` into `stochastic_assignment.binomial_share_tolerance(*, target, rows, sigma)`; `latent_attribute_tolerance` stays, as the same band at four sigma. In `sources.schema.json`, `stack_zero_weight_donors` requires `salt`, a zero-weight declaration takes either the integer maximum or the rule, and `allocate_zero_weight_prior_mass` accepts `geographic_clone_stratum`. The stage's source sentence, draw text and notes describe the share. +- **Per-clone strata.** Where the household table carries `household_clone_index`, the prior allocation's strata also split by geographic clone, so each clone keeps its own mass and its SPI weights depend on its own households and copies only. The manifest declares this as `geographic_clone_stratum: household_clone_index`. A positive-mass stratum without an SPI copy is refused with a count of strata, not their names. Without the column the strata are as before: a frame without it, or with a one-clone index, allocates bit for bit as the region and pension-age strata did, given the same copies. The small-clone-stratum fallback comes later in #931. +- **Receipt.** The stage evidence gains `expected_spi_households` (10,000 × the sample fraction), `inclusion_probability` floored to two decimals, `source_households` rounded by the receipts' rule, and the gate's name, rule and sigma, not the maximum it resolved to. `spi_households` stays exact, because the stage gate reads it. +- **Bindings.** spi_spine, the module of stages 14, 16 and 18, rounds the receipt, so `frs_hmrc_spine_leaves` joins the counting stages. spi_support now draws through the keyed-draw helpers, so `high_value_home_support` and `cgt_residential_split`, which bind it, bind them too. The spine gate kernel's hash names `stochastic_assignment` beside `stage_health`, since the latent-attribute checks' band now lives there. +- **What moves.** At K = 1 the SPI copy set is redrawn, with a binomial count around 10,000 × the sample fraction where that is below the source households. Per-stratum SPI weights move with it, and so does every SPI-channel value from `hmrc_spi_income_spine` on, the high-value and CGT selections that rank SPI rows, and calibration. Base FRS rows keep their stage-16 weights, which are their incoming weights times stratum shares that do not depend on the draw, except the row the exact-total correction lands on. On synthetic frames, `spi_income_band_donors` seats the same households, carriers and bands under the old and new draws, at the same seating scale; its funding factors and mass scale, and with them its donors' and base rows' weights, move only by rounding, since its stratum masses sum the redrawn copies' weights. The empty-stratum refusal reads the new draw, so a small frame that built before can refuse, and one that refused can build. The H2 fixture's 135 households at a sample fraction of 135 / 10,000 give π = 1 (10,000 × 0.0135 is 135.0 in float), so its copies are unchanged; its declarations and the stage's contract hash are regenerated, with the coverage manifest's source digests and `source_stages.json`. Every node re-keys. diff --git a/changelog.d/931-spine-assigns-areas.changed.md b/changelog.d/931-spine-assigns-areas.changed.md new file mode 100644 index 0000000000..58be687528 --- /dev/null +++ b/changelog.d/931-spine-assigns-areas.changed.md @@ -0,0 +1,31 @@ +Assign each geographic clone's area in the UK spine, fold `frs_brma` into `area_brma`, give SPI copies their own areas, and localise private rents (#931). The area and private-rent transforms #931 group (c) added unwired are now declared and wired: the packaged spine has 63 stages. The dense line still refuses every spine, since the roster lacks only the WAS chain's home-value localiser, and the national line still refuses every clone-first spine. (Later in #931 the WAS chain declares the localiser, so the dense line's posture check passes on the packaged spine: `931-spine-localises-home-values`; and the national line builds on a geography-first spine of one clone once the area-basis exclusion is signed for it: `931-national-one-clone-areas`.) + +- **Stages** (`sources.yaml`, then `source_stages.json` by the by-name merge): + - `atomic_area_assign`, `local_authority` and `rfdr` run after `geographic_identity` and before `frs_take_up`: each clone's area and ladder, drawn on the three conditioned supports, its engine local authority and its fuel-relief flag. + - `area_brma` takes `frs_brma`'s place after `frs_household_draws`, and the assembled boundary moves with it (`UK_SPINE_ASSEMBLED_FINAL_STAGE`). Great Britain reads `brma` off the drawn area through the area-to-BRMA layer; Northern Ireland draws from the census counts with `frs_brma`'s keys, seed and salts. `frs_brma`'s stage block retires; its module and helpers stay. + - `spi_copy_areas` runs after `spi_income_band_donors`: every SPI copy and income band donor gets a key over its own path, an area drawn at L3 and its own local authority, BRMA and flag. Northern Ireland's copies draw their BRMA on their own benefit units after one engine call, one simulation, over their households alone (C-9, as ruled on 11 Nov). Its evidence withholds every system of a kind where one system rests on one or two source households: at one clone the support channel's and the band donors' own copy counts are the kind's total, so the other systems' counts would give it back by difference. + - `private_rent` runs after `spi_housing_shell`: Great Britain private renters' rents are read off their BRMA's list, keyed per row on the stage's own stream at seed 0, with its own mass-conservation receipt. + - `frs_take_up`'s notes and `frs_person_draws`' `rate_derivation` say the targets are counted within each geographic clone, as does the take-up contract's policy line for `frs_take_up`'s anchors, which also says `spi_benefit_coherence`'s SPI-channel Universal Credit redraw still counts over every clone; `spi_support_channel`'s notes say a clone's stratum without a copy is refused; `geographic_identity`'s notes no longer name the retired dense identity node. +- **Shared vocabulary.** The source-stage schema gains six strict operation kinds (`assign_atomic_area`, `resolve_local_authority`, `assign_area_brma`, `assign_rfdr_flags`, `assign_spi_copy_areas`, `localise_private_rents`), and `ALLOWED_SOURCE_OPERATION_KINDS` the same six. Both are additive. +- **The spine build.** + - `--atomic-support-ew`, `--atomic-support-scotland` and `--atomic-support-ni` are required when `atomic_area_assign` is scheduled, refused beside `--synthetic-fixture-dir`, and each file is held to the conditioned register (`uk_atomic_support_register(conditioned=True)`) before the engine is built. A refusal names the flag and the system only. + - The three supports are raw-byte graph sources of `atomic_area_assign`, `rfdr` and `spi_copy_areas`, named by `sources.yaml`'s conditioned ids. Each of those stages holds the files it reads to the pins of the spine's geography block, which the preparation composes before any stage runs, so a support replaced after the spine was prepared is refused, naming its system and graph source alone. + - The sidecar records a `geography` block under the roster leaf's keys: `assignment` (`"atomic"`), `identity` (`"geography_household_key"`), `definition_sha256`, `support_pins` (SHA-256 and integer size per system), `stream` (the seed at index 3) and `register` (`"sources.yaml"`, or `"synthetic"` for a fixture's toys). `country_adapter.uk_spine_geography_binding` composes it, for the sidecar and for the full build's `--spine-request` route alike, and `uk_spine_geography_binding_failures` now also holds `stream` and `register`. + - The declared seeds gain `atomic_area_assign` (`uk-spine-atomic-area-v1`: 0), `spi_copy_areas` (`uk-spine-atomic-area-v1`: 0, `brma`: 0), `private_rent` (`uk-spine-private-rent-v1`: 0) and `area_brma` (`brma`: 0). +- **The graph.** The six stages' consumes and cells; `area_brma` and `spi_copy_areas` are open engine surfaces; `hmrc_spi_income_spine` joins the reader-isolation boundaries, so the graph gains `hmrc_spi_income_spine.boundary`, and `private_rent` its own boundary. The geography-first transforms read their stage names from the roster leaf, and `private_rent_localisation.STREAM_EXPERIMENT` retires. The six stages run on the bitwise stage kernel (C-2; F10 is open with Max). +- **What moves.** + - Keys: every spine node, since `graph_kernels`, which defines every UK spine kernel, changed (the geography-first transforms' kernels now also hash the roster leaf), and `frs_person_draws`' stage contract moves through its `rate_derivation` and the take-up contract it pins; notes enter no stage contract. Stage contracts are added for the six stages and retire with `frs_brma`'s. Every dense and national node re-keys but a bound spine's checkpoint import, through the country fingerprint (`sources.yaml`, `source_stages.json`, the take-up contract and the coverage manifest). Outside the UK one kernel re-keys, the shared transport gate battery (`gates.battery@1`): the source closure of its bindings reaches `source_manifest`, where `ALLOWED_SOURCE_OPERATION_KINDS` lives, so NZ's terminal gate and the export, readback and package nodes after it re-key (`nz.gates.terminal`, `nz.export.prepare`, `nz.export.readback`, `nz.package`; the NZ spec is inactive), as #931 group (b)'s two kinds did. María accepted this on 12 Nov, with the allowed list kept in `source_manifest`. A key probe on code and toy data finds no AM, BE or US key, no AM, BE, NZ or US spec fingerprint, no gate phase code and no other country's spec-engine wire moving; in the AM, BE, NZ and US bundles the six kinds move only the spec engine's grammar receipt, which each country's emitted, uncommitted bundle lock carries. + - Values, at one clone: the new columns (3 assignment, 10 ladder, `local_authority` and `in_rural_fuel_duty_relief_area`); `frs_take_up`, `frs_person_draws` and `frs_household_draws` unchanged, since they read no area column; Great Britain BRMAs read off the drawn area, while Northern Ireland's households keep the BRMA `frs_brma` drew and its SPI copies draw their own; every SPI copy's and income band donor's own key, area, local authority, BRMA and flag; Great Britain private renters' rents, households' and copies'. In policyengine-uk's reads, council tax reduction moves only in the households of English working-age claimants placed in a council whose working-age scheme it models (its default local authority, Maidstone, is not one), LHA for Great Britain renters on Housing Benefit or Universal Credit through the BRMA, with both awards following the new rents, and fuel duty for flagged households; through council tax reduction and those awards the engine's HBAI income moves for those households, and with it `was_wealth`'s income predictor and the later stages that read the engine. At more than one clone, clone 0 nests exactly on the area stages, `area_brma` and the copies' areas, and the private rents approximately; the clones differ by area. + - `geography_household_key` is unique per row up to `spi_support_channel`, whose copies, like the income band donors, repeat their parent's key until `spi_copy_areas` gives each its own. The copies later stages make (high-value home support and the CGT copies) carry their parent's key, so the exported key is no longer unique per row. +- **Defaults taken** (the #931 group (c) design): + - C-1: `area_brma` sits after the draws, in `frs_brma`'s place, since policyengine-uk's `LHA_category`, which Northern Ireland's draw reads, reads `frs_take_up`'s `would_claim_uc`. + - C-2: the six stages are bitwise; C-3 and C-43: the area draw and the private rents take seed 0 on streams of their own, and the copies' areas reuse the area draw's and the BRMA's seeds. + - C-8: the SPI copies' areas are a stage of their own after both copying stages, whose contracts do not change. C-10: copies keep their parent's IMD decile and band flag, and the draw gives every SPI-channel row missing inputs, so copies draw at the region-and-households level (L3). + - C-11: copies made after `spi_copy_areas` inherit its key. C-14: the private rents rank each row on its own key, strictly on a full-scale spine and with the thin-region rule on a sampled rung. + - C-22: the three support flags are held to the conditioned register, and refused beside a synthetic fixture, which carries its own toys. C-26: the stage names. C-37: the three-system check is an engine-tier stand-in, not Northern Ireland households in H2's root. C-41: the BRMA distribution tool's output is private. +- **Generated files.** The H2 fixture regenerates with the 63 stages and three toy conditioned supports, written from test support's invented table through the canonical support adapter as fixture inputs under the conditioned ids, in files named `toy_.npz`. `.gitignore` admits exactly those three past its licensed-output `*.npz` rule, so a real-derived support, which keeps its own name, stays ignored. `release_input_coverage_manifest.json` gains a `private_rent` family. +- **Tools.** + - `tools/verify_uk_identity_stability.py`: E4 recomputes `area_brma`'s BRMA through the stage's own derivation (Great Britain's off the stored areas, Northern Ireland's from the count table on `LHA_category` as `area_brma` read it) and compares it bitwise. `spi_copy_areas`, the one later stage that declares writing `brma`, rewrites only rows the SPI support channel copied, every one of which E4's scope drops, so its rewrite counts as taken back; E4 refuses a scoped frame that keeps such a row, naming the column only. + - `tools/emit_uk_brma_distribution.py` compares Northern Ireland's draws only and counts `cell_n` and `built_count` in rounded distinct source households. A BRMA one or two source households carry withholds every figure of its cell. Each benefit-unit share is floored to the hundredth and z rounded to one decimal, both withheld in a cell of fewer than ten source households. An `--output` inside a git work tree is refused. + - `tools/graph_uk_spine_fixture.py` builds the six transforms in its oracle, as the fixture registry builds them. +- **Tests.** The pins above, the support flags, the binding's composer and checks, the leaf's drift tests and the K-sensitive constants (engine-free). The spine CLI, parsing and preparing the packaged roster: it holds each of the three supports to the register before it builds its engine, composes the geography block from them under `sources.yaml`'s register, hands the area stages the three files, which they hold to the block's pins, and gives a full-scale spine's private rents the strict rule. The shared operator's draw keys, each held to every term: identity, system, level and both digests. Every row's area at three clones the operator's draw on its own key, with some household's area moving at each clone after the first. The packaged area and private-rent stages running as their reviewed declarations; the copy stage withholding a kind's systems; the per-clone count texts; the assembled gates reading up to `area_brma`; the `private_rent` coverage family. E4 on invented frames through the area stages, `area_brma` and the copy stage, comparing `brma` and refusing an unscoped frame. The BRMA distribution tool's rounding, suppression, precision and private output. The integration run's sidecar block, equal to the one composed from the fixture's toys. H2 parity; `required_build_stages` with `private_rent`; the full country graph's only support sources, the spine's three conditioned ones, read by `atomic_area_assign`, `rfdr` and `spi_copy_areas` alone. The clone-first H2 test at 63 stages, with the copies' areas and every area-stage receipt field but K nesting in clone 0, and, at one clone and at four, every household's area from `area_brma` on, every SPI-channel row's own key, the later copies' inherited keys and `private_rent` keeping every rent it does not re-draw (H2's root households all own their homes, so it re-draws none). A new engine-tier test (`test_uk_spine_three_systems.py`) runs England-and-Wales, Scottish and Northern Ireland households on the toy supports through the area stages, `area_brma` and `spi_copy_areas` on the installed engine, at one clone and two. It holds every local authority to the engine's enum and Northern Ireland's BRMAs, base rows' and copies', to `frs_brma`'s draw, and then, on invented housing, `private_rent` re-drawing every Great Britain private renter's rent off its BRMA's list and leaving Northern Ireland's. diff --git a/changelog.d/931-spine-geography-transforms.added.md b/changelog.d/931-spine-geography-transforms.added.md new file mode 100644 index 0000000000..fbf10e162f --- /dev/null +++ b/changelog.d/931-spine-geography-transforms.added.md @@ -0,0 +1,32 @@ +The clone-first UK spine's area, local-authority, BRMA, fuel-relief, SPI-copy area and private-rent transforms (#931), unwired: no stage declares them and no node binds the three new modules, so no output moves. (Later in #931 the packaged spine declares and runs all six, in place of `frs_brma`'s stage: `931-spine-assigns-areas`.) Each transform checks the stage it runs as against the reviewed declaration its module states (name, household grain, outputs and rewrites, the reviewed operations verbatim and its artifacts, each with `runtime_sha256_required`), which the geography declarations will carry. + +- **The conditioned area draw** (`microcosm.build.uk_runtime.atomic_area_conditioning`). + - The definition over the three conditioned supports, held to canonical bytes, keeps the FRS region as a required constraint and the IMD decile as an optional one. England and Wales and Scotland sample each household by the area's households in its observed council-tax band, or by all the area's households where it has none; Northern Ireland, which has domestic rates, samples by households. The layers are the ten ladder columns, and the stream is `uk-spine-atomic-area-v1` with seed 0. + - The fallback ladder runs from L0 (decile and band) to L3 (region and households); Northern Ireland takes L1 and L3. + - The draw reads the decile as "1" to "10" (missing for 0) and the band only where `council_tax_band_observed` is true. A row the SPI support channel copied reads neither, so it draws at L3. + - The shared chooser picks each household's level with the definition's own rule and default. +- **The area stages** (`microcosm.build.uk_runtime.spine_geography`). + - G2, `UKAtomicAreaAssignStageTransform` (`atomic_area_assign`), draws, derives and validates every household's area on one narrow frame and writes the three assignment and ten ladder columns as `StringDtype("python")`. A validation that fails refuses the stage. + - G3, `UKLocalAuthorityStageTransform` (`local_authority`), resolves `local_authority` from `local_authority_code` through the committed names roster, never defaulted, as `StringDtype("python")`. + - G4, `UKAreaBRMAStageTransform` (`area_brma`, to take `frs_brma`'s slot), makes one engine call over the whole frame for `LHA_category`. Great Britain reads `brma` off the drawn LSOA (England and Wales) or output area (Scotland) through the committed area-to-BRMA layer. Northern Ireland draws from the census counts with `frs_brma`'s keys, seed and salts, so its values are `frs_brma`'s. `brma` stays an object column, and a frame G4 cannot place is refused before the engine call. + - G5, `UKRFDRStageTransform` (`rfdr`), sets `in_rural_fuel_duty_relief_area` from the drawn output area through the committed RFDR layer. It refuses a layer that flags an output area its system's support does not hold, and reports a system with no flagged household instead of refusing. + - `UKSPICopyAreasStageTransform` (`spi_copy_areas`, to run after `spi_income_band_donors`) gives every SPI copy and income band donor a geography key over its own copy path, an area drawn at L3 with G2's definition, and then its own local authority, BRMA and flag. As María ruled on 11 Nov (C-9), Northern Ireland's copies draw their BRMA from the census counts on their own benefit units, with G4's seed and salts, after one engine call over their households alone. That call carries no clone index, so it is one simulation over every clone's copies, as at one clone: the UK adapter refuses a call over part of its pool, which a clone without a Northern Ireland copy would otherwise make it. Great Britain copies' areas are checked against the layer before the call. Each rewrite keeps its column's dtype, and every other row is left as it was. +- **Private rents** (`microcosm.build.uk_runtime.spine_private_rent`). G7, `UKPrivateRentStageTransform` (`private_rent`, to run after `spi_housing_shell`), re-draws each Great Britain private renter's rent with `private_rent_localisation.localise_rents`, every row its own spine household keyed on its `geography_household_key`. Tie-breaks are keyed on `(key, "rent_rank")` on the stream `("sha256-u53-v1", "uk-spine-private-rent-v1", 0, 0)`, so tied renters and the clones of one renter take ranks of their own and clone 0's tie-breaks are the same at every K. The sampled rung's thin-region rule counts true source households, and the stage is strict unless the spine is sampled. Only `rent` is rewritten: reported Housing Benefit, Universal Credit and mortgages, the weights and every other column are unchanged, and the stage records its household mass receipt. +- **Evidence.** Every count is distinct true source households through `rounded_count`, never per area or per cell, and no key or code of a household is printed. G2 (by system, level and fallback reason), G3 and G4 (by grain, and Northern Ireland's sources drawn) count over every clone. G5's flagged sources, the copy stage's copies, donors and flagged copies, and G7's re-drawn sources count among clone-0 rows; G7 also gives the cells ranked against their region or against Great Britain. The copy stage's evidence says its validation and roster check did not run on a frame without a copy. +- **Changed modules.** + - `frs_brma.draw_household_brmas` draws the BRMAs of any subset of households with today's keys; `add_frs_brma` calls it and its values are unchanged. Its refusals no longer print household or benefit-unit ids. + - `rfdr_area_layer.area_flags` gains `roster`. Its malformed-code refusal no longer prints a count or a code, and its unknown-system refusal names the accepted systems instead of the frame's values. + - The BRMA and RFDR layer docstrings describe the spine's stages. +- **Keys.** The stage kernels that bind `frs_brma` (`frs_brma`, `was_wealth`, `was_lisa`, `uc_take_up`, `uc_deduction_attributes` and `childcare_copy_take_up`) re-key with it, and so do the dense line's geography BRMA and RFDR nodes (`uk.full.geography.brma@1` and `uk.full.geography.rfdr@1`, after the layer modules' edits), with every node downstream of them. No transport key (NZ, BE, AM), US key or battery phase moves, and no node hashes the three new modules. No declaration, generated file or `gates.json` changes. +- **Values.** None moves. On the synthetic H2 fixture, the legacy oracle's final frame (every table, column and dtype, the weights, the strata and the mass log), its stage records and every stage's evidence digest are identical before and after this change, and the fixture regenerates byte-identically. +- **Tests**, on toy conditioned supports whose public area codes resolve against the committed resources (`test_support/microcosm_build/uk_conditioned_support_fixtures.py`), invented households, a recording stand-in for the rules engine and a stand-in for the support channel's two stages: + - each household's level and region, the decile read as "1" to "10", and the chooser called with the definition's own rule, default and ladder; + - clone 0's areas at K = 1, 2 and 3, and its copy keys, areas and BRMAs at K = 2 and 3, equal to K = 1's, and row-order independence; + - G2's and the copy stage's validation refusing a ladder the mapping does not give, and the written dtypes; + - Great Britain's BRMAs against the layer, Northern Ireland's against `frs_brma`, and `frs_brma`'s household pick written out; + - the copy stage's own keys and Northern Ireland draws, and its engine call as one simulation under the UK adapter's clone rule at K = 2 and 3 with Northern Ireland copies in clone 0 alone; + - G7 against `localise_rents` on a copy-free frame; + - count-free refusals, the frame's before any engine call; + - every evidence count byte-equal at K = 1 and K = 3. + + A mounted test, skipped without the supports, holds the chooser to the committed zero-mass tables. diff --git a/changelog.d/931-spine-home-values-and-geography-checks.added.md b/changelog.d/931-spine-home-values-and-geography-checks.added.md new file mode 100644 index 0000000000..998c4a4bbd --- /dev/null +++ b/changelog.d/931-spine-home-values-and-geography-checks.added.md @@ -0,0 +1,13 @@ +The clone-first UK spine's WAS home-value localiser (G8), the regional property uprating's factor helper and the spine geography gate checks (#931), unwired: no stage declares the localiser or a denominator and no gate declares the new checks, so no value moves. (Later in #931 `was_wealth` declares the localiser, `regional_property_uprating` the denominator, and the spine battery's gates the three checks: `931-spine-localises-home-values`.) A later commit of #931 group (c) declares them, retires `regional_home_values` and registers the gates. + +- **The area home values** (`microcosm.build.uk_runtime.area_home_values`, G8, form F). Called with the WAS chain's segment-2 draw of `main_residence_value`, each owner household with a positive draw takes its weighted mid-rank among the owner households of its FRS region with a positive draw, both support channels, at the stage's household weights, ties broken on a draw keyed on the household id (`property_localisation.owner_uniforms`). Its stream is fixed, `("sha256-u53-v1", "uk-spine-area-home-values-v1", 0, 0)`, with seed source NONE, as ruled for `regional_home_values` on 9 Nov (C-43). An England and Wales owner's value is its assigned constituency's knots at that rank (`uk/high_value_home_value_knots.json`), divided by its FRS region's regional uprating factor on the draw, so the uprating, dividing by the kept draw, returns it to the knot value to one rounding. Scotland, Northern Ireland, renters and owners without a positive draw keep the draw bit for bit, and the draw is kept on every row as `main_residence_value_rank_basis`. It refuses, by texts that name columns and declared constants only, an England and Wales owner whose `constituency_code` is outside England and Wales or another owner's inside it, a constituency without knots and a localised value that is not positive. `area_home_values_declared` holds a WAS stage's declaration of `localise_main_residence_value` to the reviewed operation the module states, its two artifacts (each with `runtime_sha256_required`), the `main_residence_value_rank_basis` output and a support clip exempting exactly `main_residence_value` and `property_wealth`, and refuses a half declaration. The knots' region map `FRS_REGION_TO_KNOTS_REGION`, now read-only, and the knots' file name `KNOTS_RESOURCE` move here from `regional_home_values`, which imports them. `clone_zero_rows` says which household rows are geographic clone 0's; the WAS stage's cap counts by it too. +- **Its evidence** (C-16, C-44): among geographic clone 0's rows of positive weight, the distinct true source households rewritten in each region (`rounded_count`) and those whose knot value is at the knots' cap (`support_band`); the knots' digest, basis year and level, the regional reference's digest, the rank basis and its column, the stream and the method. No value, sum or factor. +- **The WAS stage** (`was_wealth`). `impute_was_wealth` takes `main_residence_localiser=None`; where given it runs after segment 2, segments 3 to 11 condition on its values, the draw is kept as the rank basis and the result carries what it returned. Segment seeds are prefix-stable, so segments 1 and 2 draw bit for bit as without it. The transform builds the localiser only when its stage declares the area home values, writes the rank basis and adds an `area_home_values` block to its evidence. The support clip reads the stage's declared exemptions (`area_home_values.support_clip_exemptions`), none today. `cap_derived_totals_to_donor_range` takes `localised`, the rows whose main residence the localiser rewrote (none by default), and raises each such row's largest property total by its main residence's excess over the donor's largest, so that excess never takes other property away (other property past the donor's room still is); every other row's largest total is the donor's, so without the localiser the cap is as before. Its `*_capped_rows` fields now count the distinct true source households it adjusted among clone-0 rows, rows of zero weight included, in `extreme_count` bands (C-40), and the stage reads its frame's sources with `record_source_households`. +- **Property and uprating helpers.** `property_localisation.localise_main_residence_values` is the value half of `localise_values`, which now calls it; its refusal of a constituency without knots names no constituency. `regional_uprating.regional_reference_factors` is the stage's factor, factored out of `uprate_household_property_by_region`, which gains `denominator_column` (default `main_residence_value`): the owners stay the rows with a positive main residence, and the mean is taken over the declared column. The stage transform reads its operation's optional `denominator` (`main_residence_value` or `main_residence_value_rank_basis`, refusing any other), and the packaged stage declares none. It refuses households carrying `main_residence_value_rank_basis` while it divides by `main_residence_value`, since form F's localised homes land on their knots only where the stage divides by the kept draw (C-15). +- **Gate checks** (`stage_health`), registered by no gate yet: `atomic_mapping_integrity` holds an area stage's recorded validation to an accepted outcome (`pass` by default), the declared scope and SHA-256 digests of the definition and one per declared support system; `area_home_values` is the regional home values ranking check for the WAS stage's block (every declared region rewritten, the sources at the knots' cap within the declared maximum, flagged where a band straddles it, the declared basis year, knots level and rank basis, and clone-0 counting), with no value or sum to check; `copy_area_agreement` reads the phase's frame and holds every household a declared copy layer minted equal to its immediate parent, the id with that layer's copy-id field cleared (`copy_id_table.parent`), on the layer's declared columns. A declared list given as one string, a sources-at-cap entry that is neither a count nor a band, and household ids the copy-id table cannot decode fail by name. Every failure text names the check and the field, never a count or a value. `wealth_coherence`'s `capped_rows` detail carries the WAS cap's bands as printed, beside its counts. +- **Shared** (`microcosm.build.copy_agreement`, names no country): `rows_agree_with_parents(table, *, id_column, parent_ids, rows, columns)` compares each flagged row with its parent column by column, missing on both counting as equal, and returns per-column booleans and count-free texts. +- **Bindings** (`battery_bindings`). The stage-health binding hands the phase's frame to `copy_area_agreement` as well as to the band donor check, and routes the new checks' parameter keys (`knots_level`, `accepted_outcomes`, `validation_scope`, `support_systems`, `columns_by_copy_layer`). The UK enum-domain evaluator rebuilds the shared gate's result without counts or examples (C-32): a failure names the column and its domain, and the details keep `columns_checked`, `allowed_values` (and each nation's) and `columns_outside_domain`. Its refusal of a region with no nation names no region. `microcosm.build.gates` is untouched. +- **Count-free texts** (C-32). The geography ladder gate's and `local_authority_consistency_failures`' failure texts name the check and the column, without a row count, an example or the observed weighted share; the shares stay in the details. The ladder gate fails, naming the column, on a region it cannot read where it raised a text listing the values before, and the assignment summary's weight-length refusal carries no lengths. +- **Keys.** Every kernel the spine registry holds changes hash, since `graph_kernels`, which defines their classes, binds the WAS stage's new helpers (`area_home_values`, `property_localisation`, `regional_uprating`, `rowwise_geography`, `microcosm.graph`'s randomness and canonical encoding), adds `was_wealth` and `was_lisa` to the counting stages and has `regional_home_values` bind `area_home_values`. So do the spine gates (`stage_health`, `battery_bindings`), the full battery's preflight and calibrated gates (`battery_bindings`, `geography_ladder`), the national gates (also `copy_agreement`, now among the registry's modules), and every dense, national and export kernel that hashes `geography_ladder` or `local_authority_input`. Of the gate phase codes only the full battery's terminal code moves, as it lists `local_authority_input`, so the calibrated gates' `terminal_code` parameter changes too. No transport kernel (NZ, BE, AM) moves, and no US module imports an edited one: `microcosm.build.gates` and `country_spec` are untouched, and only the UK stage-health checks import `copy_agreement`. No declaration, generated file, `gates.json` or schema changes. +- **Forms.** A failing enum-domain, ladder or local-authority result carries no row count or example, and a passing enum result's details lose the empty `invalid_counts` and `invalid_examples` and gain an empty `columns_outside_domain`. The WAS stage's capped-row fields print bands, which `wealth_coherence`'s details carry. +- **Tests.** On invented households, toy knots and a toy regional reference: form F against the public helpers and by hand, the uprating landing localised homes on the knots within one ulp, row-order independence on non-integer draws and weights, ties broken on the declared stream, a copy keeping its parent's source ranked on its own key, clone-0 evidence at K = 1, 2 and 3, the evidence counted in distinct sources among clone 0's rows of positive weight, the count-free refusals, the declaration's whole-or-nothing form, the dense roster's operation name and the packaged spec declaring the localiser and the uprating's denominator together (`test_uk_area_home_values.py`). In the WAS chain: segments 1 and 2 bit for bit with and without a localiser, every segment from 3 on reading the localised value, rows the localiser keeps bit for bit through all eleven segments, the clip leaving the exempt columns, the cap's room beside a localised home and only there, the banded capped counts in distinct sources among clone 0's rows, at the function and through the stage, the stage handing the cap its localised rows, and an end-to-end run of a declared stage on the committed knots followed by the uprating with the declared denominator, which refuses the localised households without it. The value half against `localise_values`, the factor helper and the default keyword against the stage's former loop bit for bit, each new check passing and failing on hand-built evidence (the mapping-integrity check also on the area-draw transform's own evidence over toy supports, the copy check on an SPI copy and a band donor of a later clone), `rows_agree_with_parents` on invented tables (`test_copy_agreement.py`), and the enum, ladder and local-authority texts' pass and fail forms with no digit of data. diff --git a/changelog.d/931-spine-localises-council-tax.changed.md b/changelog.d/931-spine-localises-council-tax.changed.md new file mode 100644 index 0000000000..1a840f84b7 --- /dev/null +++ b/changelog.d/931-spine-localises-council-tax.changed.md @@ -0,0 +1,69 @@ +Localise Great Britain council tax by each clone's local authority just before the UK spine's WAS chain, and carry each bill's provenance from stage 5 (#931). The council-tax localisation #931 group (d) added unwired is now declared: `council_tax_localise` runs after `private_rent` and before `was_wealth`, in a graph version of its own, so the packaged spine has 63 stages. Stage 5 (`frs_council_tax`) writes `council_tax_bill_provenance` beside the bill, and the localisation reads it. The localisation keeps the paid bill from before it as `council_tax_frs_bill`, which the WAS chain's bill predictor reads. The release boundary drops both columns. + +- **Stages** (`sources.yaml`, then `source_stages.json` by the by-name merge): + - `frs_council_tax` declares `derive` after `impute_cell_means`, as `frs_council_tax_provenance_operation_parameters()` returns it, and outputs `council_tax_bill_provenance` after `council_tax`. Its imputation and nonnegative outputs are unchanged, and its notes gain one sentence on the flag. The transform writes the flag (`StringDtype("python")`) from the same branches that choose the bill, which stays bit for bit as before. `checkpoint_metadata()` returns the provenance receipt, `council_tax_provenance_evidence`: distinct source households before the copies, through `rounded_count`. The stage's declaration check expects `read_tables`, `impute_cell_means` and `derive`, with both reviewed parameter sets. + - `council_tax_localise` declares `council_tax_localise_operation_parameters()` verbatim and in order: + - `verify_vendored_fact_resource` for Scotland's Band D at build period 2024; + - `derive` for the kept bill; + - `localise_council_tax`; + - `record_mass_conservation_receipt`, coupled to the terminal family gate as `private_rent`'s is. + + Its two Band D resources are artifacts with `runtime_sha256_required`: `council_tax_band_d` (`council_tax_band_d_england_wales_2024_25.json`) and `scotland_band_d_facts` (`scotgov_band_d_council_tax_2024_25.json`). Its one output and nonnegative output is `council_tax_frs_bill`. It rewrites `council_tax`, `council_tax_rebate` and `council_tax_benefit_reported`. Its notes describe this behaviour. `was_wealth`'s and `was_lisa`'s declarations are unchanged. +- **Shared vocabulary**, for Max's review. The source-stage schema gains the strict `localise_council_tax` kind: every key is required and no other key is allowed. `ALLOWED_SOURCE_OPERATION_KINDS` gains the kind. +- **The WAS read.** `was_wealth.recipient_predictors`, which `was_lisa` calls, now takes its council-tax predictor from `council_tax_frs_bill`. That is the paid bill the localisation keeps from before it localises: `max(council_tax - council_tax_rebate, 0)` on the FRS bill, the expression the predictor computed before. The predictor frame keeps its columns and their order. A frame without the kept bill is refused, naming the column. So the bill predictor is bit for bit as before, and the drawn area does not enter it. The HBAI net income both stages read from the engine sees the localised bill and reduction. +- **The graph and the spine build.** + - `council_tax_localise` opens its own graph version (`_READER_ISOLATION_BOUNDARIES`). It rewrites the housing shell's person council tax reduction, which `private_rent`'s finite reader anchors on, so without a version of its own the compile would close a cycle between the two. `private_rent` reads the shell's cells, and `was_wealth` and `was_lisa` read the localisation's. The spine graph gains the stage and `council_tax_localise.boundary`. + - The stage consumes each household's draw inputs (geography key, region, IMD decile, council-tax band and band flag), its drawn area, system and local authority, the bill's provenance and the single-adult flag. It also consumes the support channel's branch of the copy lineage and its true source and clone index. Like the area draw, it reads the three conditioned supports as graph sources. Its cells are the kept bill and the three rewrites, all float64; stage 5's cells gain the flag, a string cell. + - In `graph_kernels`, the stage's module is `spine_council_tax`. It is bound with the helpers its transform computes through, among them `atomic_support_expectations`, `bounded_amounts` and `council_tax_band_d`, and counts as a counting stage. It runs on the bitwise stage kernel every spine stage runs on (F10 is open with Max). The H2 fixture's registry builds it on the toy supports at the committed Band D. + - `spine_build` builds the stage on the supports the area stages read, holding them to the spine's support pins first. So a support replaced after the spine was prepared is refused with the area stages' message before the stage builds anything. The support flags' help names the stage. The sidecar's stage records report no nonzero share of the flag or of the kept bill (`_UNREPORTED_SHARE_COLUMNS`), as for the root's area carriers. + - The geography-first roster gains `COUNCIL_TAX_LOCALISE_STAGE`, and `spine_council_tax` reads its stage name from there. So the dense line's posture check, the spine CLI's release candidate and the national line refuse every spine built before this change, naming the stage alone. +- **The release boundary.** `UK_RELEASE_EXPORT_DROPPED_COLUMNS["household"]` gains `council_tax_bill_provenance` and `council_tax_frs_bill`, which no engine variable reads. Neither is a reviewed extra of the export surface (`UK_ALLOWED_EXTRA_EXPORT_COLUMNS`), so the dense and national exports carry neither. +- **Tools.** + - `tools/graph_uk_spine_fixture.py` runs the localisation in its oracle (`UK_FIXTURE_STAGE_COUNT` 63). + - `tools/build_uk_release_input_coverage_manifest.py` gains the stage's family, in `private_rent`'s form. + - `tools/verify_uk_identity_stability.py`'s scope docstring now names the stage's in-place rewrites, which no receipted recompute reads. +- **Generated files.** + - `source_stages.json` regenerates by the by-name merge. Stage 5's unchanged operations keep their committed key order. + - The H2 fixture regenerates through the engine slot, and `--check` passes. Three files change: + - `uk_spine.json` gains the stage and its boundary; + - `sources/fixture.json` gains stage 5's new operation, output and notes sentence and the new stage; + - `sources/cgt_asset_type_facts.json` changes, because the tool sizes it to the oracle's frame before `cgt_residential_split`, which reads the stages the localised bill reaches. + + `property_rental_facts.json`, `property_purchase_targets.json`, `NORMALIZATION.md` and `PRODUCED_BY.txt` are unchanged. + - `release_input_coverage_manifest.json` regenerates without an H5, and `--check` passes. It gains the new family, and every family's source-manifest digest moves. +- **What moves.** + - Values, at one clone. Before `council_tax_localise` every existing column keeps its values: stage 5's bill is as before, and the flag is a new column. + - Great Britain's council tax, the reported council tax reduction and the rebate move by each household's ratio: its authority's 2024-25 Band D over the expected Band D of its own draw cell. + - Missing bills, and bills netted of a reduction of unknown size, take the 2024-25 statute where the band was observed. + - A reduction or rebate above the new bill is capped at it. + - SPI copies and income band donors move at their region-level cell. + - Northern Ireland's rows are unchanged. + + The WAS bill predictor is unchanged bit for bit. The HBAI income WAS and LISA read moves with the localised bill and reduction. So can their draws, the home values the WAS chain localises, and every later stage that reads wealth, council tax or the reduction. Above one clone, clone 0's FRS rows take one clone's council tax exactly, and the SPI-channel rows nest to the share class. + - Keys. + - Every spine node re-keys, and so does every dense and national node after the spine. Of the stage contracts only `frs_council_tax`'s moves, and `council_tax_localise`'s is new. The UK country and solve fingerprints move. No gate phase code moves. + - Outside the UK one kernel re-keys: the shared transport gate battery (`gates.battery@1`). Its bindings' source closure reaches `source_manifest`, where `ALLOWED_SOURCE_OPERATION_KINDS` lives. So NZ's terminal gate and the export, readback and package nodes after it re-key (`nz.gates.terminal`, `nz.export.prepare`, `nz.export.readback`, `nz.package`), the cause María accepted on 12 Nov. + - No other NZ, AM, BE or US key moves. In those four countries' bundles, the schema edit moves only the spec engine's grammar receipt. +- **Tests**, on invented data, the toy supports and H2. + - Engine-free: + - the 63 stages and their order, the localisation right after `private_rent` and right before `was_wealth`; + - no stage from the area draw to the localisation writing a draw input but the housing shell's band, and no stage after it writing its columns; + - stage 5's three operations and two outputs, the localisation's outputs, rewrites and mass receipt; + - the packaged stages equal to the reviewed declarations, and stage 5's transform writing the flag and its receipt from a pinned invented tab; + - the reader isolation and the cells each reader binds, the support readers, the consumes, and the counting stages; + - a pin that the housing shell's consumes hold no household area column; + - the roster refusing a spine without the stage, on the dense and the national lines; + - the support re-pin refusal for the stage; + - the release boundary dropping both columns on every line, and the sidecar reporting no share of either; + - the WAS predictor reading the kept bill and refusing a frame without it; + - the coverage family. + - Engine tier, through the slot: + - H2 parity; + - the driver's stage records leaving out both columns' shares; + - `required_build_stages` and the adapter's support readers gaining the stage; + - the localisation on all three systems at one clone and at two, after the real-engine BRMA and copy stages. Northern Ireland, an SPI copy among its rows, is left bit for bit. Great Britain's ratio rows take their cell's ratio. Reductions stay within the bill, and the kept bill is the paid bill from before the stage. + - After merge, H2's clone-first run at 63 stages: + - each ratio row's bill is its cell's ratio times its bill, with one where the cell holds one toy authority and other than one for some London and Scotland region-level rows, SPI-channel rows among them in each; + - household 12 and the households whose ids are multiples of 21 take the statute, and no other row does; + - every reduction lies within its bill, and the kept bill is the paid bill, which the WAS bill predictor reads, bit for bit, on the stage's output; + - clone 0's FRS rows at four clones are one clone's, bit for bit, and the stage's receipt is the same at every K but for the clone count. diff --git a/changelog.d/931-spine-localises-home-values.changed.md b/changelog.d/931-spine-localises-home-values.changed.md new file mode 100644 index 0000000000..8930049feb --- /dev/null +++ b/changelog.d/931-spine-localises-home-values.changed.md @@ -0,0 +1,37 @@ +Localise England and Wales home values inside the UK spine's WAS chain, retire `regional_home_values`, and register the spine geography gates (#931). The WAS home-value localiser #931 group (c) added unwired is now declared: `was_wealth` reads each England and Wales owner's main residence off its assigned constituency's knots after the chain's second segment, its later segments condition on that value, and `regional_property_uprating`, dividing by the draw the localiser keeps, returns each of those homes to its knot value. `regional_home_values`, which wrote the regions' Land Registry values after the support copies, retires, so the packaged spine has 62 stages. Its roster is now geography-first, so the dense line's posture check passes on a packaged spine whose sidecar records its geography block, while the national line still refuses every clone-first spine. (Later in #931 the national line builds on a geography-first spine of one clone once the area-basis exclusion is signed for it: `931-national-one-clone-areas`.) The spine battery checks the areas the spine now assigns, in one gates digest break: the area draws' mapping integrity, each later copy's agreement with its parent, the BRMA and local-authority domains and the geography ladder, and the area home values' ranking in place of the retired stage's; the local battery's BRMA recheck retires, so the local contract has six gates. + +- **Stages** (`sources.yaml`, then `source_stages.json` by the by-name merge): + - `was_wealth` declares `localise_main_residence_value` after the chain's fit, with `area_home_values.area_home_values_operation_parameters()` as returned: the segment-2 draw it reads, the rows it rewrites, the weighted mid-rank within each FRS region on its own fixed stream (`sha256-u53-v1`, `uk-spine-area-home-values-v1`, 0, 0; seed source NONE, C-43), the constituency knots, the region map, the factor it divides by and the rows it leaves. The two resources it reads are artifacts with `runtime_sha256_required` (`value_knots`, `high_value_home_value_knots.json`; `regional_reference`, `regional_land_values.json`). The kept draw, `main_residence_value_rank_basis`, is an output and a nonnegative output after `main_residence_value`; the release boundary drops it, as before. The support clip exempts `main_residence_value` and `property_wealth`, which a localised home can lift past the donor's range (C-18). A home read at the knots' cap takes the largest quotient the factor multiplies back to the cap or below, so the uprating never returns it one rounding above `uk_support`'s price-paid bound, which is the cap; the declared value says so. The stage's source now also cites the HM Land Registry knots and the public regional reference it reads, the provenance the retired stage's survey and source carried. The stage's notes are unchanged. + - `regional_property_uprating`'s operation declares `denominator: main_residence_value_rank_basis`, and its factor text and notes say the factor's mean is that draw's, over each region's FRS-base owners (C-15). + - `high_value_home_support`'s notes and its module's docstring no longer describe the retired dense value node: the stage selects on the localised values, and every copy keeps its parent's value, area and geography key. + - `regional_home_values` retires with its module (`uk_runtime/regional_home_values.py`) and its tests. The knots' constants it read stay in `area_home_values`, and `property_localisation.main_residence_rank_basis`, the reader of the column it stored, goes too. +- **Shared vocabulary**, for Max's review. The source-stage schema gains the strict `localise_main_residence_value` kind and an optional `denominator` (a string) on `uprate_to_regional_reference`; `assign_regional_home_values` leaves the schema and `ALLOWED_SOURCE_OPERATION_KINDS`, which gains `localise_main_residence_value` (C-42). +- **The graph and the spine build.** The localiser runs inside `was_wealth`, on the bitwise stage kernel every spine stage runs on (C-2; F10 is open with Max). `was_wealth` owns the kept draw's cell and `regional_property_uprating` consumes it, since the graph projects the stage's frame to what it consumes. `regional_home_values`' consumes, cells, module, helper bindings and counting-stage entry go, and neither the spine build, the H2 fixture's implementations nor the fixture tool's oracle (`UK_FIXTURE_STAGE_COUNT` 62) installs it. `UKWASWealthStageTransform.output_columns()` gives the kept draw where its stage declares the localiser (`UK_WAS_WEALTH_LOCALISED_OUTPUT_COLUMNS`, and `UK_WAS_WEALTH_LOCALISED_NONNEGATIVE_OUTPUT_COLUMNS` for the nonnegative ones), and the transform writes it beside the main residence, in the order the stage declares its outputs and the graph writes its cells, so the legacy plan and the graph give one column order (H2 parity holds the two to it). +- **The dense line's posture.** The packaged roster now has every geography-first name (`spine_posture.roster_failures` finds none missing), so `prepare_full_build` goes on past the posture check on a spine request and on a bound spine whose sidecar records its geography block, and the spine CLI's `--release-candidate` takes the dense release's 15 clones (C-38). A bound spine whose sidecar records no geography block is still refused, naming the block alone. +- **`--expected-constituency-vintage` retires**, as María ruled on 13 Nov. The dense line takes its areas from the spine, and no dense node read the ladder's vintage against the flag. `microcosm-build-uk` no longer takes it on either role, so a script that still passes it is refused as an unrecognized argument; the dense posture loses `expected_constituency_vintage`, its invariant and `UK_DENSE_CONSTITUENCY_VINTAGE`; and the run's recorded parameters no longer carry it. Records only: no kernel hashes `full_build_cli`, `rowwise_cli` or `rowwise_posture`, so no node or kernel key moves from it. The row-wise dataset tool's own flag and the geography ladder loader's keyword are unchanged. +- **Gates** (`uk/gates.json` and `uk_runtime.gate_scopes`, one digest break; no gate kind is added, so `country_spec`'s closed list, which NZ transport hashes, is untouched). + - At the assembled boundary, which closes after `area_brma`: `uk_atomic_area_mapping_integrity`, the `stage_health` check `atomic_mapping_integrity` on `atomic_area_assign`, whose recorded validation must have passed at the declared scope with SHA-256 digests of its definition and of the three conditioned supports. + - At the transferred boundary, after `spi_copy_areas`, the last spine stage that writes the area columns: + - `uk_spi_copy_area_mapping_integrity`, the same check on `spi_copy_areas`, which also accepts the `not_run` the stage records on a frame without a copy; + - `uk_copy_area_agreement`, the check `copy_area_agreement`, declared on `cgt_residential_split`, the last copy layer: every high-value support, CGT support split, CGT incidence clone and CGT residential split copy equals its immediate parent on the columns `spi_copy_areas` rewrites and on `region` and `household_clone_index`, and every SPI copy and income band donor on those two alone (C-11, C-20); + - `uk_local_authority_enum_domain`: every `local_authority` is a member of the installed engine's `LocalAuthority` enum, whose domain the spine battery's rules engine supplies (microcosm#953); + - `uk_geography_ladder`, the ladder gate (`spine_agreement`) on the transferred frame, the SPI copies' own ladders and the national line's among them (C-31); its London share is a population fact, so a synthetic smoke build records a failure without blocking on it; + - `uk_brma_enum_domain`, moved from the assembled boundary, since `spi_copy_areas` gives the SPI copies and income band donors BRMAs of their own. + - `uk_stage_regional_home_values_ranking` becomes `uk_stage_area_home_values_ranking`: the `area_home_values` check on `was_wealth`'s block, at `knots_level: constituency`. Its `maximum_sources_at_cap` is re-derived for the constituency knots rather than kept at the retired stage's zero: a weighted mid-rank stays below 1, so a source reads the knots' cap only where its constituency's knots reach the cap below the top grid point, which the committed knots do in a few constituencies and only near the top of the rank. The maximum is 9, the top of the receipt's 3-9 band, so the gate fails once the receipt prints 10-29 or more; the count is clone 0's, so it holds at any clone count. `stage_health`'s `regional_home_values` check retires with the stage. + - `uk_stage_was_wealth_support` declares the WAS support clip's exemption, `exempt_columns: [main_residence_value, property_wealth]`, and drops those two columns' row allowances. + - `uk_local_brma_enum_domain` retires: the dense line takes its BRMAs from the spine, and the transferred `uk_brma_enum_domain`, which its notes name as the replacement, checks them. The local contract has six gates (`UK_LOCAL_GATE_SCOPE`, the local scope's exclusion reasons), and `full_gates.classify_full_gate_outcomes`'s one structural local stop is the ladder gate. + - Stale notes are reworded without figures: `uk_support`'s, `uk_brma_enum_domain`'s and the terminal band check's, which named the dense line's retired nodes or `frs_brma`, and the high-value support gate's, which now says what its stratum ranks and what its copies keep. `uk_support`'s also says that the price-paid bounds give `main_residence_value` the knots' cap and `property_wealth` the cap plus the WAS donor bound's headroom over the main residence. `uk_local_area_support`'s notes are unchanged; `uk_stage_was_wealth_support`'s still describe the clip before the exemption and are left for a later spine digest break. + - The data shard's mirrors follow (`microcosm.data.contract`): the entries' gates and phases, the evidence entries, the spine certification part's scope, the dense line's six local entries and the certification's excluded ids, and the digests below; its test fixture's enum details follow the UK binding's count-free form (C-32). +- **Tools.** + - `tools/verify_uk_identity_stability.py`: E5 follows the denominator the packaged `regional_property_uprating` declares, read as the stage reads it. Dividing by the kept draw, the stage leaves no fixed point to re-apply, so E5 recomputes the main residence from that draw through both steps, the area home values (each owner's weighted mid-rank within its FRS region at the scoped frame's household weights, ties on the declared stream, its constituency's knots, divided by the region's factor) and the uprating, and holds the stored values to it at one rounding generation. The property total is named under the receipt's new `not_recomputed`, since the WAS chain drew it on the localised value; on a frame without the kept draw, a release, both property columns are, and the receipt records `uprating_denominator`. A `denominator` keyword runs the fixed-point form for a stage that divides by the main residence. The scope (`_frame_as_stage_saw`) no longer takes `regional_home_values`' rewrite back: `_SCOPE_REVERSALS` holds the salary-sacrifice conversion alone, and the reversal of `regional_home_values`' rewrite and its last-writer guard retire, since no stage after the uprating writes either property column. + - `tools/build_uk_release_input_coverage_manifest.py`: `regional_home_values`' family goes, and `was_wealth`'s carries the kept draw. + - `tools/graph_uk_spine_fixture.py`: the oracle runs the localiser through `was_wealth`'s declaration, on the committed knots and regional reference, without `regional_home_values` (62 stages). +- **Generated files.** `source_stages.json` regenerates by the by-name merge, which also aligns a stage's artifacts by role and its operations by kind, so `was_wealth`'s unchanged entries keep their committed key order and number spelling where its artifact and operation lists grew. The H2 fixture regenerates through the engine slot and `--check` passes: `uk_spine.json`, `sources/fixture.json`'s stage payloads, and the purchase targets and CGT asset-type facts the tool sizes to the oracle's frames, which move with the localised home values; `NORMALIZATION.md` and `PRODUCED_BY.txt` are unchanged. `release_input_coverage_manifest.json` regenerates without an H5 and `--check` passes: `regional_home_values`' family goes, `was_wealth`'s outputs gain the kept draw, and every family's source-manifest digest moves. +- **Defaults taken.** + - From the #931 group (c) design: C-2 (bitwise; F10 is open with Max), C-15 (form F), C-16 and C-44 (the localiser's evidence in rounded and banded clone-0 sources), C-17 (the rank within the FRS region, as ruled on 6 Nov), C-18 (the clip's exemptions and the cap's room), C-19 (support copies keep value and area), C-20 and C-31 (the gate design, the ladder at the transferred boundary), C-32 (count-free gate texts), C-38 (the spine's 15-clone release candidate on the geography-first roster), C-42 (the retired kind) and C-43 (seed source NONE on the localiser's own stream). + - Beyond it, each listed for María with its options in the commit message: the operation's place right after the chain's fit and the kept draw's right after `main_residence_value`, declared nonnegative; a plain-string `denominator` in the shared schema; `output_columns()` reading the declaration; the sources-at-cap maximum of 9; the new gates' places in `gates.json`; the WAS support gate dropping the exempt columns' allowances; no `evidence_absent_blocks` on the two domain gates; the ladder gate on its evaluator's defaults; the copy agreement holding later copies on `region` and `household_clone_index` too; the zero-weight strata row kept; E5's recompute through both steps, its `not_recomputed` field and its `denominator` keyword; the smoke run reading the new gates as not blocking rather than passed, with the ladder's and the ranking's failures held to their population facts; a home read at the knots' cap stepped down to the largest quotient the factor multiplies back to the cap or below, and the declared value saying so; `was_wealth`'s source citing the knots and the regional reference now; and the merge aligning artifacts by role and operations by kind. +- **What moves.** + - Values, at one clone: England and Wales owners' main residences come from their constituency's knots at their rank within their FRS region, in place of their region's knots after the support copies, and for those owners the chain's segments 3 to 11 (other property, pensions, financial wealth, mortgages and the rest) condition on them; the regional uprating leaves them at the knot value, to one rounding, and a home at the knots' cap at the cap or below; Scotland's and Northern Ireland's main residences are as before. High-value home support selects on the localised values, and its copies keep their parent's value and area (C-19), where `regional_home_values` gave each copy its own rank in its region's top stratum. The stages that read home values or property wealth after it move with them: property components and purchases, Universal Credit capital, CGT selection and redraw, ISA subscriptions and private school attendance. Above one clone, the rank and the factor pool over the clones, so the values nest approximately. + - Keys: every spine node, and every dense and national node through the country fingerprint, a bound spine's checkpoint import too, since the spine gate digests it holds a stored report to move (below). Of the stage contracts only `was_wealth`'s and `regional_property_uprating`'s move, and `regional_home_values`' goes. Outside the UK one kernel re-keys, the shared transport gate battery (`gates.battery@1`), whose bindings' source closure reaches `source_manifest`, where `ALLOWED_SOURCE_OPERATION_KINDS` lives: so NZ's terminal gate and the export, readback and package nodes after it re-key (`nz.gates.terminal`, `nz.export.prepare`, `nz.export.readback`, `nz.package`), the cause María accepted on 12 Nov. No other NZ, AM, BE or US key moves; in those four countries' bundles the schema edit moves only the spec engine's grammar receipt. + - Gates: spine gate reports gain the five new gates and carry the ranking gate under its new id, and a failure of one blocks the spine at its boundary as every release-blocking spine gate does (the ladder and ranking gates' not on a synthetic smoke build); local gate reports lose the BRMA recheck. The gate battery's manifest, policy and fingerprint digests move, and with them the data shard's pins of the full battery, of the spine and release-cut certification parts and of the dense line's local battery; the calibration seam's part does not move. So a spine built before this change no longer binds: the strict loader holds its stored gate report's digests to the current declarations. The transferred spine gate's own code (`gate_phase_codes`) now also hashes what its new gates compute through, `copy_agreement`, `geography_ladder`, `local_authority_input`, `rowwise_geography`, `terminal_gates` and `microcosm.frame.materialize`; the assembled gate's own code is unchanged. The full build's calibrated gates' own code moves too, since it hashes `calibration_run`, `local_authority_input` and `terminal_gates`, whose texts this change rewords (keys only); the preflight's does not. Outside the UK no key moves beyond the four NZ nodes above. +- **Tests** (invented data and the packaged spec): the 62 stages; the declared operation verbatim, its artifacts, output and exemptions; the denominator; the kept draw's cell and the uprating's read of it; the packaged roster passing the posture check, its sidecar with a block passing and one without failing on the block alone; the dense preparation going on past the check on both routes; the spine CLI taking 15 clones on the packaged roster; the retired flag refused on either role and absent from the posture and the recorded parameters; the acceptance receipt's pending roster without `regional_home_values`; the knots' region map and the release's drop of the kept draw, moved from the retired tests; the K-sensitive table's sources-at-cap row read off the area home values gate, and rows for the knots' cap, the high-value support stratum and its copies; the zero-weight strata row, now kept, since its every maximum is zero rows; the declared battery's entries, phases and strict-evidence entries; the two area domains at the transferred boundary and the local BRMA recheck gone; each packaged geography gate's parameters held to the code's constants (the scope and support systems the area stages record, the copy-id layers and the copy stage's rewritten columns, the localiser's regions, knots basis year and level and rank basis); the cap maximum resting on the committed knots, passing through the receipt's 3-9 band and failing from 10-29 at any clone count; the WAS support gate passing a receipt that marks the two columns exempt; an off-domain BRMA blocking the transferred spine gate in every posture; the six-gate local scope, its one structural stop, and the stage-health frame readers; each gate battery phase keying on its own reach. E5 on frames the area home values, the uprating and the high-value support built (with and without the CGT layers): green with the copies folded back, red when they stay in scope or a stored main residence moves off the two steps, the property total named as not recomputed, both property columns named on a release, and the fixed-point form where the stage divides by the main residence; the scope taking nothing back for the home values and no later writer of either column; the SPI rows in E5's scope, red without them; at two geographic clones, green over both and red on clone 0 alone, since the rank pools over the clones. The coverage manifest's `was_wealth` family with the kept draw, and no `regional_home_values` family. The WAS stage writing its outputs, and the graph its cells, in the stage's declared order. A home read at the knots' cap coming back from the uprating at the cap or below, within the price-paid bound, which the plain quotient fails; the release export dropping the kept draw from a frame that carries it; every high-value support copy keeping its parent's localised value, kept draw, property total, seat and geography key, and the stratum selected on the localised value rather than the kept draw; and the stage kernel the localiser runs in pinned BITWISE (C-2), so a ruling on F10 changes the pin on purpose. On the engine tier: H2 parity; `required_build_stages` without `regional_home_values`; the clone-first H2 test at 62 stages, with every England and Wales owner localised after `was_wealth`, Scotland's draws kept bit for bit, the kept draw reaching the last population and dropped at the release boundary, the localiser's block carrying its clone count at every K, and its values in the share class above one clone; `was_wealth`'s values equal to the area home values on the stage's input frame bit for bit, the uprating returning each localised home to its knot within one rounding (and a home at the cap to the cap or below), and every high-value support copy equal to its parent on every column but its ids, at one clone and at four; the integration smoke run's spine battery, in which each new or moved spine gate passes or, a population fact the fixture is not, fails recorded and not blocking (on H2 the ladder gate's London share, since its root cycles four regions, and the ranking gate's declared regions, since it holds three of England and Wales's), and those two fail on their population facts alone. diff --git a/changelog.d/931-take-up-counts-per-clone.changed.md b/changelog.d/931-take-up-counts-per-clone.changed.md new file mode 100644 index 0000000000..95bf6cd60a --- /dev/null +++ b/changelog.d/931-take-up-counts-per-clone.changed.md @@ -0,0 +1,9 @@ +`frs_take_up`'s Child Benefit, Pension Credit and Universal Credit draws and `frs_person_draws`' apprentice walk now count within each geographic clone, so clone 0 of a clone-first spine takes a one-clone spine's flags (#931; María's rulings of 6 and 10 Nov). Before, the anchored residual took `int(rate × n)` once over every clone's benefit units, and the walk interleaved every clone's employees and stopped once, so clone 0's flags were not one clone's. The stages' other draws were already keyed on each row's own id. + +- **Shared `strata`** (for Max's review). `stochastic_assignment.assign_binary_with_anchored_residual` takes a keyword-only `strata`, one integer per row. Each stratum's rows take the flags they take alone: their own `int(rate × n)` target, the need their own anchors leave and the threshold their own non-anchored rows fill. Without `strata` the function runs as before; with every row in one stratum it gives the same flags bit for bit, and without an anchor strata change nothing. Strata that do not align with the draws, and strata without an integer dtype, every column with a missing label among them, are refused in fixed sentences that name no count, label or row. The module names no country, and only UK modules and UK tools import it. +- **frs_take_up.** `derive_frs_take_up` takes `clone_index` and passes it as `strata` to the three anchored draws. The stage reads each benefit unit's clone from `household_clone_index` through the person memberships with the new `geographic_clone_indices`, which shares its mapping with `geographic_clone_zero_rows`, so its consumes do not change. A frame without the column passes None. +- **frs_person_draws.** `derive_is_apprentice` takes `person_clone_index`, as do `derive_frs_person_draws` and `derive_frs_person_draws_with_receipt`, and each clone walks its own employees to the rate times its own weight aged 21 to 24, with the unchanged nearest-target walk. The stage reads the index through `geographic_clone_indices`. The receipt keeps its keys and their formulas: the masses and the realized weight are sums over every clone, `target` is the rate times the weight aged 21 to 24 over every clone, and `cap_bound` compares the eligible weight over every clone with that target, as the `apprentice_draw` stage-health check recomputes them. The supports are still counted among clone 0's people in true source households, and withheld below the floor of three as before. A misaligned or non-integer index, or a `person_clone_zero` that disagrees with it, is refused in a fixed sentence that names no count or row. +- **E4** (`tools/verify_uk_identity_stability.py`) gives both recomputes each row's clone, read once from the scoped frame and carried by id onto the permuted tables, and holds stored `is_apprentice` flags to the walk within each clone. Its `tolerance_policy` text says so. +- **What moves.** At one clone, no value: both stages' outputs and receipts are as before, bit for bit, with or without a one-clone index, and E4's receipt differs only in its `tolerance_policy` text. On the synthetic H2 fixture the legacy oracle's final frame, its stage records and every stage's evidence digest are identical before and after, and the fixture regenerates byte-identically. Above one clone, which no build has yet run on this code, the three anchored flags and `is_apprentice` move, each clone meeting its own target. Clone 0's anchored flags are one clone's exactly. Its `is_apprentice` flags are one clone's at any power-of-two clone count, and elsewhere except at a near-tie of the walk. Whatever reads these flags follows: `frs_brma` reads `would_claim_uc` through `LHA_category`. (Later in #931 `area_brma`, in its place, does.) `spi_benefit_coherence`'s Universal Credit redraw and `uc_take_up` still count over every clone. +- **Keys.** The frs_spine root and 49 of the 57 stages after it hash `stochastic_assignment`, so every spine node re-keys, and both spine gate nodes with them. Of the gate phases' own codes, the transferred spine phase's hashes `stochastic_assignment` and moves; the assembled phase's does not. On a bound spine the national gates re-key, and so do the full build's calibrated gates and the nodes after them, through the terminal phase's code, which hashes `frs_take_up`; they do so without a re-solve, since the preflight's code and the solve's key leave that code out. No node hashes the identity tool. Of the 48 kernels the transport registry registers, from which NZ's graph is composed, all but the gate battery keep their hashes. The battery's key (`gates.battery@1`) binds the sources its gate bindings compute through, US keyed draws among them, and since microcosm#1585 those import `stochastic_assignment`, so NZ's terminal gate node re-keys, and so do the export, readback and package nodes that chain on it. No NZ, BE or AM module imports an edited file. The US keyed draws take only `stable_identity_uniforms` from `stochastic_assignment`, which this change leaves as it was, so no US draw moves; but the US seed protocol attests the module, so the seed protocol's and the seed map's digests move, and with them the US bundle's spec hash (later in #931 these pins are re-derived). No declaration, generated file, `gates.json` or schema changes: `frs_take_up`'s notes, `frs_person_draws`' `rate_derivation` and the take-up contract's policy line follow with the geography declarations. (Later in #931 they do: `931-spine-assigns-areas`.) +- **Tests** (invented data). The shared keyword against its arithmetic before strata, without strata and with one stratum in three integer dtypes; two hand-built strata, under relabelling and row permutation; strata without an anchor; and the refusals. `frs_take_up` on the clone-first stand-in at three clones over a grid of rates, and bit for bit at one clone. The walk at two and four clones, its supports and cap bound at three, an exact tie of the walk that clone 0 keeps at every power-of-two clone count up to 16 and breaks by at most one person elsewhere, the stage at four clones, and bit for bit at one clone. Both stages through `geographic_expand` at 2, 3, 7 and 15 clones, where counting over every clone instead moves clone 0 for each of the four draws. E4 at three clones at two Universal Credit rates, and the walk check within each clone. The engine-tier test at four clones, where clone 0 through `frs_brma` is now the one-clone frame on every column. diff --git a/changelog.d/931-transport-helper-bindings.changed.md b/changelog.d/931-transport-helper-bindings.changed.md new file mode 100644 index 0000000000..fa420cbc47 --- /dev/null +++ b/changelog.d/931-transport-helper-bindings.changed.md @@ -0,0 +1,11 @@ +Every transport kernel now hashes each first-party module that its own code, or an object the transport registry hands it, computes through, as far as the derived binding rule the UK stages and kernels are held to can see, apart from the core modules every node runs through and two checks that only refuse (#931). `source_hash` hashes only the files it is given, so a transport node that called a function in a module its hash did not name replayed a stale cached output when that module changed. Only node keys move. + +- **The rule, shared.** The rule's walk moved from `test_uk_graph` to `test_support/microcosm_build/derived_binding_rule.py`. Each country passes its tables as a declaration there, and the UK's walks are unchanged. The rule's refusal check now counts a store into a container the function itself creates as local. +- **The transport's tables.** `test_transport_binding_rule` holds 45 of the 48 kernels `build_transport_registry` registers to the rule: 37 of the transport package's 40, the shared atomic geography's four, the ordered solve and the three rules kernels. The other three, main's income-evidence kernels, are held to it later in #931. The transport has its own tables, each entry with a reason a test checks: the eight core modules, 19 core definitions, two refusal-only checks (`graph_inputs.require_outputs` and `reference_profiles._validate_interval_resolution`) and one stop, `target_kernels._COMPILE_MODULES`, which only implementation hashes read. The rule also walks what the registry hands the kernels: the `AxiomEngine` behind the rules kernels and the input closure, and the gate registry behind the battery, whose key also binds its bindings' source closure. +- **What is bound.** 79 bindings in 36 kernels: + - the canonical JSON encoder in 32 kernels. It joins the compile list (`transport/compile_modules.py`) for the seven kernels that hash it, and `_GRAPH_RUNTIME` now excludes only the kernel protocol and the declarations; + - the target surface's factor evidence and its Chronicle feed identity (`reference_factors`, `ledger_artifact`, `chronicle_epoch`), in the kernels that decode or check a surface; + - `artifact_types`, `calibrate.target`, `graph_atomic_geography` and the Axiom input surface where kernels name them; the gate report records (`country_spec`, `gate_battery`, `gates`) in the export and package kernels; `build.trace` in the export readback; `takeup_kernels` in the battery; `frame.rules` and `frame.materialize` in the input closure; `calibrate.hierarchy` and `monetary_binding` where targets or matrices are rebuilt; `frame.exact_money` in the counterfactual bridge, as in the zero solve; + - the ordered solve's internals (exact-k, the L0 gates and their initialization record, the hierarchy, the monetary binding and the encoder) in `calibrate.ordered_adam@1`'s own half, so `calibrate.adam@1`'s hash, which UK kernels and the parity pins embed, does not move. +- **The import record.** `test_transport_compile_modules` drops microcosm#1497's record of unclassified imports, its kernel groups and the record check. It keeps the compile list's closure checks and the geography-label re-key test, and now checks that no transport kernel hashes the compile list. The import scan's record helpers, which nothing else used, go. +- **What moves.** Node keys only. 43 of the 48 transport kernel hashes move, main's three income-evidence kernels among them; the four atomic geography kernels and the person location keep theirs. All 369 nodes of New Zealand's packaged graph re-key: 365 directly, and the atomic geography and person location nodes by chaining. Belgium composes no graph. No UK kernel hash moves (0 of 101), nor do `calibrate.adam@1`'s or `simulate.rules@1`'s, and nothing of the US's. No value, declaration or generated file moves. diff --git a/changelog.d/931-uk-annual-income-other-spells-growth.changed.md b/changelog.d/931-uk-annual-income-other-spells-growth.changed.md new file mode 100644 index 0000000000..d051137ef9 --- /dev/null +++ b/changelog.d/931-uk-annual-income-other-spells-growth.changed.md @@ -0,0 +1 @@ +The annual-income simulation carries each input's other spells by policyengine-uk's own growth parameters (#931). The factor is the product, over each year after the dataset's up to the measurement year, of one plus the engine's rate in the series its dataset projection uses for the input: `gov.economic_assumptions.yoy_growth.obr.average_earnings` for pay and `gov.economic_assumptions.yoy_growth.obr.per_capita.mixed_income` for profit. It replaces the median of the engine's year-on-year ratios over the rows a simulation held, a statistic of whichever engine block ran, so every per-clone block and one engine over the pool now use the same factor. The engine's rows must still agree with it within 1e-5, or the resolver refuses. This moves data on the national and dense lines: an annual input with other spells can move by one float32 step, and with it the `*_annual` matrix entries, calibrated weights and H5 bytes. `other_spells_engine_factors` changes in every resolver receipt that carries it. diff --git a/changelog.d/931-uk-atomic-geography-wiring.changed.md b/changelog.d/931-uk-atomic-geography-wiring.changed.md index 42edfd16e6..dbd45e16d1 100644 --- a/changelog.d/931-uk-atomic-geography-wiring.changed.md +++ b/changelog.d/931-uk-atomic-geography-wiring.changed.md @@ -1 +1 @@ -The UK full build assigns geography on the shared atomic-geography operators: households are keyed by their explicit spine lineage (`uk.full.identity`), one atomic area is drawn per household by census household count within FRS region under a keyed stream, larger geographies derive from the support mappings, and the shared gate must pass before the pool checkpoint. `--geography-assignment legacy` keeps the sequential ladder draw for measurement builds; release candidates refuse it and pin the three supports (#931). +The UK full build assigns geography on the shared atomic-geography operators: households are keyed by their explicit spine lineage (`uk.full.identity`), one atomic area is drawn per household by census household count within FRS region under a keyed stream, larger geographies derive from the support mappings, and the shared gate must pass before the pool checkpoint. `--geography-assignment legacy` keeps the sequential ladder draw for measurement builds; release candidates refuse it and pin the three supports (#931). Later in #931 the dense line retires these nodes and flags and takes its geographic clones, areas and sampling from the spine it binds, whose own stages assign them. diff --git a/changelog.d/931-uk-child-benefit-age-noise-tolerance.changed.md b/changelog.d/931-uk-child-benefit-age-noise-tolerance.changed.md new file mode 100644 index 0000000000..4f76667a67 --- /dev/null +++ b/changelog.d/931-uk-child-benefit-age-noise-tolerance.changed.md @@ -0,0 +1,6 @@ +The UK Child Benefit take-up stage gate (`uk_stage_child_benefit_take_up`) now holds each measured age within the larger of 0.05 and three standard errors of a share drawn at its published rate over its child rows (#931; María's ruling of 10 Nov). A measured age, one with at least `minimum_age_child_rows` child rows whose solved rate was not clipped, was held within a fixed `maximum_age_claim_rate_deviation`, 0.05, of HMRC's published rate for it. The band is `maximum_age_claim_rate_standard_errors` × sqrt(p (1 − p) / n), with the new parameter declared as 3, and p and n the receipt's `published_rate` and `eligible_child_rows` for the age. It treats the age's child rows as independent draws at p, an approximation as ruled: in the stage, siblings share their family's draw, reporters always claim and the realised rate is weighted. + +- **Why.** #931's keyed redraws move each age's realised claim rate by sampling noise. At an age with few child rows that noise can exceed a fixed 0.05, and because the draws are keyed, such a failure would repeat on every rebuild, including the national build. +- **Floor.** Three standard errors exceed 0.05 only below 3,600 p (1 − p) child rows, so an age with more rows keeps 0.05 and only thin ages loosen. At a published rate of 0 or 1 the term is 0, so the floor holds. The overall claim-rate check and the opt-out check are unchanged. +- **Report.** A failing age's line names the tolerance used at that age, "(tolerance t: the larger of 0.05 and 3 standard errors of a share at the published rate over that age's child rows)", and the gate's details count the measured ages held to more than the floor (`ages_tolerance_above_floor`). The gate refuses a missing, non-finite or non-positive `maximum_age_claim_rate_standard_errors`. The stage-health binding now routes the parameter, since the battery refuses a declared parameter its binding does not route. +- **What moves.** The gate's verdicts at thin ages: an age whose realised rate is more than 0.05 from its published rate but within its three standard errors now passes. No stage output or generated file moves: the gate reads the receipt and draws nothing. Of the 99 UK kernel hashes 14 move, and no phase code: the spine gates' (`stage_health`, `battery_bindings` and the solve fingerprint, which `gates.json` is part of); `uc_deduction_attributes`' and `uc_take_up`'s (`stage_health`); the full build's target compilation, selection, measures, problem and rotated holdout (the solve fingerprint), its preflight and calibrated gates (`battery_bindings` and the solve or country fingerprint) and its certification readiness (`microcosm.data.contract`, which the re-pin edits); and the national targets, problem and gates (the country fingerprint; the gates also hash the gate registry's modules). Node keys move from the assembled spine gate on: 66 of the gated spine's 83 nodes. On a bound spine the full build re-keys from its target compilation on, so it re-solves, and the national line re-keys all but the bound checkpoint. The contract re-pins the gate battery's policy, gates-manifest and spec-fingerprint digests and the release certification's spine part; the calibration seam, release cut and dense line digests do not move. diff --git a/changelog.d/931-uk-clone-receipts.changed.md b/changelog.d/931-uk-clone-receipts.changed.md new file mode 100644 index 0000000000..55c2a3ddde --- /dev/null +++ b/changelog.d/931-uk-clone-receipts.changed.md @@ -0,0 +1,18 @@ +The UK stage receipts and stage checks now read a spine of more than one geographic clone (#931; María's rulings of 10 Nov). Every new count is of distinct true source households, rounded through `receipt_counts`. The spine still builds one clone by default, so no value and no gate verdict moves. At one clone only the `spi_support_channel` and `spi_income_band_donors` receipts change, with the band donor check's details and failure texts and the clone count's refusal texts. + +- **Fold receipts.** Every FRS-trained fit folds its training rows through `household_lineage.fold_geographic_clones`: the SPI income stage's reference and stage-2 forests, the SPI employment and housing shells, the UC reporter redraw, UC capital and salary sacrifice. On more than one clone, each of these stages' evidence gains `geographic_fold` (`household_lineage.geographic_fold_receipt`). It gives the clone count; the source households the fit selected, and those whose clones differ on what the fit reads, both rounded as every receipt count is; and the share that differ, given only where ten or more of the selected households differ and none or ten or more do not, and otherwise the count of those that differ again. A one-clone spine's evidence is unchanged. +- **Twin check.** `cross_fit_transport` takes `twin_identities`, read only to find each recipient's twin among the training rows. The SPI income stage passes each copy's twin through its forests' fold, so the check reaches the copies of every clone. Each copy's uniforms stay keyed on its own identity; on one clone the two keys are the same. +- **`spi_support_channel`.** Its receipt gives the frame's clone count. Its two pension-age counts are rounded distinct source households carrying the mass beside them, on rows of positive weight: the pension-age base households', and those of clone 0's pension-age SPI copies. Both are the same at every clone count, and the masses are unchanged. +- **`spi_income_band_donors`.** + - At every seating, each funding row keeps its stratum and its rounded donor households and reads `<3` for its masses and factor. The receipt gains two booleans: `funding_within_share`, every factor at least one less the declared reallocation share, to the stage check's relative tolerance; and `funding_matches_bands`, the bands and the funding strata each carrying the reallocated mass. + - On a capped seating, each band's donors and carriers, and the donor count, are rounded distinct source households. Each band's weighted taxpayers and carriers' mean age, and the reallocated mass, read `<3`, and each band's carrier regions are left out. A full seating keeps the plan's exact band counts. + - The result keeps the exact figures, and its frame, out of its repr. + - A band short of candidates or seats is refused naming the band and, on a cloned spine, that a clone fell short, without a count. +- **The band donor check.** It reads the funding from the two booleans and requires one less the receipt's reallocation share to reach its minimum funding factor; its details drop the smallest funding factor. On a capped seating it recomputes the seats from the transferred phase's frame. A seat is a band donor row whose id holds no ordinal in a copy-id field minted after the band donors, since every later layer keeps the row it copies at its own id. Every declared band must seat a donor, every seat must hold one carrier, and each band's seats must split evenly across the clones; the bands' mass there is the stage's `funding_matches_bands`, since later layers reweight the seats. The details then carry the clone count, the seating, the mass scale, the booleans and the count of bands. No failure text gives a count. The stage-health binding hands the frame to this check alone, `copy_id_table` joins the gate registry's modules, and the transferred phase's own code hashes it. +- **Clone count.** `geographic_clones.geographic_clone_count` takes the calling stage and refuses a malformed clone index in fixed sentences naming it, with no count. The six support-clip stages, property components, the CGT support split and the fold receipts read the count through it, and the WAS wealth and Lifetime ISA stages bind the leaf for it. The copy-id field the clones' ids are minted in, `GEOGRAPHIC_COPY_ID_FIELD`, is named in the leaf, and `geographic_expand` re-exports it. +- **Clone-aware thresholds.** + - The six support-clip stages (WAS wealth, WAS Lifetime ISA, NTS bus travel, LCFS consumption, ETB VAT and ETB services) and property components give `clone_count` from two clones up. The support-clip check multiplies each declared row allowance by it, and the property components check its two row maxima. + - `regional_home_values` counts the source households at the value cap in clone 0's rows, reading each row's clone from its id through the copy-id table, so its check's one maximum holds at any clone count. (Later in #931 the stage retires, and the WAS chain's area home values count their sources at the knots' cap in clone 0's rows: `931-spine-localises-home-values`.) It refuses a frame whose clone index disagrees with its ids. + - The take-up signal receipt counts each flag's population support and distinct values in clone 0's rows, and the apprentice draw its supports. A figure measured over every clone is withheld where clone 0's count or the count over every clone is one or two. +- **Constants table.** `test_uk_k_sensitive_constants.py` classifies each count-like constant of the UK spine, and each count-like maximum in the packaged gates, by what it counts and how it meets the clone count, and pins it to the code. +- **What moves.** Node keys only. Every UK spine node re-keys. On a bound spine, so do the full build's identity, property value and private rent kernels, the dense population kernels, the export preparation kernels, the national problem, measures and gates, and the full gates with their terminal phase code. No generated file, declaration or `gates.json` changes. No transport kernel and no US, NZ or BE node hashes an edited module; only the US PUF-support builder's whole-tree resume fingerprint moves, as on any edit to packaged source. diff --git a/changelog.d/931-uk-copy-id-table.changed.md b/changelog.d/931-uk-copy-id-table.changed.md new file mode 100644 index 0000000000..27a4fdf8fd --- /dev/null +++ b/changelog.d/931-uk-copy-id-table.changed.md @@ -0,0 +1,6 @@ +The UK spine mints every copy row's ids from fixed decimal fields declared in `microcosm.build.uk_runtime.copy_id_table`, not from a power of ten above the largest id each copy layer sees (#931). Once the spine clones households at its start, that largest id would grow with the clone count K, so every layer's offset would move with K and the K=1 spine would stop being the clone-0 slice of the K=15 spine. Copy `k` in a field takes `id + k × radix` on all five id columns, so every copy id is a function of its source id and its copy path alone. The fields, least significant first: `vintage` (10 ordinals at 10^8, reserved for pooled FRS vintages, minted by nothing yet), `geographic_support` (100 at 10^9, for the clone-first spine), `spi_support_channel` (3 at 10^11: the SPI copy at 1, the income band donor at 2), `high_value_support` (6 at 10^12), `cgt_support_split` (10,000 at 10^13), `cgt_incidence_clone` (2 at 10^17) and `cgt_residential_split` (8 at 10^18, the top field). The largest mintable id is about 7.2e18, below int64's limit, which the module checks when it loads. + +- **Where.** `frs_spine` refuses raw ids at or above 10^8. The SPI support channel, the income band donors (whose sources are now read from their `{entity}_source_id` columns), the high-value split, the CGT support split, the incidence twin and the residential arms mint through the table, which refuses an ordinal outside its field, a row that already carries one and an int64 overflow. The CGT split refuses a household that would need more than 10,000 copies with a fixed message. The incidence anchor pairs each twin with the id less the twin field's radix, whatever the frame's largest id. The executor recovers each id-offset stage's parents at the stage's declared radix, refuses an incumbent id at or above it, and reads ids as exact int64 on any frame; the row-by-row walk it replaces read a row holding only integer and float columns as float64, inexact above 2^53. The UK modules' refusals on the copy path print counts, never ids: the new ones, and the older checks that listed up to five ids or printed the largest (uniqueness, links, sources, carriers and weights in the clone helpers, the SPI support and spine stages, the row-wise dataset and its tool's dry-run plans, the CGT stages and the spine sampler); the shared graph executor's own EXPAND checks, which every copy stage also passes through, are unchanged, and some of them still name ids. `create_uk_spi_support_tables` no longer takes an `id_multiplier`. The identity tool (`tools/verify_uk_identity_stability.py`) decodes the CGT support copies and the residential arms at their fields' declared radices and records the declared radix as each receipt's `id_multiplier`; it refuses, by count, a spine whose copy ids do not follow the table, as every pre-table spine with copy rows does, and such a spine is receipted with the tool at the commit that built it. +- **What moves.** Base FRS rows keep their ids. Every copy row's five id columns now sit at its parent row's ids plus its layer's offset: SPI copies 10^11, band donors 2×10^11, high-value copy k k×10^12, CGT copy k k×10^13, twins 10^17 and arm j j×10^18. Each used to be a power of ten above the largest id its layer saw. Relative id order is unchanged at K=1, so sorts, ties, ranks and folds are unchanged. Values move on copy rows through every draw keyed on the copy's own id: the SPI income stage's second stage, stages 19 to 24 and 27 to 41, every twin's Advani-Summers prior (so the CGT gains layer is re-realised), stages 48 to 54 and policyengine-uk's Warm Home Discount draw on copy households in spine engine calls. Base rows move second-order through pooled steps: the high-value selection, the regional value ranks, the landlord walk, the residential solve, the anchor, the CGT walks and calibration. Draws keyed on `person_source_id` do not move. National ids pass 2^53 from the incidence twin on, and at the CGT support split for any copy at ordinal 901 or more. The released file's identifiers, ranks of the build ids, do not move at K=1, while its `*_build_id` columns carry the new copy ids, so joining copy rows across versions goes through lineage (source id and copy path), not build ids. +- **Dense line.** The dense expand keeps its multiplier above the spine's largest id. A spine carrying residential arms (multiples of 10^18), as a production national spine does, leaves no int64 room for a second dense clone, so every dense entry point (the graph's `uk.full.expand`, the ladder and crosswalk clone routes and the row-wise tool's plans) refuses wherever the dense offset would overflow int64, in one message naming #931 and the clone-first spine. Dense builds with K > 1 of such a spine wait for #931's clone-first spine; K = 1, and a spine whose ids leave room (one without copy rows, say), expand as before. The dataset-size evaluation decodes dense ids back to spine ids on exact integers, 10^19 included. A caller can pass the multiplier the expansion recorded (`weight_tables(id_multiplier=)`); `tools/evaluate_uk_dataset_size.py` infers it from the spine's household ids or the clone-0 household ids, and a dataset the inferred multiplier does not fit is refused by count. A dense file that keeps its true sources decodes its spine stretch again instead of failing. Later in #931 the graph's `uk.full.expand` retires, as the dense line builds on the clone-first spine's own clones (`931-dense-on-the-clone-first-spine`), and the dense K pin (`931-dense-k-pin`) retires the size evaluation's decode: `weight_tables` loses its `id_multiplier` and `n_clones` arguments and reads a dense dataset's ids as its clone-first spine's, through the copy-id table's `geographic_support` field, refusing any other layout. +- **Declarations.** The `id_remapping` texts of the copy stages in `sources.yaml`, `source_stages.json` and the code dictionaries name the table field; the SPI support and band-donor operations gain the key, which the source schema now requires. The stages that read the table bind it into their hashes. The SPI support and band-donor stages also bind `spi_support`, whose clone helper mints their copies; the high-value split binds `cgt_support`, whose exact-total correction weighs its copies; and the CGT clone and anchor stages, which share `cgt_structure`, bind `rowwise_geography`, whose clone helper mints the twins. Every stage whose module or bound helpers import `microcosm.build.stochastic_assignment`, or import another module's function that draws through it, now binds it, so an edit to the keyed draws re-keys each stage they move. Every spine node re-keys; the coverage manifest's source digests and the H2 fixture are regenerated. diff --git a/changelog.d/931-uk-cut-runner-conditioned-supports.changed.md b/changelog.d/931-uk-cut-runner-conditioned-supports.changed.md new file mode 100644 index 0000000000..36b84a352d --- /dev/null +++ b/changelog.d/931-uk-cut-runner-conditioned-supports.changed.md @@ -0,0 +1 @@ +`tools/run_uk_cut.py` hands the spine the three conditioned atomic-area supports (#931). The spine requires `--atomic-support-ew`, `--atomic-support-scotland` and `--atomic-support-ni` whenever `atomic_area_assign` runs, which the packaged roster always schedules, so the runner's spine step was refused. `tools/uk_cut_inputs.json` now declares the supports as one input whose files are pinned to `sources.yaml`'s conditioned atomic-area support register (`members_pin: conditioned_atomic_supports`), the register the spine holds them to, and `spine_argv` passes each file under its flag. No hosting location is named yet: until one is, `fetch` refuses the input with its note unless the three files are already in `inputs/atomic_area_supports/`, where they are verified against their pins like every other input. diff --git a/changelog.d/931-uk-engine-blocks-pool-shares.changed.md b/changelog.d/931-uk-engine-blocks-pool-shares.changed.md new file mode 100644 index 0000000000..2912b36646 --- /dev/null +++ b/changelog.d/931-uk-engine-blocks-pool-shares.changed.md @@ -0,0 +1,6 @@ +Per-clone engine blocks now stand for the pool when the pool's clones differ (#931). In every simulation a dense measure block builds (`--engine-blocks K`), policyengine-uk's `shareholding`, `corporate_land_value` and `consumption_shareholding` are set, at each period the block computes, to the block's own value times `c_b = f_b·S_b/S`: its pool-mass factor times its allocation-key total over the pool's. The block first checks that the engine's own key total is `f_b·S_b` and that the engine carries each key and the weights by one factor per year. A guard then records every formula each simulation runs and judges policyengine-uk's whole-population formulas against a declaration: the share formulas corrected, `months_since_last_birthday` and `attends_private_school` stored, the engine's deciles and relative-poverty flags refused, and the rest zero under the parameters checked. Nothing a block or batch computes may fall in the Universal Credit rebalancing years (2026 on), where policyengine-uk draws new-claimant status by benefit-unit position. + +- **What exact means.** `engine_population_representation.exact` now holds only when every block applied its correction, passed its checks and got a clean verdict, and then means equal to one engine over the pool to float32 rounding, not bitwise. `exact_basis` (`pool_share_injection_with_population_guard`) names this basis, and `exact_scope`, which the release's `engine_measurement` limitation copies, is rewritten. A receipt without `exact_basis` predates the change, when exact meant identical copies. The measure step records a violation, which leaves the run not releasable. The release verdict, the local pre-flight, the dense assembler and the data contract read the same fields as before; none requires `exact_basis`. +- **Spine batches.** A spine engine call whose frame carries `household_clone_index` with more than one clone runs one simulation per clone, each reading the pool's shares (`c_b = S_b/S`) and judged under a pass that raises. A frame without the column, or with one clone, takes one simulation as before. Stage 52 (`private_school_attendance`) restates policyengine-uk's attendance over the whole frame from household-local engine values, with microdf's `MicroSeries.percentile_rank` and the engine's own parameters. Over one simulation it stores the engine's flags after checking the restatement bit for bit; clone by clone it stores the restatement. Its evidence gains `rank_route` and `engine_parity_checked`. +- **The shared pieces.** The country-agnostic machinery (pool shares, the checks, injection, the formula-run recorder, judging, `EngineBlock` and batches) is `microcosm.frame.adapters.engine_population`, and the UK declaration is `microcosm.frame.adapters.policyengine_uk_population`. Each block and batch first checks the declaration against the installed policyengine-uk; a single block and an unbatched spine call never read it. A re-pin that changes the whole-population list or a pinned formula source therefore leaves every dense block uncorrected and the run not releasable, and refuses every batched spine call, until the declaration is reviewed again. +- **What moves.** A spine built at the default `--n-clones` is one clone (`geographic_expand` writes `household_clone_index`, 0 throughout), so every spine call takes one simulation and no spine data moves (frame, H5, stored flags), but stage 52's evidence gains its two fields on both lines, so its stage-evidence digest moves. Identical copies take `c_b` within float64 rounding of 1, so the correction leaves their measures bitwise as they were, and a single block, the national line's, gains none of the new receipt fields. Today's dense receipts gain `exact_basis`, `share_correction`, `population_reach` and `annual_income_projection_factors`; `block_sensitivity` now names engine variables, so `present_in_this_run` can list them, drops the SLC entry and gains `corrected_formulas`; and each block's resolver receipt gains `engine_pool_shares` and `engine_population_reach`. Every spine stage and the full and national target kernels re-key, and the engine-calling stages also bind the two new modules. The annual-income factor, the one data change, has its own fragment. diff --git a/changelog.d/931-uk-fits-fold-geographic-clones.added.md b/changelog.d/931-uk-fits-fold-geographic-clones.added.md index 4cc44ba98e..8a46418da7 100644 --- a/changelog.d/931-uk-fits-fold-geographic-clones.added.md +++ b/changelog.d/931-uk-fits-fold-geographic-clones.added.md @@ -1,16 +1,16 @@ The UK fits trained on the frame's FRS rows now fold geographic clones into each survey record before they fit (#931). This covers the SPI stage-2 and reference forests, the housing shell, the UC reporter redraw, the UC capital donor cells, the salary-sacrifice forest and the SPI employment shell. -- **How the fold works.** Each fit passes what it reads: its predictors and targets, or the columns they derive from. A record's clones that agree on it keep the record's first training row, with the weight of those clones summed exactly. Clones that differ, through a draw keyed on their own ids or a stage run per area, stay rows of their own, each variant with its own clones' summed weight. SPI, high-value and CGT copies stay rows of their own, as now. -- **No change today.** Today's spine has no geographic clones, so every fit's rows and weights stay as they were, value for value. +- **How the fold works.** Each fit passes what it reads: its predictors and targets, or the columns they derive from. A record's clones that agree on it keep one training row, the lowest-id one among them (clone 0 whenever clone 0 is selected and agrees, whatever the row order), with the weight of those clones summed exactly. Clones that differ, through a draw keyed on their own ids or a stage run per area, stay rows of their own, each variant with its own clones' summed weight. On the clone-first spine each clone carries its own SPI transport key (`person_source_id`), so each variant trains under its own, and the SPI income stage refuses a frame whose differing clones share one. SPI, high-value and CGT copies stay rows of their own, as now. +- **No change today.** The default spine is one geographic clone (`geographic_expand` runs at K = 1), so every fit's rows and weights stay as they were, value for value. - **What changes once the spine clones before enrichment:** - the fits see each record once wherever its clones agree, and every clone's value at its share wherever they differ; - the reference transport meets no duplicate twins from agreeing clones; - - the UC capital cells' donor minimum counts records, not identical clones. + - the UC capital cells' donor minimum counts survey records, so neither identical nor differing clones grow it. - **The country-neutral pieces.** `microcosm.build.source_lineage` gains three functions: - `copy_paths`: every household's ordered copy path from a declared lineage, never inferred from ids, weights or values; - `replica_keys`: a key per record of any entity, shared only by its replicas in the named branches, with a member's rank in its household telling members apart; - - `collapse_replicas`: the fold itself, which takes the fit's `inputs` and folds only replicas that agree on them. -- **The UK entry.** `household_lineage.collapse_geographic_clones` is the UK's entry and requires those inputs. `uk_household_clone_paths` now reads the clone path through `copy_paths` on `UK_COPY_LINEAGE`, with identical paths and draw keys; three identity-test messages name the shared reader's branches. + - `collapse_replicas`: the fold's training rows and weights (`fold_replicas` returns the whole fold), folding only replicas that agree on the fit's `inputs`. +- **The UK entry.** `household_lineage.fold_geographic_clones` is the UK's entry and requires those inputs; `collapse_geographic_clones` returns its rows and weights. `uk_household_clone_paths` now reads the clone path through `copy_paths` on `UK_COPY_LINEAGE`, with identical paths and draw keys; three identity-test messages name the shared reader's branches. - **Declarations and tests.** - The collapsing stages declare the lineage live where they run, and a test refuses one whose surface would hide it. - Replicated-fixture tests hold each fit to the rows and weights it fits without clones. diff --git a/changelog.d/931-uk-floors-count-true-sources.changed.md b/changelog.d/931-uk-floors-count-true-sources.changed.md index 2617489db2..7381a68bdc 100644 --- a/changelog.d/931-uk-floors-count-true-sources.changed.md +++ b/changelog.d/931-uk-floors-count-true-sources.changed.md @@ -1 +1 @@ -Every UK floor now counts true source households, the survey household each copy and clone keeps in `source_household_id`, through one reader (#931). The reader is the new country-neutral `microcosm.build.source_lineage.record_sources`: a country declares its copy lineage once (the entity copies are made of, its source column and each copying step's marker and ordinal columns), and every record of any entity reads its source through household membership. `receipt_counts.record_source_households` is the UK's entry on `household_lineage.UK_COPY_LINEAGE`, and `receipt_counts.SOURCE_HOUSEHOLD_COLUMN` is the shared `source_household_id`, no longer `household_source_id`, the support channel's immediate parent. The reader refuses a frame without the source column, and a null source; a stage that runs before the SPI support channel says so (`before_copies`), and its households are then their own sources unless the frame already carries copy lineage. The take-up signal receipts, the apprentice receipt and the annual income receipts (stages 10 to 12 and 15), the SPI pay floor and pension coherence (which read `household_source_id` without the null refusal), company cars, the SPI benefit coherence, UC take-up's pools, the legacy closure, salary sacrifice's receipt and bonus donor floor, Carer's Allowance, the trading allowance, the support floor, the single-year age report, the dataset-size evaluation, the sampled-rung private-rent region floor, and the council-tax, BRMA-distribution, weighted-integrity, benefit-cap and UC-relationship tools count through it. The two columns agree on every household row of today's spine and pool, so no spine output moves. The single-year age report and the dataset-size evaluation now count a zero-weight row as carrying nothing and a household's copies once, so their support counts can fall; the weighted-integrity tool's carriers are now distinct sources, not rows. The five finite stages that declared `household_source_id` declare `source_household_id`, UC take-up's pooling text and the Carer's Allowance and support-floor notes name it, and a static test refuses any distinct count of `household_id`, `person_household_id` or `household_source_id` in the UK runtime and tools outside two named structural checks. The receipts' counting modules (`receipt_counts`, `household_lineage`, `source_lineage`) are bound into the implementation hash of every stage whose module or helpers import one of them; eight stages that counted through `receipt_counts` did not bind it, so an edit to it alone left their cache keys in place. +Every UK floor now counts true source households, the survey household each copy and clone keeps in `source_household_id`, through one reader (#931). The reader is the new country-neutral `microcosm.build.source_lineage.record_sources`: a country declares its copy lineage once (the entity copies are made of, its source column and each copying step's marker and ordinal columns), and every record of any entity reads its source through household membership. `receipt_counts.record_source_households` is the UK's entry on `household_lineage.UK_COPY_LINEAGE`, and `receipt_counts.SOURCE_HOUSEHOLD_COLUMN` is the shared `source_household_id`, no longer `household_source_id`, the support channel's immediate parent. The reader refuses a frame without the source column, and a null source; a stage that can run before the first copying step says so (`before_copies`), and on a frame without the source column or copy lineage its households are their own sources; on the clone-first spine that step is `geographic_expand`, which writes the column, so a stage after it that says so reads it. The take-up signal receipts, the apprentice receipt and the annual income receipts (stages 10 to 12 and 15), the SPI pay floor and pension coherence (which read `household_source_id` without the null refusal), company cars, the SPI benefit coherence, UC take-up's pools, the legacy closure, salary sacrifice's receipt and bonus donor floor, Carer's Allowance, the trading allowance, the support floor, the single-year age report, the dataset-size evaluation, the sampled-rung private-rent region floor, and the council-tax, BRMA-distribution, weighted-integrity, benefit-cap and UC-relationship tools count through it. The two columns agree on every household row of today's spine and pool, so no spine output moves. The single-year age report and the dataset-size evaluation now count a zero-weight row as carrying nothing and a household's copies once, so their support counts can fall; the weighted-integrity tool's carriers are now distinct sources, not rows. The five finite stages that declared `household_source_id` declare `source_household_id`, UC take-up's pooling text and the Carer's Allowance and support-floor notes name it, and a static test refuses any distinct count of `household_id`, `person_household_id` or `household_source_id` in the UK runtime and tools outside two named structural checks. The receipts' counting modules (`receipt_counts`, `household_lineage`, `source_lineage`) are bound into the implementation hash of every stage whose module or helpers import one of them; eight stages that counted through `receipt_counts` did not bind it, so an edit to it alone left their cache keys in place. Main's carer-link share (microcosm#1095 A3), which the SPI channel's refresh in `hmrc_spi_income_spine` reads after `geographic_expand`, counts its base the same way: `frs_disability.carer_link_share` takes `support_keys`, and the refresh passes each FRS-channel person's true source, so the share's floor and support band count a survey household's clones once. Without it the share counted clone households, K times its sources; at one clone the two counts agree, so no spine output moves. diff --git a/changelog.d/931-uk-helper-bindings.changed.md b/changelog.d/931-uk-helper-bindings.changed.md new file mode 100644 index 0000000000..f67ae05374 --- /dev/null +++ b/changelog.d/931-uk-helper-bindings.changed.md @@ -0,0 +1,76 @@ +Every UK spine stage now hashes each first-party module its transform computes through, and each stage that calls the engine also each one the injected engine computes through for it. Every other UK graph kernel hashes each one its own code computes through, the target and problem kernels among them, and each phase of a gate battery each one its own gates compute through (#931). `source_hash` hashes only the files it is given, so a node that called a function in a module its hash did not name replayed a stale cached output when that module changed. + +- **The rule.** `test_uk_graph` derives what each stage's hash must name. It starts at the transform `graph_kernels` builds for the stage, and for a stage that calls the engine also at the engine the builds inject (see The engine), and follows every definition the code names into the module that defines it, to a fixed point. Annotations count, and so do re-exports, module attributes and `type` aliases, which read as assignments of their values. So do the names `microcosm.graph` exports lazily through its module `__getattr__`: the walk resolves each to the runtime module that defines it, from the table that `__getattr__` reads, and a test checks that every first-party module with a module `__getattr__` is listed. It then checks that the stage's hash covers every module it meets. Every definition is a value except for these: + - eight listed refusal-only checks, each with its reason, which a test checks write nothing and return nothing but None or their sole argument; two are kernel admission reads (see Kernels), one is the gate kernels' stale phase-code check (see Gate batteries), and one is the target compilation's household-type check (see Target and problem kernels); + - the stage's own declaration (`country_spec`, `source_manifest` and the spec engine), whose content the node key already hashes as the stage contract; + - the listed core modules every node runs through (see Scope); + - a single string, a function returning one column name, and a record class whose field types are of these kinds. Their module must be hashed by the stage or by a node upstream of it, whose key the stage's key chains on. Otherwise they are listed with a reason; two such names are listed. + + A collection of strings is a value, like a number, because it selects rows, columns or codes. A second test applies it to the UK's other graph kernels (see Kernels). Two more cover the transforms the production spine build installs itself; a transform installed inside the source adapter or the rung sampler must come from its stage's module or be a listed adapter, since neither hands the node what the wrapped transform declares. Control tests pin each way of naming a helper that the rule follows, a lazy export among them, each kind it skips, the core-module stop, each way a refusal-only check could write, and how the engine's second root picks and follows the methods a stage calls. +- **Scope.** The rule follows every first-party package: `microcosm-build`, `-calibrate`, `-data`, `-diagnostics`, `-fit`, `-frame` and `-graph`, all under the `microcosm.` namespace. What the stages compute through outside `microcosm.build` is now bound as it is inside it: 72 bindings in 48 stages, found by the walks from their transforms, and 16 more in the engine stages, found by the walk from their engine (see The engine): + - `microcosm.fit.qrf` and `fit.model` in 11 stages: the nine that fit through the regime-gated QRF, and `spi_employment_shell` and `aps_occupation`, whose classifier resolves its fit weights through `fit.model` and whose held-out scores and folds use the QRF's; + - `microcosm.frame.materialize`, whose `engine_tables` copies a frame's tables, in the three stages of `spi_spine` and the band donors, and in 16 of the 17 engine stages, whose engine builds each simulation's dataset through it (main's `uc_limited_capability` binds it later: see Main's newer code); + - `microcosm.frame.rules` in `uc_take_up` and `pension_credit_take_up`, which type their engine by the adapter's `ProjectingRulesEngine`, a protocol that extends its `RulesEngine`; + - the engine adapter in every spine stage, beside `national_frame`, which stamps each frame it builds with the adapter's ISA income contract and savings income projection; 39 stages that call no engine gain it; + - `microcosm.graph.randomness` and its canonical encoder `microcosm.graph.canonical` in `regional_home_values`, which draws its keyed uniforms through them; + - `microcosm.calibrate.geography_constants`, the UK region tier, in `hmrc_cgt_gains_spine`, `isa_subscriptions` and `spi_housing_shell`. + + Listed instead, each with a reason a test checks: + - the core modules every node runs through, where the rule stops, since binding one would re-key every node on any edit to it. They are the Frame's three modules (`microcosm.frame.bundle`, `schema` and `weights`), which a stage's transform meets all three or none of; the graph executor's (`microcosm.graph.executor` and `population`); and the modules that compute node keys (`keys`, `decl` and `kernel`), which no stage's transform meets. No spine key hashes them. H2 cannot catch an edit to the Frame, since the graph's spine and the legacy stage plan both run through it, and H1, whose fixtures run four wrapped kernels through `run_graph` against stored bytes, catches one only on those kernels' frames; an edit that changes a UK value is otherwise left to the Frame's own tests, and a test pins that `Weights.total`, which the UK exact-total corrections target, is NumPy's sum of the weights. One to the executor's modules that changes a value on H2's fixture fails H2, which compares the graph's spine with the legacy stage plan's. One that changes what `node_key` or `source_hash` returns changes every key. A test checks that the walk from `run_graph` meets each of them. `microcosm.graph.canonical` also encodes every key, but `regional_home_values` draws through it, so it is bound there instead. (Later in #931 that stage retires, and the WAS stage, whose area home values draw through it, binds it: `931-spine-localises-home-values`.) That stage draws through `microcosm.graph.randomness`, whose contract asks for `SeedSource.KEYED`. Every spine stage kernel, that stage's included, keeps `SeedSource.NONE`, as María ruled on 9 Nov: its stream is a stage parameter, which its contract hash covers, and it reads no generator the executor seeds; + - `microcosm.frame.rules`' `assert_rules_engine_country`, the fourth refusal-only check, which every stage that takes an engine calls first; + - `SOURCE_HOUSEHOLD_ID_COLUMN` from `microcosm.frame.concepts`, CopyLineage's default source column, the second listed name. `geographic_expand` writes the column through the constant and declares it by its literal name, and its transform refuses other outputs, so a renamed constant stops the spine build before any node runs. Every stage after it that reads the constant is given the column, and one that counts before the copies the clone index too; `frs_annual_income_spells`, before it, is given neither and counts each household as its own source (`931-uk-spine-geographic-expand`). + + Still outside the rule: calls through any other object whose class the code never names, a name looked up by a computed string, and data files read by path. +- **The engine.** The 17 stages that call the engine compute through the `PolicyEngineUKEngine` the builds inject, whose class no stage's transform names. As María ruled on 9 Nov, a second walk is rooted at that class, the one `graph_kernels`' hermetic H2 builder and the spine build both construct (checked). It follows the methods a stage's code names on the engine, by attribute on any receiver or through `getattr`, with the constructor and every method they call on `self`, and then the methods its own reach names on any receiver outside the engine's classes, as a helper the engine hands itself to might call, to a fixed point, at the same scope and with the same tables. The stage rule's tests hold what it reaches as part of the stage's walk. It matches methods by name, as the engine-call scan does, so it can only over-count. The engine stages call `materialize`, `variable_metadata` and, through `getattr` in `private_school_attendance`, `parameters` and `simulation_batches`. The walk reaches the three hand-bound engine modules and one more, `microcosm.frame.materialize`, through which the adapter builds each simulation's dataset. That module is the new `_ENGINE_COMPUTE_MODULES` group in `graph_kernels`, which every engine stage binds (main's `uc_limited_capability` once it is bound: see Main's newer code) and which a test derives exactly. It sits beside `_ENGINE_MODULES`, which no other stage binds, because the SPI spine's three stages and the band donors bind `frame.materialize` for their own tables; any other stage binds it only where its own transform computes through it (checked). The engine reaches the concept vocabulary (`policyengine_uk_concepts`, `concept_mapping` and `concepts`) and `frame.rules` only through methods no engine stage calls (`concept_mapping` and `export_contract`), so none is bound for the engine. No spine stage hashes the concept vocabulary, and only `uc_take_up` and `pension_credit_take_up` hash `frame.rules`, for the protocol they type their engine by. The hermetic fixture's transform resolver no longer names the engine as held. +- **What is bound.** `national_frame` joins the modules every spine stage hashes. The stage helper table gains 231 bindings of 54 modules in 49 stages. Among them: + - in the CGT and high-value stages: `property_localisation`, `cgt_structure`, `spi_support`'s exact-total helpers, `cgt_imputation`, `cgt_asset_type` and `advani_summers`; + - `employer_pension_contributions` in `frs_hmrc_spine_leaves` and `hmrc_spi_income_spine`; + - the tables the narrower rule finds: `tenure_constants` in `was_wealth`, `was_lisa`, `high_value_home_support` and `regional_home_values`; `council_tax_constants` in `frs_spine`; `rowwise_geography` in `lcfs_consumption`; `etb_vat` in `etb_services`; `cgt_imputation` in `cgt_incidence_clone`; `frs_annual_income` and `hmrc_source_contract` in `hmrc_spi_income_spine`; + - `household_income` in `was_wealth` and `was_lisa`, for its predictor names. + + Two groups recur: the vendored-fact reader with the chronicle feed and its schema epochs (20 stages), and `frs_spine`'s pinned tab reader with `trace` (14 stages). The older import tests then ask for more, because the new helpers import these modules: + - `stochastic_assignment` is bound into 8 more stages, and the SPI support channel's exemption goes; + - the copy-id table is bound into `frs_hmrc_spine_leaves` and `hmrc_spi_income_spine`, and 15 stages join the copy-id test's name-only list; + - `frs_spine`, `spi_support_channel`, `spi_income_band_donors`, `uc_deduction_attributes` and `etb_services` join the counting stages. + + The dense nodes' shared hash adds `rowwise_geography`, `frame_sampling`, `national_frame`, `local_authority_input`, `gates` and `spi_support`, whose replacement strata the sample reads. +- **The sampled spine root.** The spine build wraps the root's transform in a rung sampler, which samples the root below f100. That sampler lived in `spine_build`, which no node hashes, and sampled through `frame_sampling`. It now lives in `national_sampling` as `UKSampledSpineRootTransform`, with `sample_uk_spine_root`. Through `graph_implementation_dependencies`, which the root node hashes, it declares `national_sampling` at every rung, since it resets the root's indexes at f100 too, and `frame_sampling` only below f100, where it samples. Its behaviour is unchanged. +- **Kernels.** A second test applies the rule to every kernel the UK registries register: the spine's create, identity, claim, stage and expand kernels; the dense population and shared atomic geography kernels; the full build's target compilation, selection, measures and problem and the national targets and problem (see Target and problem kernels); the calibration and size kernels; the bound spine; the export, package, inspection and certification kernels; the rotated holdout; the national gates and readback; and, phase by phase, the spine gates and the full build's preflight and calibrated gates (see Gate batteries). It starts at what a kernel runs: its methods in method resolution order, without the implementation hash, which computes the key, or a method only that hash calls, such as the full gates' `_country_resources`. It checks the result against what that hash really hashes, recorded at every `source_hash` call the hash makes. Beside the stage rule's tables it stops at four listed kernel stops, each with a reason a test checks: + - the hermetic fixture's transform resolver, since the stage rule holds each stage's transform and, through its second root, the engine the resolver's builder injects into the engine stages (see The engine); + - `_minimal_frame`'s rebuild of the SPI support allocation's mass record, which only `hmrc_spi_income_spine` takes, and whose reach that stage's node hashes. The spine gates rebuild their frame through `_minimal_frame` too, and no spine gate node names a stage (checked); + - `UK_GATE_REGISTRY`, which the bound spine reaches only for its checkpoint's gate digests. The gate batteries and the national gates evaluate through it, and the rule holds them past it (see Gate batteries); + - `microcosm.graph.codecs`' `load_source`, which the walk now sees through `microcosm.graph`'s lazy exports. The create kernel calls it only where no root transform is bound, the hermetic registry's data-only path, which loads the H2 fixture's captured root tables. The spine build installs the root's transform and H2 parity binds it too, so neither takes that path (checked). Binding the codecs, and the content store they name, would re-key the spine root, and with it every spine node and every node downstream of the spine, on edits to code no build runs. As María ruled on 9 Nov, it is a stop instead, and a test checks that no spine kernel and no kernel of the national line hashes the codecs. What the codecs decode is held by `microcosm-graph`'s codec tests. + + The dense solve's `_admit` and the new `graph_terminal._require_full_gate_admission` are listed as refusal-only admission reads. The export description and the rotated holdout now call the latter in place of their inline refusals, with the same messages. A test checks that the kernels enumerated are every kernel class the UK runtime and the shared atomic geography define. Another holds the executor's and key modules' definitions that the kernels' own code names to a listed table, each with a reason: the dtype tokens' map, which the executor casts every node's owned cells through anyway, two artifact declaration records, the capabilities record and its five enums, and the kernel context and result records. The kernels name six of them through `microcosm.graph`'s lazy exports, which the walk now follows: the capabilities record, three of its enums, and the context and result records. A control test pins the walk's parts. The dense population kernels' own test folds into this one, with its pins. What it binds, by kernel: + - the seven dense population kernels' shared hash: the engine adapter, the canonical encoder and the weight-update receipt; + - the geography identity: `household_lineage`, `source_lineage`, `microcosm.frame.concepts` and the canonical encoder; + - property value: `source_lineage`, `frame.concepts`, `rowwise_dataset`, `tenure_constants`, the keyed uniforms and the canonical encoder; + - property band: `tenure_constants`; BRMA and RFDR: `receipt_counts`; + - private rent: `source_lineage`, `frame.concepts`, `rowwise_dataset`, `national_frame`, `receipt_counts`, the keyed uniforms and the canonical encoder; + - the spine's identity and claim kernels: `national_frame`, and the claim kernel `graph_frames` too, through which it casts the cells it claims (see the bound spine); + - the four atomic geography kernels' shared hash: the content store, which `load_source_bytes` takes, and `frame.materialize`; + - the seven calibration and size kernels' shared hash: `national_frame`, `calibrate.hierarchy`, `calibrate.monetary_binding` and the canonical encoder. The size search and refit hash `target_weights`, `local_doctrine`, `local_targets`, `national_doctrine` and `calibrate.geography_constants` too, in their own implementation hashes, since the dense solve and the other kernels sharing that hash do not compute through them; + - the bound spine: `frame.materialize`, the engine adapter, `content_identity`, `hmrc_uprating`, `gate_battery`, `gates`, `trace`, `gate_scopes`, `graph_frames` and the canonical encoder. The create-frame normalisation it shares with the spine's root, and the owned-cell cast the spine's kernels share, moved unchanged from `graph_kernels` into the new `graph_frames`, which the spine's create, claim and stage kernels hash, so an edit to the spine's kernel module no longer re-keys the national line; + - the export description: `gates`, `atomic_area_support`, `graph_population`, `hmrc_uprating`, `local_authority_input`, `rowwise_dataset`, `rowwise_geography`, `frame.materialize` and the canonical encoder. The inspection export binds the same modules, and `geography_ladder`, `national_frame` and `weight_year` too; + - the export readback: `artifact_files`, `trace`, `geography_ladder`, the engine adapter, `frame.materialize` and the canonical encoder; the package inventory: `artifact_files`, `trace` and the canonical encoder; + - the certification: `artifact_files`, `trace`, `gate_battery`, `gates`, `graph_evidence`, `gate_scopes` (the spine gate manifest's scope) and the canonical encoder; + - the rotated holdout: the 35 modules that its problem reconstruction, folds and rowwise solves compute through. They are the local surface's modules, `holdout`, `local_target_census`, `size_checkpoint`, `solve_progress`, `weighted_integrity`, microcosm.calibrate's solver modules, `gate_battery`, `stage_evidence`, the engine adapter and the canonical encoder; + - the national gates: `gates`, microcosm.calibrate's `artifacts`, `matrix`, `registry`, `target`, `hierarchy`, `monetary_binding` and `solve`, `graph_targets`, `graph_population`, `national_frame`, `etb_services`, `cgt_projection`, `donor_uprating`, `gate_scopes` and the canonical encoder, and everything the gate registry's bindings compute through (see Gate batteries); + - the national readback: `artifact_files`, `trace`, `frame.materialize`, the engine adapter, `geography_ladder`, `graph_population`, `hmrc_uprating` and the canonical encoder. + + The LAD23 names' pin stays listed, now for every kernel that reads it. The geographic clone index columns' name (`geographic_clones.ladder_clone_index_column`) is listed for the calibrated gates, whose support-family fold reads it on the dense frame: `geographic_expand` writes those columns on the spine and the dense expand only on a spine without them, and both hash `geographic_clones`, as do the national gates, whose fold reads them on the national frame (`931-uk-spine-geographic-expand`). It is listed for the dense measures and the national problem too (see Target and problem kernels). `test_uk_target_compile_modules`'s hash inputs gain the seven modules that `graph_national` now imports only inside implementation hashes: the national gates' and readback's, and for three of them the national problem's. +- **Gate batteries.** As María ruled on 9 Nov, each phase of a gate battery keys on what it computes through, and on nothing only another phase computes through. A battery's phases run one kernel's code, and that kernel's hash is the code they share. What only one phase computes through is that phase's own code, which its node carries as `_code`, hashed as `energy_pricing_code` and `terminal_code` were. A test derives each phase's reach. It follows the kernel's code, taking the branch of each `if phase == ...` the phase takes, and the binding of each gate the manifest declares for the phase. Into the stage-health checks it follows only those the phase declares, by the `check` each entry names, and into the spine gates' evidence-artifact table only the entries the phase's gates read, so the CGT asset type domain is the transferred phase's alone. It then checks that the shared hash holds what every phase computes through and nothing only some do, and that each phase's own code holds exactly the rest. Where a phase's gates read the engine their kernel hands them, the walk also follows the methods they call on it, from the engine's class, as for an engine stage (see The engine): the spine gates' rules engine is the spine build's `PolicyEngineUKEngine`, and the full gates' engine is the driver's coverage engine. Another test checks each part of that model against the code, the engines among them. So the assembled gate's own key, which the spine stages after it chain on, carries nothing only the transferred phase computes through, and the preflight's, which the dense solve chains on, nothing only the terminal phase does. Key chaining still re-keys them wherever a node upstream hashes the edited module. Of the 17 modules only the transferred phase computes through, stages at or before `frs_brma`, the assembled phase's last, hash six: `stochastic_assignment` and `take_up_contract`, and, since `frs_hmrc_spine_leaves` and `frs_annual_income_spells` run before it (`931-uk-spine-geographic-expand`), `frs_annual_income`, the vendored-fact reader, the chronicle feed and its schema epochs. So an edit to any of the six re-keys the assembled gate; no stage before it hashes the other 11. Of the 38 only the terminal phase computes through, most re-solve too, through the spine stages, the target and problem kernels, the population kernels or the dense solve, which hash them as well. Eight never re-solve, on any line: `high_value_home_gates`, `microcosm.diagnostics`' catalogs and schema, `calibrate.hierarchy_labels`, the target loss attribution, `input_mass`, `cgt_projection` and `source_runtime`, so a disclosure fix between two builds that must reuse the solve still belongs in `high_value_home_gates`. `uk_runtime.diagnostics` re-solves only the national line, whose problem kernel hashes it. `calibration_run` and `release_certification` re-solve only on a bound spine, whose kernel hashes both, and seven modules that only the spine's nodes hash before the solve re-solve only in the in-graph build. The preflight's own code, the shared ledger's parity report, is also hashed by the full build's target compilation, selection, measures and problem, by the national line's targets, problem and gates and, in the in-graph build, by the `etb_services` and `lcfs_consumption` stages and the transferred gate, so an edit to it re-keys those too. + - The spine gates' hash binds the 12 modules both phases compute through. It gains `gates`, `stage_evidence`, `graph_kernels` (for `_minimal_frame`), `national_frame`, `receipt_counts`, `council_tax_constants`, `calibrate.geography_constants` and the engine adapter, through whose `PolicyEngineUKEngine.enum_domain` each phase's enum gates resolve a domain no artifact supplies, and `stochastic_assignment` moves out to the transferred phase. The assembled node carries `assembled_code`, `frs_relationships`, for the ONS household type domain its enum gate checks. The transferred node's `energy_pricing_code` becomes `transferred_code`, 17 modules: the energy level; the bus, consumption, excise and gambling checks' facts and pricing; the annual income, apprentice and latent-attribute checks' modules; and the CGT asset type domain. The spine gate kernel refuses a stale or missing `_code`, as the full gates do (see The phase codes, below). + - The full gates' hash binds the 26 modules both phases compute through. It loses `local_rowwise`, `weighted_integrity` and `diagnostics`, which only the terminal phase computes through. It binds the spine and local gate scopes and the scoped gate manifest, which the preflight computes through, in a new `gate_scopes` module, not in `calibration_run`, the national calibration runner they lived in. They moved there unchanged, and `calibration_run` re-exports them, as microcosm#1492 moved the target kernels' two functions into `ledger_surface`, so in the in-graph build an edit elsewhere in the runner re-keys neither the preflight nor the solve. The bound spine, the national gates and the certification, whose code reaches them too, hash `gate_scopes` as well. The calibrated gates' `terminal_code` binds `calibration_run` itself, for the admin totals and CGT projection artifact their gate context is built with. The preflight's release input coverage gate computes through `stage_evidence`, `parity_reference`, `release_identity` and `release_input_coverage`, which its key used to miss, so on a bound spine an edit to any of them now re-keys the preflight and re-solves; in the in-graph build so does one to `parity_reference` or `release_identity`, while the other two already re-solved through the spine stages that hash them. The preflight node carries a new `preflight_code`: the shared ledger's registry parity, behind the ledger compile parity gates only the preflight declares. The calibrated node's `terminal_code` now hashes 38 modules. `high_value_homes` and `calibrate.geography_constants` move into the shared hash, since the preflight computes through them too; the target kernels hash both, so an edit to either already re-solved. The kernel refuses a stale or missing `preflight_code`, as it refuses a stale `terminal_code`. + - The phase codes. As María ruled on 9 Nov (late), the four functions that compute them and their two tables live in a small module of their own, `uk_runtime.gate_phase_codes`, unchanged in what they hash, with one stale-code check, `require_current_phase_code`, which both batteries' kernels call before they read anything of their node but its parameters. In `graph_evidence` and `graph_terminal`, which define the gate kernels and which every spine stage kernel, the dense population kernels and the export kernels hash too, an edit to a phase's list of modules, which the battery test demands whenever a phase's evaluators gain one, re-keyed every node whose kernel hashes either file, and by key chaining every node after one: for a spine phase's list every node of the spine and of the in-graph build and all but the checkpoint of the local line on a bound spine, for a full phase's list the preflight, the dense solve and every node after them. Now only the graph construction, which sets each node's `_code`, and the check read the module, and no kernel hashes it: the check is listed refusal-only, since it reads its parameters and the phase's code and refuses, writing nothing, and a phase's own code reaches its key through the parameter, which `node_key` hashes. So an edit to a phase's list moves only that phase's parameter, which re-keys that phase's node and what follows it. `source_hash` leaves the kernels' listed core names: only the full gates' run named it, through the phase-code table, which only the check reads now. The spine gates' run now refuses a stale or missing `_code` too, a new refusal, so no gate evaluates its phase under a key that names other phase code. A record stored under a gate node's key is still reused without running its kernel, so the check does not guard a cache hit. Tests check that no UK kernel's key covers the module; that both gate kernels call the check first, on their own battery's table, with no return before it and their phase bound once, before it; that each spine phase refuses a stale and a missing code and passes its node's own; and that the check refuses a code on a phase its battery's table leaves out, and a battery with no table. + - The national gates evaluate whichever gates their node's scope names, so their hash binds everything the gate registry's bindings compute through. That is 50 modules, which `battery_bindings.UK_GATE_REGISTRY_MODULES` lists and a test holds exactly to the registry's derived reach. Main's newer gate code moves that reach by five modules, and the list follows it (see Main's newer code). Their stop at the registry, the one held kernel stop, goes. +- **Target and problem kernels.** As María ruled on 9 Nov, the derived rule replaces microcosm#1497's per-kernel import record for the full build's target compilation, selection, measures and problem and the national targets and problem, which that record held until now. No kernel stays held. + - `test_uk_target_compile_modules` loses the record (`UNCLASSIFIED_IMPORTS`), its kernel groups and the check that read them. It keeps the shared list's closure check, the tests of what the kernels hash and its hash inputs, the seven above among them. A new check covers what the derived rule does not: each hash input is still imported where listed, and a kernel hashes one only where another module it hashes imports it too. So no kernel hashes the compile list or the solve fingerprint's module, and an edit to either re-keys none. + - What all four full kernels compute through goes in the hash they share, and what only one computes through goes in its own implementation hash, as for the size search and refit. The shared hash gains the canonical encoder. The compilation adds `chronicle_epoch`, whose schema versions and fact-key epochs `ledger_artifact` checks the facts against and records. The measures add the 15 modules that resolving the measures and compiling their national problem compute through: `graph_population` and `national_frame`, for the frame they rebuild; `atomic_area_support`, `frame.materialize` and the engine adapter, for the engine's scratch frame; `dfe_spring_term`, `frs_annual_income`, `hmrc_income`, `income_tax_rate_classes`, `lcfs_consumption` and `uc_target_measurements`, for the resolver's inputs and measures; `chronicle_epoch`, for the vendored facts' schema checks; `national_calibration`, for the measure inputs it injects; `local_rowwise`, for the national rows; and `calibrate.monetary_binding`, for the constraint matrix's check. The problem adds `graph_population`, `national_frame`, `local_geography` (the local surface matrix's area columns) and `calibrate.monetary_binding`. + - The national targets' and problem's shared hash gains `chronicle_epoch`, `graph_targets` (the register codec) and the canonical encoder. The national targets compute through nothing else, so they have no hash of their own. The national problem adds the measures' modules but `chronicle_epoch` and `national_calibration`, which its shared hash already binds, and adds `diagnostics` and `gate_battery`, whose target geography levels and JSON-safe values its payload records. + - The measures and the national problem also bind the annuity and trust column's name. The resolver reads that column where the frame carries it and reads zeros where it does not, so on the national line's bound spine a renamed constant would change the problem without re-keying it, and the name cannot be listed as unbound. As María ruled on 9 Nov (late), the name moved unchanged from `frs_spine` into a module of its own, `uk_runtime.annuity_trust_column`, which holds nothing else, and `frs_spine`, `measure_simulation`, `spi_spine` and `spi_income` each import it from there. So the two kernels bind that module, not `frs_spine`, whose every edit would otherwise re-key them. The `frs_spine` root, which writes the column, binds it too, and so does `hmrc_spi_income_spine`, which zeros it on the rows it draws, though that stage could chain on the root's key; a test holds every stage and kernel whose code reads the name to its module, and every module that reads the name to reading it from there, not through another module that imports it. The same test holds every key that covers `frs_spine` to the create kernel and the spine stages, none of which runs on the local or the national line on a bound spine. The clone index columns' name stays listed for them: the dense measures read it only to split their frame into the engine blocks they are asked for, and refuse a frame without it, and their population is the expand's; the national problem resolves its measures in one block, which never reads it. Later in #931 the dense line's population is the spine's version (`931-dense-on-the-clone-first-spine`), and both read the frame's clone count, through a refusal-only check that holds it to the K their node binds (`931-dense-k-pin`). A test checks these facts. + - `ledger_targets`' household-type check is listed refusal-only, as María ruled on 9 Nov (late). It compares the contract's `ons_household_type` bindings with the ONS household type domain and refuses a binding outside it or composition rows that do not cover it once each. It writes only into the dict and set it creates, which the shared refusal check counts as local. Its one caller, `compile_uk_target_registry`, calls it before the loop that compiles the references, on the compilation's and the national targets' paths alike. So neither kernel binds `frs_relationships`, which the rule asked of them only through this check. A test checks the check's place in its caller, that neither kernel computes through `frs_relationships` another way, and that the kernels that hash it for their own reasons keep it: the `frs_relationships` stage, whose transform it defines, and the assembled spine gate, whose own code holds the domain its enum gate checks. + - No kernel drops a module it hashed before, though each still hashes some modules its walk does not reach. The full kernels' shared hash still binds the whole compile list and the modules beside it in all four, so the selection's key covers 52 modules while its code computes through 6. + - `transport/compile_modules.py`'s reason for leaving the graph runtime off the transport compile list said that no kernel in the repository hashes a `microcosm.graph` module, which was untrue. It now says what the listed modules use those modules for, that the compile kernels hash none of them, and that other kernels hash the canonical encoder, each UK graph kernel whose code computes through it among them. No kernel hashes that file, so the edit re-keys nothing. +- **Main's newer code (microcosm#1095), under the same rule.** `uc_limited_capability`, one of the 18 stages that call the engine, binds the engine stage modules, `frs_release` and the keyed-draw helpers, and joins the copy-id test's name-only list for `frs_spine`'s carrier codes. `pip_points` and `frs_council_tax` bind the vendored-fact reader with the chronicle feed and its schema epochs. `frs_disability` and `hmrc_spi_income_spine` bind `carers_allowance_take_up`, through whose balanced positions the carer link places its outside carers, and the root binds `frs_disability`, whose reported benefit columns its LCWRA proxy reads. `frs_spine` now imports `receipt_counts`, and `frs_education`, `frs_employment`, `frs_legacy_proxies` and `frs_relationships` bind it for names that count nothing, so the counting test lists them as name-only and holds their walks to reach no counting module; `frs_employment` stays its control. The gate registry's whole reach gains `cut_reference`, `reference_surface`, `source_lineage` and `household_lineage` and loses `parity_reference`, so `UK_GATE_REGISTRY_MODULES` lists 56 modules. The full gates' shared hash gains `receipt_counts` and `reference_surface`, which both phases compute through, so an edit to either re-keys the preflight and the dense solve after it; the terminal phase's own code gains `source_lineage`, `household_lineage` and `cut_reference` and loses `receipt_counts`. The bound spine binds `microcosm.data.gate_signing_keys`, the registry its checkpoint's signed reports are held to, and the kernel table lists `SOURCE_HOUSEHOLD_ID_COLUMN` for the gates, with a reason adapted from the stage table's. +- **What moves.** Node keys only: no value, declaration or generated file moves. Every spine kernel hashes `graph_kernels.py`, so every spine node re-keys, and with them every node downstream. The root node also hashes the rung sampler's module at every rung, and the shared core below f100. Every kernel in `graph_population.py`, and the calibration and size kernels that hash it, re-key too. So do the kernels bound above; the full build's preflight and calibrated gates and the national targets and problem, whose own modules changed; the national line from its bound spine root; and, through the shared atomic geography kernels, the transport graphs' atomic geography nodes, the NZ transport graph's among them, and what they feed. The bound spine hashes `graph_frames`, not `graph_kernels`, so no node of the national line re-keys on an edit to `graph_kernels`. Of the calibration and size kernels, only the size search and refit hash the five modules named under Kernels. The national line runs neither, and none of the kernels it runs hashes `local_doctrine` or `target_weights`, so none of its nodes re-keys on an edit to either. Its targets and problem kernels hash the other three: the targets compute through `calibrate.geography_constants`, and the problem through `local_targets` and `national_doctrine`. Sixteen of the 17 engine stages also hash `microcosm.frame.materialize` (main's `uc_limited_capability` once it is bound: see Main's newer code), and an edit to `microcosm.graph.codecs` re-keys no spine node. The gate batteries' keys move once with their new shared hashes and phase codes: the spine gates, and every spine stage after the assembled gate, which waits on it; the preflight, the dense solve keyed on it and every node after it; the calibrated gates; and the national gates. `graph_evidence.py`, which every spine stage, expand and create kernel and the dense population kernels also hash, changed with them, so every spine node and every node downstream re-keys once. The target and problem kernels' keys move once with their new bindings. `graph_targets.py` and `graph_national.py`, which define them, changed with them, and the preflight and calibrated gates, the rotated holdout and the national gates and readback each hash one or both of those files too. So in the full build the target compilation, selection, measures and problem, the pool checkpoint (`uk.full.pool`), the preflight, the rotated holdout, and the dense solve and every node after it re-key, and on the national line every node after its bound spine root; no spine node and no transport node re-keys with them. The phase codes' move re-keys once more every node whose kernel hashes `graph_evidence.py`, `graph_terminal.py`, `gate_scopes.py` or `calibration_run.py`, and by key chaining every node after one: every node of every UK graph, the local and national lines on a bound spine from their root, whose kernel hashes the last two; of the phase codes only `terminal_code` moves, since it hashes `calibration_run`. The column's move and the listed household-type check re-key once more every node whose kernel hashes a file they edit (`frs_spine.py`, `graph_kernels.py`, `graph_targets.py`, `graph_national.py`, `measure_simulation.py`, `spi_spine.py` or `spi_income.py`), and by key chaining every node after one: every spine node and every node of the in-graph build, 26 of the local line's 33 nodes on a bound spine, and the national line's every node after its bound spine root. After them, on a bound spine an edit to `frs_spine.py` or `frs_relationships.py` re-keys no node of the local or the national line, where it re-keyed 12 and 4 of their nodes for `frs_spine.py` and 14 and 5 for `frs_relationships.py`, so neither line re-solves for one; in the in-graph build either still re-keys its stage and every node after it, as before. An edit to `annuity_trust_column.py` re-keys the spine root, the measures and the national problem, and what follows them. `uk_spine.json` and the H2 fixture hold stage contract hashes, not code hashes, and no fixture holds a gate node, so neither regenerates. diff --git a/changelog.d/931-uk-identity-tool-high-value-copies.fixed.md b/changelog.d/931-uk-identity-tool-high-value-copies.fixed.md new file mode 100644 index 0000000000..594d19defc --- /dev/null +++ b/changelog.d/931-uk-identity-tool-high-value-copies.fixed.md @@ -0,0 +1,6 @@ +The UK identity tool (`tools/verify_uk_identity_stability.py`) now scopes its receipts past the high-value home copies, and E5 checks the regional uprating on the values that stage left (#931). + +- **The copies.** The tool's stacking map gains `household_is_high_value_support_copy`, the flag of `high_value_home_support` (stage 26, microcosm#1240). The receipts that scope to an earlier stage now drop the copies and every CGT row made from them: E5's wealth block and E4's FRS-only frame. The copies' mass folds back by family. After the CGT folds, each copy's weight is added onto its root, so the root is back at its weight before the split. No uniform factor could do this, since only the top value stratum is divided. The fold decodes each copy at the copy-id table's `high_value_support` field. It refuses, by count, a copy without a root, a flag that disagrees with the stored index, an index that disagrees with the id, and a family whose decoded size disagrees with `high_value_support_copies`. Dropping the copies while a CGT layer stays in scope is refused. +- **E5's stored values.** `regional_home_values` (stage 27, microcosm#1112) rewrites `main_residence_value` and `property_wealth` after the uprating, so E5's fixed-point check of the uprating was comparing stage 27's values. The scoping now reverses that rewrite for every stage before stage 27. The value comes back from `main_residence_value_rank_basis`, which the stage keeps, and the wealth moves back by the same change. An artifact without the rank basis, such as a release, is read as before. The reversal refuses a roster in which a later stage declares either column, since it would then hide that stage's rewrite. (Later in #931 `regional_home_values` retires and the reversal with it: the WAS chain's area home values keep their draw beside the value they write, and E5 recomputes the main residence from that draw through them and the uprating, which divides by it: `931-spine-localises-home-values`.) +- **E8's band donors.** E8 reran the income band donor stage (stage 17) on the frame its CGT checks read, where only the CGT layers were folded away. The high-value copies stayed in the rerun as households the stage saw, and the stored seats, carriers and band weights read a donor's copies as donors of their own. Once the copy-id table gave each layer its own field, the rerun was refused outright: the donors it appended sorted below the copies' ids. The check now scopes the artifact as E5 does (`_frame_as_stage_saw`). Every row minted after stage 17, the high-value copies and every CGT row, is dropped with its people and benefit units, and its mass folds back onto the row it came from. So the check reads the frame stage 17 wrote, at the weights it left, and each donor once, and reruns the stage on that frame less its donors, at the weights the stage read. A row minted after stage 17 that the scope leaves in place is refused by count. +- **What moves.** Tool verdicts only. On the synthetic H2 oracle frame, E5's stored-column check failed with the tool before this change and passes now. On any spine built since `regional_home_values` joined the roster, the check no longer fails on that stage's rewrite. The `was_wealth` scope drops the 115 high-value copies (of 404 rows) it kept. On that frame every high-value root is an SPI row, which E4 drops anyway, so E4 reads the same frame as before, and the E6, E7 and E9 receipts are identical. On a spine whose high-value roots include FRS households, E4's scope and weights change too. E8's band-donor scope moves E8's receipt and no other: on any spine with high-value copies, the check reads each donor once, and its rerun, which needs the SPI tape, now completes where it raised before. No stage, value, node key or generated file moves, since nothing hashes the tool. diff --git a/changelog.d/931-uk-land-rows-diagnostic.changed.md b/changelog.d/931-uk-land-rows-diagnostic.changed.md new file mode 100644 index 0000000000..0a7f853120 --- /dev/null +++ b/changelog.d/931-uk-land-rows-diagnostic.changed.md @@ -0,0 +1,4 @@ +The UK calibration holds two of the three ONS national balance sheet land rows out of the solve as measured diagnostics (#931; María's ruling of 6 November 2026, workstream 2 of microcosm#1521). `ons.land.corporate_land_value` (E43B, non-financial corporations' land) reads policyengine-uk's `corporate_land_value`, which hands out the engine's own aggregate by each household's share of the weighted corporate-sector wealth: a released file meets that aggregate at any weights, and the solve, fitting the column measured at the starting weights, pinned only the weighted mass of the WAS corporate and private pension wealth, not land. `ons.land.land_value` (E44R, the total economy) also counts the land of financial corporations, government and NPISH, which neither the household row (E44N) nor the corporate row covers, while the engine's `land_value` is the sum of the two columns those rows read, so beside them it asked for £323,750m that no weights could supply. Both are held under reviewed exclusions to 2027-02-06 and stay on the run's compiled register. No build step reports a held row's estimate; a tool that scores the un-excluded register, such as `tools/evaluate_uk_incumbent_surface.py`, measures them, as it does every held row. `ons.land.household_land_value` stays bound, and its binding note now names the households sector, the one E44N covers, not households and NPISH; the two held rows' binding notes record the hold. + +- **What moves in the solve.** Two rows leave it on both lines. Both lines give each target family an equal share of the objective and split it evenly over the family's bound rows (on the dense line, within each grain, where the land rows are amounts), so `ons.land.household_land_value`, now the `ons_land` family's only bound row, carries the family's whole share, three times its loss weight before. No other row's weight changes, since the family keeps a row. The calibrated weights move slightly through both effects, which a calibrate-only rerun on the move's spine is to measure together. The released corporate land value still equals the engine's aggregate, and the released land value moves only through household land. +- **What else moves.** The national frozen scoring register (`national_target_registry.json`, the approved register) loses the two rows, so a national run given a frozen register derived before this change refuses, as with `obr.ni_employer`; the dense line checks its frozen register against the reconciled register, which keeps them. The incumbent and release-cut scores, `uk_target_fit` and `calibration_diagnostics.json` read only the rows the solve binds, so the two rows leave them. Neither line resolves `land_value` or `corporate_land_value` any more: the dense measure receipt's `national_inputs` falls by two, and a per-block dense run's `block_sensitivity.present_in_this_run` is empty. The dense release lists the two exclusions among its limitations (`measure_exclusion:`, with the first 300 characters of each reason). The edited files are UK country-spec resources, so the national and dense target, measure and solve nodes re-key; spine nodes, `source_stages.json` and the H2 spine fixture do not move. diff --git a/changelog.d/931-uk-spine-clone-count.added.md b/changelog.d/931-uk-spine-clone-count.added.md new file mode 100644 index 0000000000..795a6d7611 --- /dev/null +++ b/changelog.d/931-uk-spine-clone-count.added.md @@ -0,0 +1,19 @@ +The UK spine build takes its geographic clone count (#931). `tools/build_uk_frs_spine.py --n-clones K` builds the clone-first spine at K clones, K a whole number from 1 to 100, the `geographic_support` copy-id field's bound. Without the flag the spine builds one clone, as before. K goes to the `geographic_expand` node, to the transform that makes the clones and to the rules engine. Every engine call after `geographic_expand` runs one simulation per clone, and the engine refuses a frame that carries another count. The engine identity does not take K, so it is the same at every K. + +- **Guards.** A release candidate takes one clone until #931 group (c) pins the clone count a release builds at. (Later in #931 it also takes the dense release's 15, on a geography-first roster: `931-dense-k-pin`.) `--release-candidate` with any other K is refused when the arguments are parsed, and again when the full build marks a parsed spine request a release candidate. A `--synthetic-fixture-dir` whose `config.geographic_clones` differs from `--n-clones` is refused while the spine is prepared, before any stage runs. +- **Sidecar and Logbook.** The build sidecar records `geographic_clone_count`, read from the frame's `household_clone_index`; a frame without that column counts as one clone. Its rules engine record keeps the engine's provenance at every K and adds the adapter's batch record only when the final frame carries more than one clone, as before. A spine of more than one clone binds its count into the Logbook identity digest; a one-clone build's digest is unchanged. +- **Memory guard.** `--stage-profile-dir` writes, for each spine graph node and under its id, the node's entry, peak and exit RSS and its wall time to `stage_profile.json`. The profiled kernels key as the kernels they wrap, so node keys do not move and a profiled run reuses an unprofiled run's store. A record keeps the message of a failure its node raised, so the directory must stay private. Before any node runs, the build refuses a directory inside the repository, a path that is not a directory, and a `stage_profile.json` there that is not a stage profile of the current schema, which the profiler would refuse only after the first node had run. `.gitignore` ignores `stage_profile.json` and its lock file. +- **National line.** Every spine from #931 group (b) on carries `household_clone_index`, at one clone too, and the national line drops the clone indices from its export only from group (c) on. So the national problem, the first national node that reads the bound frame, refuses such a spine before it resolves anything, in one fixed sentence (María's ruling of 10 Nov, decision 4). Without it, a national build on a one-clone spine would solve in full and then fail its terminal degenerate-surface gate. (Later in #931 the national line builds on a geography-first spine of one clone, refuses a spine of any other clone count by its build sidecar, and refuses every build until the area-basis exclusion is signed for the national line: `931-national-one-clone-areas`.) +- **Identity tool** (`tools/verify_uk_identity_stability.py`). + - `--spine-sidecar` takes the spine's build sidecar. The tool refuses a frame whose clone count differs from the sidecar's `geographic_clone_count` or, where the spine ran `cgt_support_split`, from the count that stage counted its copies on; the refusal gives only the clone counts. It refuses in a fixed sentence a sidecar that lacks those records, or whose output digest is not `--input-h5`'s. Every receipt records the counts under `geographic_clones`. + - E4's rules engine takes the frame's clone count (`PolicyEngineUKEngine(clones=K)`), so its `LHA_category` call runs clone by clone, as the stages' calls did. + - E4 and E5 no longer compare with the store a column that a later stage declares writing in the packaged source-stage manifest. They name it under `rewritten_later`, with those stages (María's ruling of 10 Nov). A rewrite the receipt's scope takes back does not count: `salary_sacrifice` and `regional_home_values`, where the artifact keeps the column each reversal reads. (Later in #931 `salary_sacrifice` alone, since `regional_home_values` retires: `931-spine-localises-home-values`.) E5 reports `student_loan_balance`, which `student_loans` re-splits. E4 reports four flags that later take-up and coherence stages rewrite: `would_claim_uc`, `would_claim_pc`, `would_claim_child_benefit` and `child_benefit_opts_out`. It also reports the six childcare columns that `childcare_copy_take_up` declares rewriting, which by its notes it re-draws on copies only. So E4 also names, under `rewritten_later_still_equal`, the rewritten columns the store still holds bitwise. + - E4 recomputes on what its stages read. `LHA_category` is materialized on the frame `frs_brma` read, as far as the artifact restores it. (Later in #931 the frame `area_brma` read.) Columns first written after that stage are dropped, and the flags `frs_take_up` drew that later stages rewrite are put back from the recompute. `lha_category_scope` names what stays as stored, with the stages that write it, `salary_sacrifice` among them, since its reversal takes back only the records it converted. The household weights are taken back through the band donors' funding and the SPI support channel's declared prior-mass shares, to rounding. `is_apprentice` is held exactly or, where those weights move the walk by rounding, to the walk on them, and `held_to_tolerance` then names it. + - E8's band donor block gives the layer in the forms of the stage's capped receipt, at every seating: + - the donors and each band's donors, in rounded distinct source households; + - the stage's funding rows, with their masses and factor withheld; + - `funding_within_share`, the stage's funding rule recomputed from the stored weights, which E8 also checks (`band_donor_funding_share`). It holds each rebuilt factor at E8's weight tolerance (relative 1e-9), not the stage gate's few ulps, since the stage's exact-total correction on one incumbent row moves a small stratum's rebuilt factor by more than that. + + It no longer writes the funding factors, band weights, weighted taxpayers or mass scale. Its checks still compare them. +- **SPI income band donors.** The stage's clone-index refusals are fixed sentences naming the column, with no count. +- **What moves.** At one clone, no spine value and no gate verdict moves. The sidecar gains `geographic_clone_count`, and the national line refuses a spine from group (b) on at its first node that reads the frame. Node keys move from `spi_income_band_donors` on, and across the national line. The identity tool's receipts gain the fields above. diff --git a/changelog.d/931-uk-spine-geographic-expand.changed.md b/changelog.d/931-uk-spine-geographic-expand.changed.md new file mode 100644 index 0000000000..224c7135e6 --- /dev/null +++ b/changelog.d/931-uk-spine-geographic-expand.changed.md @@ -0,0 +1,35 @@ +The UK spine gets its geographic clones after its source stages (#931). `frs_hmrc_spine_leaves` and `frs_annual_income_spells`, which join the raw FRS tabs on raw person ids, move ahead of `frs_take_up`, and two new stages follow them: `geographic_expand` (G1) gives every FRS household K geographic clones, and `geographic_identity` (G1b) keys each clone. `frs_take_up`, `frs_person_draws`, `frs_household_draws` and `frs_brma` then run on every clone (María's ruling of 10 Nov, decision 3). The spine build runs one clone by default, so no value moves. Positional stage numbers in the #931 fragments, comments and tests still follow the order before this change: 14 is `frs_hmrc_spine_leaves`, 15 `frs_annual_income_spells`, 10 to 13 the draws and `frs_brma`, and 16 onward as before. + +- **Order.** The two moved stages read only employment and self-employment income, age and gender. The draws read none of their outputs and write none of their inputs, and none of these stages sets weights. `frs_brma` stays the assembled checkpoint's last stage, so the assembled phase now covers the moved stages, G1, G1b and the draws, and the assembled gate admits `spi_support_channel`. (Later in #931 `area_brma` takes its place and the boundary with it: `931-spine-assigns-areas`.) +- **Declarations.** The sources schema and `ALLOWED_SOURCE_OPERATION_KINDS` gain two strict operation kinds. + - `expand_geographic_clones` declares the entity, the clone-index columns, the ordinal bound of 100, the copy-id table's `geographic_support` field at radix 10^9, clone-major order, an equal split of design weights, the household source columns, conservation to the split's rounding, a declared factor of 1.0 and a mass reason that names no clone count. + - `key_households_by_clone_path` declares the source `frs` and `frs_release.json`, which G1b also declares as its one artifact with `runtime_sha256_required`, so the release's bytes enter its stage contract. + + Each transform refuses any other declaration of its operation. G1 refuses any artifact, and G1b any but that one. +- **Clone count.** No declaration names K. `uk_spine_graph(geographic_clones=K)` puts it on G1's node, a whole number from 1 to 100, and refuses K above 1 for a manifest without G1. So K enters that node's key and every later one, never a stage contract. + - G1 is an EXPAND that conserves mass at design weights. Its kernel refuses a transform whose clone count differs from its node's, printing only the two counts. + - The spine build runs G1 at the clone count `--n-clones` asks for, one by default, and builds its engine with that count. `_rules_engine`'s `clones` defaults to None, so the full build's engine installs are unchanged. + - The H2 registry builds G1 and its engine from the fixture's `config.geographic_clones`. + - The dense line's `uk.full.identity` declares the geography key as a rewrite where the spine carries it, so the dense graph still compiles. +- **Source columns.** `source_household_id`, `source_year` and `source_household_key` are G1's outputs. `spi_support_channel` no longer declares or writes them, and its copies inherit them. It and the dense expansion default them from a household's own ids only on a table without `household_clone_index`, and refuse, naming the columns, a clone-indexed table that lacks any of them. `source_person_id` and `source_benunit_id` are not minted (decision 5). Their names live in the `geographic_clones` leaf beside the clone-index names (`UK_GEOGRAPHIC_SOURCE_COLUMNS`, `SOURCE_YEAR_COLUMN`, `SOURCE_HOUSEHOLD_KEY_COLUMN`), and so does the geography key's (`GEOGRAPHY_HOUSEHOLD_KEY_COLUMN`, which `atomic_area_support.IDENTITY_COLUMN` re-exports). So G1, G1b, the SPI support channel and the dense line read one definition, which their keys hash. +- **Counting and folding.** `frs_take_up`, `frs_person_draws` and `frs_household_draws` are given `source_household_id` and `household_clone_index`, so their receipts count true source households on every clone. `spi_employment_shell`, `spi_housing_shell`, `uc_capital_coherence` and `property_components` are given `household_clone_index`. +- **Root carriers.** `frs_spine` writes two typed household carriers, and `council_tax_band` itself is unchanged: + - `imd_decile` (int64): the household's decile on its own nation's index of multiple deprivation (IMDE, IMDW, IMDS or IMDN), 1 to 10, else 0; + - `council_tax_band_observed` (bool): False in Northern Ireland, for a code outside 1 to 9, for a band the household's region does not use and for an unknown region. + + The root refuses a household tab without one of the four IMD variables, naming it. Both carriers leave at the release boundary and stay out of every receipt and of the spine sidecar's nonzero shares. The root binds `calibrate.geography_constants`, from which both read the household's nation. The area assignment of #931 group (c) is their first reader. +- **microdf.** microdf-python's version joins every spine node's numerical dependencies (`not-installed` when absent), and the sidecar records it as `microdf_python` beside `engine_batches` when a call batches (decision 2). `pyproject.toml` and `uv.lock` are unchanged; the next lock bump declares it. +- **Texts.** + - The `copy_rule` of both support splits, `high_value_home_support` (stage 26) and `cgt_support_split` (stage 42), names the order the exact-total candidates are tried in. + - The notes of `isa_subscriptions` (stage 53) say tied balances share one weighted mid-rank and the Lifetime ISA reuses `balance_rank_weight` (decision 1), and `docs/uk-isa-subscriptions-1415.md` steps 3, 5 and 6 follow the code. + - The weight rule of `spi_income_band_donors` (stage 17) names one clone's donor count, and its draw a spine without the clone index. + - The bonus draw of `salary_sacrifice` (stage 48) draws within the adult's own clone from cells pooled over every clone, and says the lowest cell joins the one above. + - The source and notes of `spi_support_channel` (stage 16) and `private_school_attendance` (stage 52), the notes of `spi_income_band_donors`, `cgt_support_split`, `salary_sacrifice` and `uc_capital_coherence`, and `frs_spine`'s notes (the two carriers) say what each does on a clone-first spine. + - `childcare_copy_take_up`'s notes say `frs_take_up` draws each geographic clone on its own ids before any other copy exists, and that geographic clones keep those draws. + - Earlier #931 fragments that said G1 was unwired, or that no spine carries a clone index, now say what this change does. +- **Bindings.** The national gates hash `geographic_clones`, since their support-family fold reads the clone index on the national frame of a clone-first spine. The copy-id readers gain G1. The stage table keeps only the source column's name, with its reason rewritten, and the kernel table's clone-index reason no longer covers the national gates. +- **Dense and national.** The dense expansion refuses a spine carrying G1's clone index at any `n_clones`, naming #931 group (c), which builds the dense line on one. No national or dense build runs on such a spine before (c) (decision 4). The coverage manifest's new `geographic_expand` family requires G1's mass record under its declared reason, so a release from a spine built before this change fails the coverage gate too. +- **What moves.** No value at one clone. On the synthetic H2 fixture, the legacy oracle's final frame keeps the parent's rows, existing columns, weights, strata and mass records, in a new column order. It gains the three clone indices at 0, `geography_household_key` (which copies made after G1b repeat until (c)), the two root carriers and G1's mass record, and the funding strata of `spi_income_band_donors` read `REGION|0`. + - Every UK spine node re-keys. Seven stage contracts move with their declarations (`frs_spine`, `spi_support_channel`, `spi_income_band_donors`, `high_value_home_support`, `cgt_support_split`, `salary_sacrifice`, `private_school_attendance`), and G1's and G1b's are new. + - Outside the UK, adding the two kinds to `source_manifest` re-keys the shared transport gate battery, so NZ's terminal gate and its export and package nodes re-key. The US, NZ, BE and AM specs keep their fingerprints. + - `source_stages.json`, the H2 fixture and the coverage manifest are regenerated. diff --git a/changelog.d/931-uk-stages-ready-for-clones.changed.md b/changelog.d/931-uk-stages-ready-for-clones.changed.md new file mode 100644 index 0000000000..856867673b --- /dev/null +++ b/changelog.d/931-uk-stages-ready-for-clones.changed.md @@ -0,0 +1,50 @@ +The UK spine gains G1 and G1b, the steps that give it geographic clones (`931-uk-spine-geographic-expand` declares and wires them), and three of its stages are made ready for clones: salary sacrifice draws SPI bonuses within each clone, UC capital counts its donors in survey records, and the CGT support split reads the clone count from a module of its own (#931). No output moves: every spine is one clone today. + +- **Clone-index names.** `ladder_clone_index_column`, `ARTIFACT_CLONE_INDEX_COLUMN` and `geographic_clone_count` move unchanged to `microcosm.build.uk_runtime.geographic_clones`, which imports nothing first-party. `rowwise_dataset` and `cgt_support` re-export them, so every importer keeps working. The module joins the spine's counting modules, so every stage that counts by lineage hashes it. The CGT support split, which reads the clone count, binds it as a helper. The full build's population and export kernels, which read the names, hash it too: `uk.full.expand` and the other population kernels, `uk.full.identity`, `uk.full.property.value`, `uk.full.private_rent` and both export descriptions. +- **Geographic expansion (G1).** `geographic_expand.expand_uk_geographic_clones(frame, clones=K)` and its stage transform, `UKGeographicExpandStageTransform(stage, clones)`, turn a spine frame into K clones of every FRS household, clone-major. They call the shared clone expansion core, as María ruled on 10 Nov, with the UK's declaration, `UK_GEOGRAPHIC_CLONES`: + - clone `k` takes `id + k × 10^9` on all five id columns, in the copy-id table's `geographic_support` field, through two named functions that call the table (`mint_geographic_clone_ids`, `require_room_for_geographic_clones`); clone 0 keeps the frame's ids; + - each table carries its clone index (`person_clone_index`, `benunit_clone_index`, `household_clone_index`), 0 to K - 1, with exactly K values; + - each clone takes `w / K` of its household's design weight, the same float in every clone, and the mass log gains one record whose reason names no clone count; + - each household carries the source columns the SPI support channel wrote before G1 took them over: `source_year` (the frame's time period) and `source_household_key` (`"{source_year}:{source_household_id}"`, the values that channel wrote, as Python strings in an object column, the dtype the legacy plan gives it; the graph casts it to its string dtype), written before the clones are made, and `source_household_id`, its clone-0 id, which the core writes. + + It refuses, by name or by count only: + - a clone count outside 1 to 100 (`GEOGRAPHIC_CLONE_LIMIT`, the field's bound); + - weights on any entity but the household, or of any kind but design; + - a frame already carrying a clone index, a source column or a column of the UK copy lineage; + - id or membership columns that are not int64, the dtype the table mints, at every clone count, so one clone takes the same frames as more; + - a time period that is not a year; + - incoming ids at or above the field's radix; + - whatever the core refuses. + + Its evidence gives K, the copy-id field and its radix, the distinct source households rounded through `receipt_counts` at the floor of three, and the weight split and kind, with no row count, id or weight. +- **Geography identity (G1b).** `geographic_identity.key_uk_households_by_clone_path(frame)` and `UKGeographicIdentityStageTransform(stage)` give every household of G1's frame the key the dense line's `uk.full.identity` writes, `geography_household_key`. The key is `household_draw_key` over: + - the source `frs`; + - the FRS vintage frs_release.json pins for the household's `source_year`, which maps the release's survey year and its base year, the frame's time period, to its vintage; + - its `source_household_id`; + - its copy path, read from the lineage columns through `source_lineage.copy_paths`: empty for clone 0, and the `geographic_support` branch at `k` for clone `k`. + + So each clone keys as the dense identity keys that clone, and a larger K keeps every key. The key is in the graph's `string` dtype (`StringDtype("python")`). It refuses, by count or by name and never printing a key, a frame that already carries the key or lacks G1's int64 columns, a source year frs_release.json pins no vintage for, and repeated keys. Its evidence gives the distinct source households rounded through `receipt_counts`, that the keys are unique, the source, the vintages and the columns it reads. +- **SPI bonuses per clone (salary sacrifice).** `draw_spi_bonuses` takes `clone_indices`, each row's geographic clone. The stage passes each person's household `household_clone_index`, and a frame without the column is one clone. + - The pay cells and the donor-source floor still pool every clone's donors, the floor counted in distinct source households. + - Each SPI-channel adult now draws its bonus share from the donors of its own clone in its cell, in id order, on the same person-keyed uniform. + - At one clone the draw is unchanged, value for value. Where the clones agree on the donors' pay, clone 0 of K clones draws on the one-clone pool at `w / K`, up to the 1/K split's rounding, which can move a pooled cell edge where a quantile falls exactly on a cumulative weight share. + - A cell with no donor in an SPI adult's clone is refused, with no count, id or value in the message. + - The declared draw text (`spi_bonus_draw`) changes with the declarations. +- **UC capital donors counted in survey records** (María's ruling of 10 Nov). The stage folds its donors through `fold_geographic_clones`, and a cell stops widening once it holds `minimum_cell_donors` survey records: the distinct replica keys (`ReplicaFold.key_group`) among its donor rows, not the rows. + - A unit's geographic clones count once, whether they fold into one row or differ and donate apart. Any other copy, such as a high-value support copy, counts as a record of its own. + - A donor variant donates from its lowest-id clone. + - Without geographic clones every row is its own record, so the count and every draw are unchanged. + - The stage's notes change with the declarations. +- **What moves.** No output, at one clone or on any spine: both stages give the parent commit's output bit for bit on invented frames, with or without a one-clone index. Node keys move: + - every UK spine node. Every spine kernel hashes `graph_kernels`, which this commit edits, and the stages that count by lineage now also hash `geographic_clones`. Keys chain, so in the in-graph build every node after the spine moves too; + - on a bound spine, whose checkpoint kernel hashes no edited module, the whole dense line after the checkpoint: its population kernels hash `rowwise_dataset`, `graph_population` and now `geographic_clones`; + - on a bound spine, the national line from its problem on. The national problem and the calibration nodes hash `graph_population`, which this commit edits to bind `geographic_clones` into the population kernels' keys, and the national gates' registry modules include `uc_capital_coherence`. The national targets keep their key; + - the full gates' terminal phase code, which hashes `uc_capital_coherence` and `graph_population`. + + No generated file, declaration or `gates.json` changes. No transport kernel and no US, NZ or BE node hashes an edited module; only the US PUF-support builder's whole-tree resume fingerprint moves, as on any edit to packaged source. +- **Tests.** + - `test_uk_geographic_expand.py` holds G1 to the clone-first stand-in `clone_first_frame` bit for bit at K = 1, 2, 3, 7 and 15. It also checks the one-clone frame and the mass log it is handed, the layout at K = 3, the source key's dtype under either string inference, the mass record, the evidence, the declaration's named functions and each refusal, the int64 one at one clone and at two. + - `test_uk_geographic_identity.py` holds every clone's key to the dense identity kernel's at K = 1 and 3. It also checks the clone path, uniqueness and nesting across K, the dtype, the vintage for the release's survey and base years, the evidence and each refusal. + - `test_uk_clone_first_nesting.py` runs an invented frame through G1 and stages 16, 17 and 42 at K = 1, 2, 3, 7 and 15. Clone 0 equals the one-clone frame on its ids, G1's columns, stage 16's copies and stage 17's seats, and every frame and copy keeps the clone-index invariant. K × each clone-0 weight matches one clone's within stated rounding classes, stage 42 nests approximately, and through G1 rule B's correction lands on a copy at 7 and 13 clones. + - The salary-sacrifice and UC capital tests cover the per-clone draw and its refusal, the stage at one and three clones, a one-clone index, clones that differ, and a high-value copy. + - `test_uk_cgt_support.py` holds every spelling of the household clone index to the module's name, and its nesting scenario's data move to `test_support` for the nesting test to share. `test_uk_clone_expansion_core.py` takes G1's declaration, and `test_uk_copy_id_table.py` lists `geographic_expand` among the copy-id readers. `test_uk_graph.py` re-derives the binding tables, checks which nodes write the clone index columns, and checks that an edit to `geographic_clones` re-keys every counting stage. diff --git a/changelog.d/931-uk-was-hbai-net-income.changed.md b/changelog.d/931-uk-was-hbai-net-income.changed.md new file mode 100644 index 0000000000..e5538812dd --- /dev/null +++ b/changelog.d/931-uk-was-hbai-net-income.changed.md @@ -0,0 +1,4 @@ +The UK WAS wealth and Lifetime ISA stages (`was_wealth`, `was_lisa`) match the WAS donor's household net income to the engine's closest concept, its HBAI net income (#931), which the LCFS and ETB services stages also read as a predictor (microcosm#1164). The donor's income is WAS `DVTotinc_bhcR8`, net regular household income before housing costs less council tax. The recipient's is policyengine-uk's `hbai_household_net_income_before_reset_to_zero`, HBAI income before housing costs and before negative incomes are reset to zero, in place of `household_net_income`, which also takes off stamp duty, the TV licence and the business rates and corporate stamp duty that `shareholding` spreads over the households the engine simulates, and adds the imputed value of NHS, education and transport spending. Both sides call the predictor `hbai_household_net_income`. The two incomes are close but not the same: the recipient's also counts HBAI's in-kind items (free school meals, school milk and fruit, the free TV licence's value, Healthy Start vouchers and the Tax-Free Childcare top-up) and the Warm Home Discount, which WAS's regular income is not documented to include, and how WAS treats pension contributions, maintenance paid, student loan repayments and negative incomes is not verified. + +- **What moves.** The WAS chain keeps its segments, order and seeds, but every WAS draw can move. So do the Lifetime ISA draws, the later stages that read wealth (means-tested capital, CGT selection, property components and high-value support), and so the calibrated weights on both lines. A sampled f010 arm measures the change. +- **Declarations.** `was_wealth` declares the donor column in its `direct_maps`, and the stage refuses a declared direct map its donor cleaning does not read. `was_lisa`'s declared engine reads now list what it materializes, without `num_adults` and `num_children`, which come from the FRS family roles. Its realised profile keys its income tertiles `by_hbai_household_net_income_tertile`, and the ownership coefficients `log1p_hbai_household_net_income`. diff --git a/docs/agent-guide.md b/docs/agent-guide.md index 181ae3f60e..a36c7b55d4 100644 --- a/docs/agent-guide.md +++ b/docs/agent-guide.md @@ -42,16 +42,18 @@ or `uv sync --all-packages --all-extras`. To keep both countries set up, point PR CI (`.github/workflows/test.yml`) has `lint`, `engine-free`, `engine-us`, `engine-uk`, `integration-uk`, `wheels`, and `installed-emitter` jobs, plus the -`select-countries` orchestration job and the `ci` job. +`select-countries` orchestration job and the `ci` job. The same workflow's +`engine-uk-after-merge` job runs on pushes to `main` only, after merge. `ci` gives the `main` ruleset one check that covers the country jobs. It fails unless every other job passed, or skipped because `select-countries` succeeded -and reported `false` for its country. A skipped matrix job reports under its -unexpanded name, so a ruleset cannot require `engine-us` or `engine-uk` by -name. The ruleset also names each other check that runs on every pull -request: `lint`, `wheels`, `select-countries`, `installed-emitter` and both -`engine-free` legs. When `engine-free`'s matrix changes, change the ruleset -too, because a leg's check name holds its Python version. +and reported `false` for its country, or skipped because it runs after merge +and the run's event (`GITHUB_EVENT_NAME`) is not a push. A skipped matrix job +reports under its unexpanded name, so a ruleset cannot require `engine-us` or +`engine-uk` by name. The ruleset also names each other check that runs on +every pull request: `lint`, `wheels`, `select-countries`, `installed-emitter` +and both `engine-free` legs. When `engine-free`'s matrix changes, change the +ruleset too, because a leg's check name holds its Python version. The workflow runs on a pull request only when it targets `main`. Retargeting a pull request to `main` starts no run, because GitHub reports a base change as @@ -60,7 +62,11 @@ request retargets, push a commit or close and reopen it, then merge once its checks pass. The plan names the jobs the selection may leave out: each job whose test -groups all need the same country's engine. The lint job runs +groups all need the same country's engine. It also names the jobs that run +after merge: each job whose test groups are all marked `after_merge`. Such a +job's `if` is `github.event_name == 'push'` and it needs no selection; `ci` +needs it, so a push must run it, and `require-results` passes its skip on any +other event. The lint job runs `tools/ci_test_plan.py verify-workflow`, which parses the workflow with PyYAML and refuses: @@ -71,6 +77,7 @@ and refuses: - a job the selection may leave out that doesn't need `select-countries`, and a job that needs one the selection may leave out, unless both follow the same country's selection; +- a job that needs an after-merge job, unless it runs after merge too; - a `ci` job that lacks `if: always()`, omits a job from `needs`, or does anything but check out the repository and run `require-results` on its `needs` context; @@ -268,6 +275,9 @@ of these directories below its package's `tests/` directory: aging, fiscal refresh, release preparation, scoring, or another complete data workflow. Every module in this category runs in a fresh pytest process. - `engine/uk/`: imports or executes the UK engine. +- `engine_after_merge/uk/`: imports or executes the UK engine but is too slow for + every pull request; runs through the `engine-uk-after-merge` job on pushes to + main only, after merge. - `integration/uk/`: runs serially through the `integration-uk` job with the explicit `--run-integration` option. - `installed_emitter/shared/`: starts the separately installed diff --git a/docs/uk-atomic-household-lineage.md b/docs/uk-atomic-household-lineage.md index d5c10c3f76..dd9951f61a 100644 --- a/docs/uk-atomic-household-lineage.md +++ b/docs/uk-atomic-household-lineage.md @@ -29,7 +29,7 @@ The UK's copy lineage is declared once, as data, in `uk_runtime.household_lineag - `record_sources` gives every record of any entity its true source through household membership. Every UK floor counts distinct sources through it (`receipt_counts.record_source_households`), so a household's copies and clones count once. - `copy_paths` reads every household's ordered clone path; `uk_household_clone_paths` is the UK's strict entry for the identity kernel. -- `replica_keys` and `collapse_replicas` keep one training row per true source and path without the geographic branch, with the clones' weights summed. The FRS-trained fits fold their training rows through `household_lineage.collapse_geographic_clones`: the SPI stage-2 and reference forests, the housing shell, the UC reporter redraw, the UC capital donor cells, the salary-sacrifice forest and the SPI employment shell. Once geographic clones precede enrichment, each fit still sees each survey record once. A frame without geographic clones keeps its rows and weights value for value. +- `replica_keys` and `fold_replicas` keep one training row per true source and path without the geographic branch, with the clones' weights summed; through the UK entries each variant trains on its lowest-id clone, whatever the row order. `collapse_replicas` returns the fold's rows and weights alone. The FRS-trained fits fold their training rows through the UK entry (`household_lineage.fold_geographic_clones`; `collapse_geographic_clones` returns its rows and weights alone): the SPI stage-2 and reference forests, the housing shell, the UC reporter redraw, the UC capital donor cells, the salary-sacrifice forest and the SPI employment shell. UC capital also counts its cells' donors in survey records. On a frame of more than one geographic clone, each of these stages records the fold in its evidence as `geographic_fold` (`household_lineage.geographic_fold_receipt`): the clone count, the true source households the fit selected and those whose clones differ on what it reads, rounded as every receipt count is, and the share that differ, given only where ten or more of the selected sources differ and none or ten or more do not. Once geographic clones precede enrichment, each fit still sees each survey record once. A frame without geographic clones keeps its rows and weights value for value. - Each fit passes what it reads (its predictors and targets, or the columns they derive from), and clones fold only where they agree on it. Clones that differ, through a draw keyed on their own ids or a stage run per area, stay rows of their own. Each variant carries the summed weight of the clones that share it, so the fit sees every clone's value at its share and nothing is dropped. ## Verification scope diff --git a/docs/uk-cut-anyone.md b/docs/uk-cut-anyone.md index ce82d62e4e..124936ab94 100644 --- a/docs/uk-cut-anyone.md +++ b/docs/uk-cut-anyone.md @@ -155,9 +155,11 @@ MICROCOSM_UK_CUT_MODAL_RUN=1 modal run --detach tools/modal_uk_cut.py --commit < `tools/uk_cut_inputs.json` says where each input lives. Every expected digest is read at run time from the pins the build already checks: -`uk/source_stages.json`, `uk/frs_release.json`, `uk/chronicle_feed.json` and -`uk/gates.json`. A file is installed only if it is byte-identical to what the -build will accept, and nothing unverified is ever left at an input's path. +`uk/source_stages.json`, `uk/frs_release.json`, `uk/chronicle_feed.json`, +`uk/gates.json` and the conditioned atomic-area support register in +`uk/spec/sources.yaml`. A file is installed only if it is byte-identical to +what the build will accept, and nothing unverified is ever left at an input's +path. The build the cut compares with is not a fixed input: the `reference` step resolves it ([The reference build](#the-reference-build)). @@ -177,6 +179,7 @@ is what admits a builder to the licensed inputs. | NTS 2002-2024 (6 tabs) | `policyengine-uk-data-private` `nts_2002_24.zip` @ `559b69e9` | fetched, every member matches its pin | | LCFS 2024-25 (2 tabs) | `policyengine-uk-data-private` `lcfs_2024_25.zip` @ `11051052` (a repack of the extracted files) | fetched, every member matches its pin | | APS April 2024-March 2025 (1 tab) | `policyengine-uk-data-private` `aps_2024_25.zip` @ `ebebd1e0` (a repack of the extracted files) | fetched, the member matches its pin | +| Conditioned atomic-area supports (3 files, microcosm#931) | not hosted yet: no location is named | pinned to `sources.yaml`'s conditioned register, which the spine holds `--atomic-support-ew`, `--atomic-support-scotland` and `--atomic-support-ni` to. Until a location is named, `fetch` refuses them unless the three files `tools/build_uk_conditioned_area_supports.py` writes are already in `inputs/atomic_area_supports/`, where they are verified like any other input | ### NTS, LCFS and APS diff --git a/docs/uk-dense-release-assembly-runbook-762.md b/docs/uk-dense-release-assembly-runbook-762.md index ed874baded..364ed35d36 100644 --- a/docs/uk-dense-release-assembly-runbook-762.md +++ b/docs/uk-dense-release-assembly-runbook-762.md @@ -126,13 +126,28 @@ manifest, and refuses the national role's flags. `--release-candidate` pins the doctrine (bound 10, `nation_grain_family_equal_sqrt_count`, K=15, 2000 epochs). The rotated holdout runs only with `--rotated-holdout` (an opt-in, report-only diagnostic; `--skip-holdout` no longer exists, and the national -role refuses `--rotated-holdout`). The engine may resolve in a single block or per clone -(`--engine-blocks K`): each block's engine weights are scaled to the pool so -weight-share formulas see the pool's denominator, and the measures receipt -records whether the blocks were identical copies (an exact representation); -a per-block run is releasable only when it was. Best-effort staging -telemetry uploads every 300 s by default on this driver (the Hub allows about -128 commits per hour per repository). +role refuses `--rotated-holdout`). The engine may resolve in a single block or +per clone (`--engine-blocks K`). Per clone, each block's engine weights are +scaled to the pool's mass, and in every simulation the block builds, +policyengine-uk's share formulas (`shareholding`, `corporate_land_value`, +`consumption_shareholding`) are corrected to the pool's denominator. A guard +judges every whole-population formula the block ran against the UK +declaration. Nothing a block computes may fall in the Universal Credit +rebalancing years (2026 on), where policyengine-uk draws new-claimant status +by benefit-unit position (microcosm#931). The measures receipt records the +representation exact only when every block applied the correction, its checks +of the engine's key totals and growth held, and its guard verdict was clean; +`exact_basis` (`pool_share_injection_with_population_guard`) names that basis. +Exact then means equal to one engine over the pool to float32 rounding, not +bitwise. A receipt without `exact_basis` predates the correction, when exact +meant identical copies. A per-block run is releasable only when it is exact. +While policyengine-uk draws new-claimant status by position, a calibration +year of 2026 or later therefore leaves every per-clone run inexact, and so +not releasable, and refuses every batched spine call. A block's scratch +engine holds no year after the calibration year (microcosm#1615): a measure +dated later is refused, and the block computes at no such year. +Best-effort staging telemetry uploads every 300 s by default on this +driver (the Hub allows about 128 commits per hour per repository). For scale: at K=25 with `--engine-blocks 25` the graph driver's dense solve took about 3 hours (2,000 epochs over 1.6 million households, M1 Max), each of three size-search probes about as long, and the build peaked at 24.7 GiB (the @@ -451,14 +466,18 @@ uv run --no-sync python tools/preflight_uk_local_release_candidate.py --candidat The pre-flight checks the manifest and the signed gate report for everything the contract will demand: release posture attested, shippable, every -release-blocking gate passed, an exact engine population (single block, or -per-block with identical copies scaled to the pool), the doctrine values (for -a sized run, the size doctrine, a realized count equal to the request and the -size search's L0 bracket), the -A15 census household uprating and A17 tenure application, the measure -exclusions and their windows, the holdout (recorded as not requested, or, when -`--rotated-holdout` asked for it, measured with five folds and a finite mean, -and in agreement with the request), the Logbook row, the artifact digest. +release-blocking gate passed, an exact engine population (a single block, or +one block per clone that the build recorded exact), the doctrine values (for a +sized run, the size doctrine, a realized count equal to the request and the +size search's L0 bracket), the A15 census household uprating and A17 tenure +application, the measure exclusions and their windows, the holdout (recorded +as not requested, or, when `--rotated-holdout` asked for it, measured with +five folds and a finite mean, and in agreement with the request), the Logbook +row, the artifact digest. A build now records one block per clone exact only +when every block was corrected to the pool's share denominator with a clean +guard. The pre-flight does not require `exact_basis`, so a receipt from before +the correction, when exact meant identical copies, passes too; whether it +should is open for Max's review. `--incumbent-manifest /incumbent_local_surface_manifest.json` checks that the extraction manifest you will attach exists. The pre-flight no @@ -509,15 +528,18 @@ uv run --no-sync python tools/evaluate_uk_incumbent_surface.py \ with `--incumbent-diagnostic` (or when `--steps` names them). The build's release-blocking fit gates read the measures it resolved, per -clone when `--engine-blocks K`; the build records that representation as exact -only for the weight-share scaling -(`engine_population_representation.exact_scope`), and engine draws that depend -on a record's position can differ between per-clone and single-block runs. The -release states this in its `engine_measurement` limitation; nothing re-measures -the released file on one engine unless this evaluation is run. Its assessment -marks any national or local row more than 25% from its target, or any family -of five or more rows with fewer than half within 25%. Signed deferrals and -measure exclusions do not exempt a row. +clone when `--engine-blocks K`. `engine_population_representation.exact_scope` +says what the representation's `exact` covers. When `exact` holds, the blocks +were corrected to the pool's share denominator with a clean guard, and nothing +they computed fell in the Universal Credit rebalancing years (2026 on), where +policyengine-uk draws new-claimant status by benefit-unit position. The +measures then equal one engine over the pool to float32 rounding, at the +measure step's starting weights. The release states this in its +`engine_measurement` limitation; nothing re-measures the released file on one +engine unless this evaluation is run. Its assessment marks any national or +local row more than 25% from its target, or any family of five or more rows +with fewer than half within 25%. Signed deferrals and measure exclusions do +not exempt a row. Use the actual metrics and weights filenames from the extraction manifest. Only one engine block is accepted. The evaluation records the pinned row diff --git a/docs/uk-full-build-graph.md b/docs/uk-full-build-graph.md index f8d0ff0ebe..86ad2bd684 100644 --- a/docs/uk-full-build-graph.md +++ b/docs/uk-full-build-graph.md @@ -65,15 +65,15 @@ For smaller outputs, add `--dataset-households 150000` to use the common informe ## The national role -`--release-role national` builds the certified national line on the same graph. The driver parses and validates it with the same posture-aware validator (the national role refuses the dense role's arguments, `--release-candidate` among them, and requires a bound `--input-h5` with `--input-sha256`; a `--spine-request` is refused because the national line solves a pinned checkpoint as bound) and composes the national graph (`uk_runtime.graph_national`): `uk.full.spine_checkpoint` admits the bound checkpoint, `uk.full.national_targets` compiles the national register from the pinned Chronicle artifact (measure exclusions, band edges, the frozen-register check, the feed pin with `--allow-unpinned-feed`; no ladder, no local surface), `uk.full.national_problem` refuses non-finite inputs, resolves the engine measures on the bound frame, materialises the register and encodes one ordered problem with the national doctrine's `family_equal` loss weights, mass reason and bounds in its bindings, `uk.full.dense` and `uk.full.calibrated` are the shared calibration nodes under the doctrine's epochs, learning rate and seed (the dense node's admission is the bound spine's provenance artifact rather than a preflight battery, because the national line runs no pre-solve battery: its source and reference gates are the release-cut certifier's), and `uk.full.gates.calibrated` evaluates the seven `UK_CALIBRATION_GATE_SCOPE` gates on the calibrated population. The driver then materialises the seam-shaped evidence from the stored artifacts: the diagnostics through the seam writer with the build block (the digest measured from the file), the target-support sidecars, the calibration-seam battery replayed from the stored phase report and signed (`GateBatteryRun.record_phase`), `microcosm_uk_2024_25.h5`, a graph readback of the written H5 (`uk.full.national.readback`), the schema-1 `build_record.json` that `tools/certify_uk_release_cut.py` certifies, the frozen `national_target_registry.json` and `national_contract_registry.json`, and the schema-4 `rowwise_candidate_manifest.json` (`uk_runtime.national_role`). The Logbook row is on the seam's `uk-frs-calibration` pipeline with the seam's attempt id; staging telemetry, the staged bundle and the end-of-build comparison with the reference (the current national build, microcosm#1095) follow the dense role's switches. A blocked battery leaves the report on disk (`blocked_at_phase: "terminal"`), writes no H5 and records a failed row. The [national calibration runbook](uk-national-calibration-runbook-623.md) and the [national release assembly runbook](uk-national-release-assembly-runbook-806.md) describe the outputs. The in-process calibration seam (`run_uk_calibration`, `UKNationalCalibrationStage`) is retired; its contracts (gate scopes, sidecar authentication, provenance blocks, scoped signing) stay in `uk_runtime.calibration_run`. Parity with the seam is a solved question of construction (the same register, the same matrix row for row, the same solver under the same doctrine and seed, the same mass record) checked on the licensed spine. +`--release-role national` builds the certified national line on the same graph. The driver parses and validates it with the same posture-aware validator (the national role refuses the dense role's arguments, `--release-candidate` among them, and requires a bound `--input-h5` with `--input-sha256`; a `--spine-request` is refused because the national line solves a pinned checkpoint as bound) and composes the national graph (`uk_runtime.graph_national`): `uk.full.spine_checkpoint` admits the bound checkpoint, `uk.full.national_targets` compiles the national register from the pinned Chronicle artifact (measure exclusions, band edges, the frozen-register check, the feed pin with `--allow-unpinned-feed`; no ladder, no local surface), `uk.full.national_problem` refuses non-finite inputs, resolves the engine measures on the bound frame, materialises the register and encodes one ordered problem with the national doctrine's `family_equal` loss weights, mass reason and bounds in its bindings, `uk.full.dense` and `uk.full.calibrated` are the shared calibration nodes under the doctrine's epochs, learning rate and seed (the dense node's admission is the bound spine's provenance artifact rather than a preflight battery, because the national line runs no pre-solve battery: its source and reference gates are the release-cut certifier's), and `uk.full.gates.calibrated` evaluates the seven `UK_CALIBRATION_GATE_SCOPE` gates on the calibrated population. The driver then materialises the seam-shaped evidence from the stored artifacts: the diagnostics through the seam writer with the build block (the digest measured from the file), the target-support sidecars, the calibration-seam battery replayed from the stored phase report and signed (`GateBatteryRun.record_phase`), `microcosm_uk_2024_25.h5`, a graph readback of the written H5 (`uk.full.national.readback`), the schema-1 `build_record.json` that `tools/certify_uk_release_cut.py` certifies, the frozen `national_target_registry.json` and `national_contract_registry.json`, and the schema-4 `rowwise_candidate_manifest.json` (`uk_runtime.national_role`). The Logbook row is on the seam's `uk-frs-calibration` pipeline with the seam's attempt id; staging telemetry, the staged bundle and the end-of-build comparison with the reference (the current national build, microcosm#1095) follow the dense role's switches. A blocked battery leaves the report on disk (`blocked_at_phase: "terminal"`), writes no H5 and records a failed row. The [national calibration runbook](uk-national-calibration-runbook-623.md) and the [national release assembly runbook](uk-national-release-assembly-runbook-806.md) describe the outputs. The in-process calibration seam (`run_uk_calibration`, `UKNationalCalibrationStage`) is retired; its contracts (gate scopes, sidecar authentication, provenance blocks, scoped signing) stay in `uk_runtime.calibration_run`, except the spine and local gate scopes and the scoped gate manifest, which are defined in `uk_runtime.gate_scopes` and re-exported by `calibration_run`. The full gate kernels hash `gate_scopes` in place of the runner, and the preflight's own code leaves the runner out, so in the in-graph build an edit to the runner re-keys only the calibrated gates and what follows them; on a bound spine the checkpoint, whose kernel hashes the runner, still re-keys the preflight and the solve, and on the national line every node. Parity with the seam is a solved question of construction (the same register, the same matrix row for row, the same solver under the same doctrine and seed, the same mass record) checked on the licensed spine. ## Graph owners and shared contracts -The spine graph is the 56-stage source roster declared by the UK spec, built by `uk_runtime.spine_build` (the spine tool moved into the package; `tools/build_uk_frs_spine.py` is a six-line shim over it). The superseded `frs_hmrc_retained_leaves` and `hmrc_spi_income` stages and the `UK_SPINE_EXCLUSIONS` list that hid them are removed; the active HMRC path is `frs_hmrc_spine_leaves`, `spi_support_channel`, `spi_income_band_donors` and `hmrc_spi_income_spine`, and the FRS HMRC leaf columns come from `uk_runtime.frs_hmrc_source`. The spine's assembled and transferred gate batteries are graph nodes (`uk_runtime.graph_evidence`); stage evidence and fit-weight records are read back from the content store through the shared `stage_evidence` artifact rather than from in-memory collectors; the HMRC replay sidecar is rebuilt from the SPI stage's checkpoint metadata; the spine sidecar records the operation inventory and the graph manifest, and the graph manifest is saved as `spine.graph.json` under the checkpoint root. +The spine graph is the 63-stage source roster declared by the UK spec, built by `uk_runtime.spine_build` (the spine tool moved into the package; `tools/build_uk_frs_spine.py` is a six-line shim over it). The superseded `frs_hmrc_retained_leaves` and `hmrc_spi_income` stages and the `UK_SPINE_EXCLUSIONS` list that hid them are removed; the active HMRC path is `frs_hmrc_spine_leaves`, `spi_support_channel`, `spi_income_band_donors` and `hmrc_spi_income_spine`, and the FRS HMRC leaf columns come from `uk_runtime.frs_hmrc_source`. The spine's assembled and transferred gate batteries are graph nodes (`uk_runtime.graph_evidence`); stage evidence and fit-weight records are read back from the content store through the shared `stage_evidence` artifact rather than from in-memory collectors; the HMRC replay sidecar is rebuilt from the SPI stage's checkpoint metadata; the spine sidecar records the operation inventory and the graph manifest, and the graph manifest is saved as `spine.graph.json` under the checkpoint root. | Operation | Graph owner | | --- | --- | -| Raw FRS and donor preparation, enrichment, support channels | The 56 UK spine stage nodes and their declared composite operations | +| Raw FRS and donor preparation, enrichment, support channels | The 63 UK spine stage nodes and their declared composite operations | | Assembled and transferred spine gates | `spine.gates.assembled`, `spine.gates.transferred` | | Bound checkpoint admission | `uk.full.spine_checkpoint`, when resuming a saved spine | | Pool sample and mass normalization | `uk.full.sample`, `uk.full.normalize` | @@ -108,6 +108,14 @@ Shared machinery includes graph execution/storage/replay, typed artifacts, expli The full target compiler preserves the unreduced band-edge register, reference-period compilations, approved exclusions and frozen-register completeness checks before target selection. A nonempty country-only problem can have zero local rows. Local holdout is then inapplicable, while source, identity, mass and geographic integrity checks remain applicable. Omitted rows are never reported as fitted constraints. +## Engine blocks and spine batches + +`--engine-blocks K` resolves `uk.full.measures` with one policyengine-uk engine per clone rather than one over the pool; above one, it must equal `--n-clones`. Each block's household weights are scaled to the pool's mass. In every simulation the block builds (the measure simulation, its baseline, and the annual-income and counterfactual simulations with theirs), `shareholding`, `corporate_land_value` and `consumption_shareholding` are set, at each period the block computes, to the block's own value times `c_b = f_b·S_b/S`. That is the block's pool-mass factor times its allocation-key total over the pool's, both from totals the representation records (microcosm#931). Before it sets them, the block checks that the engine's own key total is `f_b·S_b` and that the engine carries each key and the weights by one factor per year. The annual-income simulation carries other spells by the engine's own growth parameters, so every block uses the same factor. A guard records every formula each simulation runs once the shares are set, and judges policyengine-uk's whole-population formulas against the UK declaration (`microcosm.frame.adapters.policyengine_uk_population`, on the shared `engine_population` machinery). The three share formulas must be the corrected inputs at every period, and `months_since_last_birthday` and `attends_private_school` stored inputs. The engine's income and wealth deciles and relative-poverty flags are refused, and any other such formula that runs must be zero under the parameters checked. Any work in the Universal Credit rebalancing years (2026 on) is refused too, since policyengine-uk draws new-claimant status there by benefit-unit position. The measure step records a violation rather than raising it. Once the blocks ran, `engine_population_representation.exact` holds only if every block applied its correction, passed its checks and got a clean verdict, and `exact_basis` (`pool_share_injection_with_population_guard`) names that basis. The measures then equal one engine over the pool to float32 rounding, not bitwise. An inexact per-block run is not releasable. So while policyengine-uk draws new-claimant status by position, a calibration year of 2026 or later leaves every per-clone run inexact, and so not releasable, and refuses every batched spine call. A block's scratch engine holds no year after the calibration year (microcosm#1615): a measure dated later is refused, and the block computes at no such year. A single block, as on the national line, is exact by construction and records no correction or guard fields. + +A spine engine call whose frame carries more than one clone runs one simulation per clone, since one simulation over a cloned spine does not fit in memory. `PolicyEngineUKEngine.materialize` decides from the frame, not from a setting: a call whose household table carries `household_clone_index` with more than one clone runs its simulations in ascending clone order, and each is released before the next is built. Each clone's simulation is built as one over the frame is (`PolicyEngineUKEngine._read_year_simulation`): its dataset is projected to the call's year and no further, and the Universal Credit rebalancing loop stops at that year while it is constructed (microcosm#1615). A frame without the column, or with one clone, takes one simulation as before. A batched call's requests are judged first. A period in the rebalancing years, a refused formula, a frame without `months_since_last_birthday`, or a request for `attends_private_school`, `private_school_vat`, `baseline_private_school_vat` or `gov_tax` before `private_school_attendance` stores attendance refuses the call before any engine is built. Until then the spine's pass lets attendance run only where the three parameters the private-school VAT rate reads equal their baseline's and `private_school_vat_change` comes out zero, and only if `gov_tax` does not run. Unlike a measure block, each batch reads the pool's shares at the frame's own weights, unscaled (`c_b = S_b/S`), and is judged under the spine's pass, which raises on any violation rather than recording it, since the values come back as plain arrays. People come back in the frame's order and groups in ascending id order, as one simulation returns them. Each simulation holds the frame's ids where policyengine-core's 32-bit store keeps them apart; an entity whose stored ids would repeat takes 1 plus each id's rank in that simulation's frame, and each person's membership takes the new id of the group it names, so rows keep their order and the frame keeps its ids. Every spine frame from `geographic_expand` on carries `household_clone_index`, 0 throughout while the spine build runs one clone, so every spine call takes one simulation today, and batching moves no spine value. + +policyengine-uk's `attends_private_school` ranks each household's market income plus benefits, less its income tax and National Insurance, over the people a simulation holds, so the `private_school_attendance` stage restates the formula's last step over the whole frame. It computes that income from household-local engine values, ranks it with microdf's `MicroSeries.percentile_rank`, and reads the attendance rates and population factor from the engine's own parameters. Over one simulation it also asks for the engine's flags, stores them once the restatement matches them bit for bit, and raises otherwise. Clone by clone it asks only for the household-local values and stores the restatement. Its evidence records `rank_route` (`engine_with_parity` or `pooled_restatement`) and `engine_parity_checked`. The spine sidecar's `rules_engine` gains `engine_batches`, and beside it `microdf_python`, the microdf-python version whose weighted percentile rank `private_school_attendance` then stores, only on a spine whose calls run clone by clone, and the engine identity does not move with batching. Every stage that calls the engine through the adapter binds the adapter, `engine_population` and `policyengine_uk_population` into its implementation hash, and `microcosm.frame.materialize` and `microcosm.frame.adapters.engine_identifiers`, through which the adapter builds each simulation's dataset and the ids it holds. + ## Replay and outputs The shared store defaults to `/.graph-store`. `--graph-store` can reuse another store. `--resume require` requires completed numerical nodes and evidence to be available; output materialization and byte readback still verify the recreated files. `--require-solve-reuse` requires only the solve, so a second release build can re-run the calibrated gates on it (see [a second build that reuses the solve](#a-second-build-that-reuses-the-solve)). `--resume-size-checkpoint` imports a legacy size-search checkpoint only after validating invocation identity, ordered target/household axes, initial weights and recomputed losses. It skips the saved dense solve and search. New runs persist their intermediates as graph artifacts before drawing. diff --git a/docs/uk-isa-subscriptions-1415.md b/docs/uk-isa-subscriptions-1415.md index 645cd99458..2dc184b3e4 100644 --- a/docs/uk-isa-subscriptions-1415.md +++ b/docs/uk-isa-subscriptions-1415.md @@ -82,7 +82,10 @@ How the tables relate, on HMRC's own figures: (`shortfall_weight_unmet`); no holder is invented. 3. **How much.** Within each income band and class, amounts follow Table 9.7's subscription bands: uniform inside a band, exactly £20,000 at the top. The - rank blends the person's balance rank with a keyed uniform. + rank blends the person's rank on the class's balance (cash for cash only, + stocks and shares for stocks and shares only, the two together for both) + with a keyed uniform, half each (`balance_rank_weight`). Holders tied on a + balance share its rank, so the keyed uniform alone orders them. 4. **The both class.** Each person's amount is split by their balance shares, shifted on the logit scale so the class's stocks-and-shares total equals Table 9.4's 2023-24 stocks-and-shares amount less the stocks-and-shares-only @@ -90,19 +93,27 @@ How the tables relate, on HMRC's own figures: 5. **Growth to 2024-25.** Each class's people grow by Table 9.4's account growth (cash accounts for cash only, stocks-and-shares accounts for stocks and shares only, both together for both), added from eligible holders not yet - subscribing. Amounts then scale, one factor per type, to Table 9.4's 2024-25 - amounts (£95.6bn of cash and £37.2bn of stocks and shares). Subscribers - already at the £20,000 overall limit stay there. A person whose combined - amount lies between £18,000 (Table 9.7's last band floor below the limit) and - the limit moves on a line from their scaled £18,000 up to it, so the range just - below the limit stays filled; the rest are capped at it. + subscribing. The added people are selected, and their amounts drawn as in + step 3, in walks of their own, apart from the 2023-24 subscribers; those + added to the both class split at the shift step 4 found. Amounts then + scale, one factor per type, to Table 9.4's 2024-25 amounts (£95.6bn of cash + and £37.2bn of stocks and shares). Subscribers already at the £20,000 + overall limit stay there. A person whose combined amount lies between + £18,000 (Table 9.7's last band floor below the limit) and the limit moves on + a line from their scaled £18,000 up to it, so the range just below the limit + stays filled; the rest are capped at it. 6. **Lifetime ISA payments.** Every holder aged 18 to 47 with room under the - overall limit pays in; a holder with none pays nothing. 18 to 47 are the ages - that can hold one in 2024-25, since accounts opened at 18 to 39 from April - 2017. Amounts lie in (0, £4,000], drawn by a truncated-exponential tilt so - the total equals Table 9.4's 2024-25 Lifetime ISA amount (£2,818m from - 1,136,000 accounts), so the baseline leaves nothing above the overall limit. - The rank blends the balance and income ranks with a keyed uniform. + overall limit pays in; a holder with none pays nothing, and so does a holder + over 47. 18 to 47 are the ages that can hold one in 2024-25, since accounts + opened at 18 to 39 from April 2017. Amounts lie in (0, £4,000]: each is a + truncated-exponential draw capped at the holder's room under the overall + limit, with the tilt set so the total equals Table 9.4's 2024-25 Lifetime + ISA amount (£2,818m from 1,136,000 accounts). The cap, not the tilt, is what + leaves nothing above the overall limit in the baseline. The rank blends the + person's Lifetime ISA balance rank with a keyed uniform, as in step 3 and + with step 3's `balance_rank_weight`, since the Lifetime ISA operation + declares none of its own, but over every holder aged 18 to 47, not by + income band. ## Decisions (28 October 2026) diff --git a/docs/uk-lisa-1003.md b/docs/uk-lisa-1003.md index 14d03b9d03..8e91a33103 100644 --- a/docs/uk-lisa-1003.md +++ b/docs/uk-lisa-1003.md @@ -106,7 +106,16 @@ and its own WAS financial draws before the LISA draw. weight each person carries. Predictors: five age groups (the public file has five-year age bands only), the logarithm of the person's earnings, household net income, gross financial wealth, cash ISA, stocks-and-shares ISA and - savings, private renting, sex and the number of children. The intercept is + savings, private renting, sex and the number of children. Household net + income is the `was_wealth` predictor: WAS `DVTotinc_bhcR8` (net regular + income before housing costs, less council tax) on the donor, and on the + recipient the engine's closest concept, policyengine-uk's HBAI net income + before housing costs and before negative incomes are reset to zero + (microcosm#931). The two are close but not the same: the recipient's also + counts HBAI's in-kind items, such as free school meals and the free TV + licence's value, and the Warm Home Discount, and how WAS treats pension + contributions, maintenance paid and negative incomes is not verified. + The intercept is unpenalised, so the model's weighted mean probability on the donor equals the donor's weighted ownership share. A recipient aged 18 or over holds when a uniform keyed on the person id falls below their probability, so adding or diff --git a/experiments/901-uk-main-rebase-receipts.md b/experiments/901-uk-main-rebase-receipts.md index 3f995ce2c9..19d8ec7a33 100644 --- a/experiments/901-uk-main-rebase-receipts.md +++ b/experiments/901-uk-main-rebase-receipts.md @@ -131,7 +131,13 @@ closed on 2026-09-28 by restoring the tool's behaviour on the driver. 4. `candidate_f100_does_not_call_any_sampler`: covered by `test_uk_national_sampling.py::test_full_fraction_is_a_structural_no_op` and `::test_source_sampling_cannot_be_reapplied_as_pool_sampling`; the compact national sampler - is not reachable from the graph's population node. + is not reachable from the graph's population node. (Note of 2026-11-08: microcosm#931 + retired the compact national sampler, `sample_uk_national_frame`, which no build or tool + had called since 8adc836b9, together with the first test. The second test is in + `test_uk_full_build_cli.py` and checks only that the config refuses a second sampling + fraction. `uk.full.sample`'s f100 branch is now pinned by + `test_uk_national_sampling.py::test_the_sample_node_calls_no_sampler_at_f100`, which runs + the node at fraction 1.0 with every sampler patched to fail and finds the spine unsampled.) 5. `candidate_engine_surface_reuses_one_resolver`: covered by `test_uk_full_measure.py::test_full_measure_reuses_one_resolver` (direct port). 6. `candidate_engine_surface_resolves_real_per_clone_blocks`: covered by diff --git a/packages/microcosm-build/src/microcosm/build/atomic_support_conditioning.py b/packages/microcosm-build/src/microcosm/build/atomic_support_conditioning.py index b84c9e1f18..147a5d63e0 100644 --- a/packages/microcosm-build/src/microcosm/build/atomic_support_conditioning.py +++ b/packages/microcosm-build/src/microcosm/build/atomic_support_conditioning.py @@ -15,7 +15,10 @@ - :func:`extend_atomic_support` adds declared columns to a support's bytes; - :func:`zero_mass_cells` lists every constraint cell, at each fallback level, where a weight column has no sampling mass, so a draw can choose its - fallback ahead of time rather than fail on an empty cell. + fallback ahead of time rather than fail on an empty cell; +- :func:`choose_fallback_levels` makes that choice for each household: the + first level of a declared ladder (:class:`FallbackLevel`) whose cell has + mass in the weight the household samples by there. """ from __future__ import annotations @@ -306,3 +309,242 @@ def zero_mass_cells( empty.append(list(cell)) table.append({"constraints": list(level), "weight": weight, "cells": empty}) return table + + +def _name(value: object) -> bool: + return isinstance(value, str) and bool(value) and value.strip() == value + + +@dataclass(frozen=True) +class FallbackLevel: + """One level of a conditioned draw's fallback ladder. + + ``constraints`` names the household inputs the level keeps as exact-match + constraints, each a key of :func:`choose_fallback_levels`' + ``constraint_columns``. ``weight_by`` says whether the level samples by + the weight column chosen for each household (``True``) or by the default + weight (``False``). A draw at the level reads every input it does not keep + as missing. + """ + + name: str + constraints: tuple[str, ...] + weight_by: bool + + def __post_init__(self) -> None: + _require(_name(self.name), "a fallback level needs a name") + _require( + type(self.constraints) is tuple + and all(_name(column) for column in self.constraints) + and len(set(self.constraints)) == len(self.constraints), + "a fallback level keeps a tuple of distinct input columns", + ) + _require(type(self.weight_by) is bool, "a fallback level's weight_by is a bool") + + +def _observed(series: pd.Series) -> tuple[np.ndarray, np.ndarray]: + """Where a household input is present, and the input as Python objects.""" + present = ~series.isna().to_numpy(dtype=bool) + codes = series.to_numpy(dtype=object) + _require( + all(_name(code) for code in codes[present]), + "present input codes must be nonempty strings", + ) + return present, codes + + +def _cell_mass( + support: AtomicSupport, columns: Sequence[str], weights: Sequence[str] +) -> tuple[pd.MultiIndex | None, np.ndarray]: + """The support's cells over ``columns``, each with its summed weights. + + Row ``i`` of the mass table is cell ``i``'s total in each of ``weights``, + in order. Without columns the whole support is one cell (``None``). + Totals stay exact: the decoder holds each weight column's total within + the exact sampling range. + """ + values = np.stack( + [np.asarray(support.arrays[weight], dtype=np.int64) for weight in weights], + axis=1, + ) + if not columns: + return None, values.sum(axis=0, keepdims=True) + codes, cells = pd.MultiIndex.from_arrays( + [support.arrays[column] for column in columns] + ).factorize() + mass = np.zeros((len(cells), len(weights)), dtype=np.int64) + np.add.at(mass, codes, values) + return cells, mass + + +def choose_fallback_levels( + support: AtomicSupport, + households: pd.DataFrame, + *, + constraint_columns: Mapping[str, str], + weight_by: Mapping[str, object] | None, + default_weight: str, + levels: Sequence[FallbackLevel], +) -> np.ndarray: + """Each household's first fallback level whose support cell has mass. + + The shared draw (:func:`~microcosm.build.atomic_geography.assign_atomic`) + refuses a household whose constraint cell has no sampling mass in the + weight column it samples by. A country that conditions its draw declares + a fallback ladder, ``levels``, in order. A household takes the first level + whose inputs are present (every constraint the level keeps and, where the + level samples by the household's own weight, a code ``weight_by`` maps) + and whose cell, the support's areas matching the household on every + constraint the level keeps, has positive summed weight in the column the + household samples by there. A household no level takes is refused; the + refusal names no count, household or value. + + ``constraint_columns`` maps each household input a level may keep to the + support code column it must match. ``weight_by`` is ``None`` or + ``{"input": column, "map": {code: weight column}}``: a level whose + ``weight_by`` is true samples by the column ``map`` names for the + household's ``input`` code. Every other level samples by + ``default_weight``, the draw's ``weight_by`` default or its one + ``weight``. The rule's input may not also be a key of + ``constraint_columns``: the draw samples by a present code's weight + whatever the level, so a level could not keep that input without + sampling by the code's weight, nor drop it and keep the household's own + weight. A present input must be a nonempty string without surrounding + whitespace, as the draw requires. A draw that keeps a household's level's + inputs, reads every other input as missing and samples each stage by this + rule then meets no empty cell. + + Vectorised: one grouped mass table per level, read for the households no + earlier level took. Returns each household's level name, in row order. + """ + _require( + isinstance(support, AtomicSupport), "support must be a decoded atomic support" + ) + _require(isinstance(households, pd.DataFrame), "households must be a DataFrame") + columns = support.metadata["columns"] + + def is_weight(name: object) -> bool: + return _name(name) and columns.get(name, {}).get("kind") == "weight" + + _require( + isinstance(constraint_columns, Mapping) + and all( + _name(column) + and _name(matched) + and matched != "area" + and columns.get(matched, {}).get("kind") == "code" + for column, matched in constraint_columns.items() + ) + and len(set(constraint_columns.values())) == len(constraint_columns), + "constraint_columns must map inputs to distinct support code columns " + "other than the area", + ) + _require( + is_weight(default_weight), "default_weight must be a support weight column" + ) + _require( + weight_by is None + or ( + isinstance(weight_by, Mapping) + and set(weight_by) == {"input", "map"} + and _name(weight_by["input"]) + and isinstance(weight_by["map"], Mapping) + and bool(weight_by["map"]) + and all( + _name(code) and is_weight(weight) + for code, weight in weight_by["map"].items() + ) + ), + "weight_by must map codes of one input to support weight columns", + ) + _require( + weight_by is None or weight_by["input"] not in constraint_columns, + "the weight_by input must not also be a constraint input", + ) + _require( + isinstance(levels, Sequence) + and bool(levels) + and all(isinstance(level, FallbackLevel) for level in levels), + "levels must be a nonempty sequence of FallbackLevel", + ) + levels = tuple(levels) + _require( + len({level.name for level in levels}) == len(levels), + "fallback level names must be distinct", + ) + _require( + all(set(level.constraints) <= set(constraint_columns) for level in levels), + "a fallback level keeps an input constraint_columns does not map", + ) + weighted = any(level.weight_by for level in levels) + _require( + weight_by is not None or not weighted, + "a fallback level samples by weight_by, but no weight_by is given", + ) + inputs = {column for level in levels for column in level.constraints} + if weighted: + inputs.add(weight_by["input"]) + missing = sorted(inputs - set(households.columns)) + _require(not missing, f"households lack input columns {missing}") + _require( + not households.columns[households.columns.isin(sorted(inputs))] + .duplicated() + .any(), + "households repeat an input column", + ) + observed = {column: _observed(households[column]) for column in sorted(inputs)} + + # Each household's own weight, as a position in the rule's weight columns + # (-1 where its code is missing or unmapped). + rows = len(households) + rule_weights: tuple[str, ...] = () + own_weight = np.full(rows, -1, dtype=np.int64) + if weighted: + rule = weight_by["map"] + rule_weights = tuple(dict.fromkeys(rule.values())) + present, codes = observed[weight_by["input"]] + mapped = pd.Index(list(rule)).get_indexer(codes[present]) + column_of_code = np.asarray( + [rule_weights.index(rule[code]) for code in rule], dtype=np.int64 + ) + holders = np.flatnonzero(present) + own_weight[holders[mapped >= 0]] = column_of_code[mapped[mapped >= 0]] + + chosen = np.full(rows, -1, dtype=np.int64) + for position, level in enumerate(levels): + open_rows = chosen < 0 + for column in level.constraints: + open_rows &= observed[column][0] + if level.weight_by: + open_rows &= own_weight >= 0 + candidates = np.flatnonzero(open_rows) + if not len(candidates): + continue + cells, mass = _cell_mass( + support, + [constraint_columns[column] for column in level.constraints], + rule_weights if level.weight_by else (default_weight,), + ) + if cells is None: + cell = np.zeros(len(candidates), dtype=np.intp) + else: + cell = cells.get_indexer( + pd.MultiIndex.from_arrays( + [observed[column][1][candidates] for column in level.constraints] + ) + ) + weight = ( + own_weight[candidates] + if level.weight_by + else np.zeros(len(candidates), dtype=np.int64) + ) + found = cell >= 0 + has_mass = np.zeros(len(candidates), dtype=bool) + has_mass[found] = mass[cell[found], weight[found]] > 0 + chosen[candidates[has_mass]] = position + _require( + bool(np.all(chosen >= 0)), + "a household has no fallback level whose inputs are present and whose " + "cell has sampling mass", + ) + return np.asarray([level.name for level in levels], dtype="U")[chosen] diff --git a/packages/microcosm-build/src/microcosm/build/atomic_support_expectations.py b/packages/microcosm-build/src/microcosm/build/atomic_support_expectations.py new file mode 100644 index 0000000000..6a89749d4c --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/atomic_support_expectations.py @@ -0,0 +1,283 @@ +"""The exact expected value of an area code's value under a conditioned draw cell. + +A conditioned draw (:func:`microcosm.build.atomic_geography.assign_atomic`) +draws each household's atomic area from its cell, the support's areas that +match the household on every constraint input it has, with probability +proportional to the weight column the household samples by. A stage that +later prices something by the area drawn, such as a local tax level, may need +that value's expectation over the household's own cell, exactly. Where the +expectation is not zero, the ratio of the drawn area's value to it averages +one when each area is drawn with probability its weight over the cell's mass, +which the draw realises on the 2**53 grid of its uniforms (microcosm#931). +This module computes it, and names no country's sources or columns: + +- :func:`draw_cell_expectations` gives each household the mean of + ``values``, keyed by a support code column, over its cell, weighted by the + column the household samples by, as an exact + :class:`fractions.Fraction`. + +It takes :func:`~microcosm.build.atomic_support_conditioning.choose_fallback_levels`' +arguments but the ladder, with the household inputs already at each +household's level, and the values to average. Refusals name declared columns +only, never a count, a household, a code or a value. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from fractions import Fraction + +import numpy as np +import pandas as pd + +from microcosm.build.atomic_geography import AtomicSupport + +__all__ = ["draw_cell_expectations"] + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise ValueError("Atomic support expectations: " + message) + + +def _name(value: object) -> bool: + return isinstance(value, str) and bool(value) and value.strip() == value + + +def _exact(value: object) -> bool: + """An int that is not a bool, or a Fraction: never a float.""" + return type(value) is int or isinstance(value, Fraction) + + +def _observed(series: pd.Series) -> tuple[np.ndarray, np.ndarray]: + """Where a household input is present, and the input as Python objects.""" + present = ~series.isna().to_numpy(dtype=bool) + codes = series.to_numpy(dtype=object) + _require( + all(_name(code) for code in codes[present]), + "present input codes must be nonempty strings", + ) + return present, codes + + +def _cell_code_mass( + support: AtomicSupport, + columns: Sequence[str], + value_column: str, + weights: Sequence[str], +) -> tuple[pd.MultiIndex | None, np.ndarray, list[str], np.ndarray]: + """The support's (cell, code) groups, each with its summed weights. + + A cell is a combination of ``columns``' codes (without columns the whole + support is one cell, ``None``); a group is a cell's areas that share a + ``value_column`` code. Groups are sorted by cell, then code, so cell + ``i``'s groups are rows ``starts[i]`` to ``starts[i + 1]`` of the codes + and of the mass table, whose row ``g`` is group ``g``'s total in each of + ``weights``, in order. Totals stay exact: the decoder holds each weight + column's total within the exact sampling range. + """ + amounts = np.stack( + [np.asarray(support.arrays[weight], dtype=np.int64) for weight in weights], + axis=1, + ) + code_of_row, codes = pd.factorize(np.asarray(support.arrays[value_column])) + if columns: + cell_of_row, cells = pd.MultiIndex.from_arrays( + [support.arrays[column] for column in columns] + ).factorize() + cell_count = len(cells) + else: + cell_of_row = np.zeros(len(code_of_row), dtype=np.int64) + cells, cell_count = None, 1 + pair = cell_of_row.astype(np.int64) * len(codes) + code_of_row.astype(np.int64) + pairs, group_of_row = np.unique(pair, return_inverse=True) + mass = np.zeros((len(pairs), len(weights)), dtype=np.int64) + np.add.at(mass, group_of_row, amounts) + starts = np.searchsorted( + pairs, np.arange(cell_count + 1, dtype=np.int64) * len(codes) + ) + group_codes = [str(code) for code in np.asarray(codes)[pairs % len(codes)]] + return cells, starts, group_codes, mass + + +def draw_cell_expectations( + support: AtomicSupport, + households: pd.DataFrame, + *, + constraint_columns: Mapping[str, str], + weight_by: Mapping[str, object] | None, + default_weight: str, + value_column: str, + values: Mapping[str, int | Fraction], +) -> list[Fraction]: + """Each household's exact expected value under its draw cell, in row order. + + ``households`` holds each household's inputs at the level it is drawn + at, every input its level drops already missing, so the shared draw of + the frame draws each household at its level. The inputs are read as the + draw reads them: each input a key of ``constraint_columns`` names is, + where present, an exact-match constraint on the support code column it + maps to and, where missing, skipped, as the draw skips an input declared + ``required: false``. The household samples by the weight column + ``weight_by["map"]`` names for its ``weight_by["input"]`` code where that + code is present, and by ``default_weight`` where it is missing or + ``weight_by`` is ``None``. The arguments follow + :func:`~microcosm.build.atomic_support_conditioning.choose_fallback_levels`: + ``weight_by`` is ``None`` or ``{"input": column, "map": {code: weight + column}}``, the draw's rule without its default, and its input may not + also be a key of ``constraint_columns``. + + The household's cell is the support's areas that match it on every + constraint it has, and its expectation is + ``sum(w[a] * values[support[value_column][a]]) / sum(w[a])`` over the + cell, ``w`` its weight column: the mean the drawn area's value takes + when each area is drawn with probability ``w[a] / sum(w)``, the law of a + draw of one stage at the atomic area, or of stages that all sample by + that column. The draw realises that law on the 2**53 grid of its + uniforms: its integer inverse CDF places each stage's probabilities + within 2**-53 of their exact values. The expectation is exact: the + integer weights are summed by cell and ``value_column`` code, each sum + is multiplied by its code's int or :class:`~fractions.Fraction` value, + and the total is divided by the cell's mass once, so the result is a + ``Fraction``. A code without mass in the cell is not read. Vectorised: + one grouped mass table per set of constraints present, and one + expectation per distinct cell and weight column. + + Refused, with texts that name declared columns only, never a count, a + household, a code or a value: ``constraint_columns``, ``weight_by`` or + ``default_weight`` as the chooser refuses them; a ``value_column`` that + is not a support code column; ``values`` that are not ints or Fractions + (a float or a bool among them) keyed by codes; a frame that lacks or + repeats an input column; a present input that is not a nonempty string + without surrounding whitespace, as the draw requires; a present + ``weight_by`` code the rule does not map, which the draw also refuses; a + household whose cell has no support row, or no mass in the weight it + samples by; and a code with mass in a household's cell that has no + value. + """ + _require( + isinstance(support, AtomicSupport), "support must be a decoded atomic support" + ) + _require(isinstance(households, pd.DataFrame), "households must be a DataFrame") + columns = support.metadata["columns"] + + def kind(name: object) -> object: + return columns.get(name, {}).get("kind") if _name(name) else None + + _require( + isinstance(constraint_columns, Mapping) + and all( + _name(column) and matched != "area" and kind(matched) == "code" + for column, matched in constraint_columns.items() + ) + and len(set(constraint_columns.values())) == len(constraint_columns), + "constraint_columns must map inputs to distinct support code columns " + "other than the area", + ) + _require( + kind(default_weight) == "weight", + "default_weight must be a support weight column", + ) + _require( + weight_by is None + or ( + isinstance(weight_by, Mapping) + and set(weight_by) == {"input", "map"} + and _name(weight_by["input"]) + and isinstance(weight_by["map"], Mapping) + and bool(weight_by["map"]) + and all( + _name(code) and kind(weight) == "weight" + for code, weight in weight_by["map"].items() + ) + ), + "weight_by must map codes of one input to support weight columns", + ) + _require( + weight_by is None or weight_by["input"] not in constraint_columns, + "the weight_by input must not also be a constraint input", + ) + _require(kind(value_column) == "code", "value_column must be a support code column") + _require( + isinstance(values, Mapping) + and all(_name(code) and _exact(value) for code, value in values.items()), + "values must map codes to ints or Fractions", + ) + inputs = tuple(constraint_columns) + read = (*inputs, weight_by["input"]) if weight_by is not None else inputs + missing = sorted(set(read) - set(households.columns)) + _require(not missing, f"households lack input columns {missing}") + _require( + not households.columns[households.columns.isin(list(read))].duplicated().any(), + "households repeat an input column", + ) + observed = {column: _observed(households[column]) for column in read} + + # Each household's weight column, as a position in ``weights``. + rows = len(households) + weights: tuple[str, ...] = (default_weight,) + weight_of_row = np.zeros(rows, dtype=np.int64) + if weight_by is not None: + rule = weight_by["map"] + weights = tuple(dict.fromkeys((default_weight, *rule.values()))) + present, codes = observed[weight_by["input"]] + mapped = pd.Index(list(rule)).get_indexer(codes[present]) + _require( + bool(np.all(mapped >= 0)), + "a present weight_by code has no declared weight column", + ) + column_of_code = np.asarray( + [weights.index(rule[code]) for code in rule], dtype=np.int64 + ) + weight_of_row[present] = column_of_code[mapped] + + # The constraints each household has, as a bit per input. + pattern = np.zeros(rows, dtype=np.int64) + for bit, column in enumerate(inputs): + pattern |= observed[column][0].astype(np.int64) << bit + + result = np.empty(rows, dtype=object) + for kept_bits in np.unique(pattern).tolist(): + kept = [column for bit, column in enumerate(inputs) if kept_bits >> bit & 1] + members = np.flatnonzero(pattern == kept_bits) + cells, starts, group_codes, mass = _cell_code_mass( + support, + [constraint_columns[column] for column in kept], + value_column, + weights, + ) + if cells is None: + cell = np.zeros(len(members), dtype=np.int64) + else: + cell = cells.get_indexer( + pd.MultiIndex.from_arrays( + [observed[column][1][members] for column in kept] + ) + ).astype(np.int64) + _require(bool(np.all(cell >= 0)), "a household's cell has no support row") + keys, group_of_member = np.unique( + cell * len(weights) + weight_of_row[members], return_inverse=True + ) + expectations = [] + for key in keys.tolist(): + at, weight = divmod(key, len(weights)) + first, end = int(starts[at]), int(starts[at + 1]) + amounts = mass[first:end, weight].tolist() + total = sum(amounts) + _require( + total > 0, + "a household's cell has no mass in the weight it samples by", + ) + numerator: int | Fraction = 0 + for code, amount in zip(group_codes[first:end], amounts, strict=True): + if amount: + _require( + code in values, + "a code with mass in a household's cell has no value", + ) + numerator += amount * values[code] + expectations.append(Fraction(numerator, total)) + table = np.empty(len(expectations), dtype=object) + table[:] = expectations + result[members] = table[group_of_member] + return result.tolist() diff --git a/packages/microcosm-build/src/microcosm/build/bounded_amounts.py b/packages/microcosm-build/src/microcosm/build/bounded_amounts.py new file mode 100644 index 0000000000..4f81aa2f3d --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/bounded_amounts.py @@ -0,0 +1,208 @@ +"""Whether amounts stay within a bound each record carries. + +A check on a finished frame may need to know whether an amount never exceeds +a bound held on the same record, such as a reduction within the bill it +reduces, where the amount sits on the record itself or on its members, summed +to it through their membership column (microcosm#931). This module answers +that for any tables: the caller names the bound's entity, its id and bound +columns, the amount columns on the same table, and each member table's amount +and membership columns. + +It names no country and imports nothing first-party. Exclusions are the +caller's: a check that leaves records out drops them, and their members, +before calling. Its results are booleans and its texts name tables and +columns only: never an id, a value or a count of rows, so a check that prints +them prints nothing of a record. +""" + +from __future__ import annotations + +import math +from collections.abc import Mapping, Sequence + +import numpy as np +import pandas as pd + +__all__ = ["FINITE_NONNEGATIVE", "amounts_within_bound"] + +#: The result's key for whether every amount and bound read is finite and +#: nonnegative; no amount column may take the name. +FINITE_NONNEGATIVE = "finite_nonnegative" + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise ValueError("Bounded amounts: " + message) + + +def _name(value: object) -> bool: + return isinstance(value, str) and bool(value) and value.strip() == value + + +def _names(value: object) -> bool: + """A sequence of names that is not itself a string.""" + return ( + isinstance(value, Sequence) + and not isinstance(value, str) + and all(_name(name) for name in value) + ) + + +def _table(tables: Mapping[str, pd.DataFrame], entity: str) -> pd.DataFrame: + _require( + isinstance(tables.get(entity), pd.DataFrame), + f"the {entity} table is not a DataFrame", + ) + return tables[entity] + + +def _column(table: pd.DataFrame, entity: str, column: str) -> pd.Series: + _require(column in table.columns, f"the {entity} table has no {column} column") + _require( + not table.columns[table.columns == column].duplicated().any(), + f"the {entity} table repeats its {column} column", + ) + return table[column] + + +def _amounts(table: pd.DataFrame, entity: str, column: str) -> np.ndarray: + """A numeric column as float64, a missing value as NaN.""" + series = _column(table, entity, column) + _require( + getattr(series.dtype, "kind", None) in ("i", "u", "f"), + f"{entity}.{column} is not a numeric column", + ) + return series.to_numpy(dtype=np.float64, na_value=np.nan) + + +def amounts_within_bound( + tables: Mapping[str, pd.DataFrame], + *, + entity: str, + id_column: str, + bound_column: str, + same_entity_columns: Sequence[str], + member_columns: Mapping[str, Sequence[str]], + membership_columns: Mapping[str, str], + absolute_tolerance: float, +) -> dict[str, bool]: + """Per named column, whether every record's amount is within its bound. + + ``tables[entity]`` holds one row per record, its ids in ``id_column`` and + its bound in ``bound_column``. Each of ``same_entity_columns`` is an + amount on the same table. Each of ``member_columns[member]`` is an amount + on ``tables[member]``, each row of which names its record's id in + ``membership_columns[member]``; a record's amount is the sum of its + members' (zero where it has none), summed as float64 in row order. + + Returns one boolean per named column, keyed by the column, in the order + named (the same table's, then each member table's), each whether every + record's amount is at most its bound plus ``absolute_tolerance``; then + :data:`FINITE_NONNEGATIVE`, whether every value read of the bound and of + each named column, member rows included, is finite and nonnegative, so + any NaN, infinite or negative value read fails it. A missing value is + NaN: a NaN amount fails its column, and a NaN bound fails every column. + An amount of plus infinity fails its column unless its bound is plus + infinity too; minus infinity, like any amount at most its bound, passes + it. A table without rows passes. + + Exclusions are the caller's: it drops the records a check leaves out, + and their members, before calling. + + Refused, with texts that name tables and columns only: tables that are + not a mapping, or a table that is not a DataFrame; a name that is not a + nonempty string without surrounding whitespace, or a column list that is + a string; a bound column that is the id column; no named column, a + column named twice or named :data:`FINITE_NONNEGATIVE`, or an amount + that is the bound or id column or its own table's membership column; a + member table that is the record's own, or membership columns not given + for exactly the member tables; a column a table lacks or repeats; a + bound or amount column that is not of an integer or float dtype; ids + that are missing or repeated; a member row whose membership names no + record; and a tolerance that is not a finite nonnegative number. + """ + _require(isinstance(tables, Mapping), "tables must map entities to DataFrames") + _require( + _name(entity) and _name(id_column) and _name(bound_column), + "entity, id_column and bound_column must be names", + ) + _require(id_column != bound_column, "the bound column must not be the id column") + _require(_names(same_entity_columns), "same_entity_columns must list columns") + _require( + isinstance(member_columns, Mapping) + and all( + _name(member) and _names(columns) + for member, columns in member_columns.items() + ), + "member_columns must map member tables to lists of columns", + ) + _require( + entity not in member_columns, + "a member table must not be the record's own table", + ) + _require( + isinstance(membership_columns, Mapping) + and set(membership_columns) == set(member_columns) + and all(_name(column) for column in membership_columns.values()), + "membership_columns must name one column for each member table", + ) + named = [ + *same_entity_columns, + *(column for columns in member_columns.values() for column in columns), + ] + _require(bool(named), "no amount column is named") + _require(len(set(named)) == len(named), "an amount column is named twice") + _require( + FINITE_NONNEGATIVE not in named, + f"no amount column may be named {FINITE_NONNEGATIVE}", + ) + _require( + not {id_column, bound_column} & set(same_entity_columns), + "an amount column is the id or the bound column", + ) + _require( + all( + membership_columns[member] not in columns + for member, columns in member_columns.items() + ), + "an amount column is its table's membership column", + ) + _require( + isinstance(absolute_tolerance, int | float) + and not isinstance(absolute_tolerance, bool) + and math.isfinite(absolute_tolerance) + and absolute_tolerance >= 0, + "absolute_tolerance must be a finite nonnegative number", + ) + + table = _table(tables, entity) + ids = pd.Index(_column(table, entity, id_column).to_numpy()) + _require( + not ids.hasnans and ids.is_unique, + f"{entity}.{id_column} must name each row once", + ) + bound = _amounts(table, entity, bound_column) + limit = bound + float(absolute_tolerance) + read = [bound] + result: dict[str, bool] = {} + for column in same_entity_columns: + amount = _amounts(table, entity, column) + read.append(amount) + result[column] = bool(np.all(amount <= limit)) + for member, columns in member_columns.items(): + rows = _table(tables, member) + membership = _column(rows, member, membership_columns[member]) + position = ids.get_indexer(membership.to_numpy()) + _require( + not membership.isna().any() and bool(np.all(position >= 0)), + f"a {member} row's {membership_columns[member]} names no {entity} row", + ) + for column in columns: + amount = _amounts(rows, member, column) + read.append(amount) + total = np.bincount(position, weights=amount, minlength=len(ids)) + result[column] = bool(np.all(total <= limit)) + result[FINITE_NONNEGATIVE] = all( + bool(np.all(np.isfinite(values) & (values >= 0))) for values in read + ) + return result diff --git a/packages/microcosm-build/src/microcosm/build/clone_expansion.py b/packages/microcosm-build/src/microcosm/build/clone_expansion.py new file mode 100644 index 0000000000..b46e63ac1e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/clone_expansion.py @@ -0,0 +1,368 @@ +"""Clone-major copies of a frame's records, numbered by a country's rule. + +A build can turn a frame into ``K`` clones of every record. A later stage may +treat each clone as a record of its own (a clone that takes an area of its +own, say), while together the clones stand for the record once, each at an +equal share of its weight. This module makes those clones the same way for +every country (microcosm#931). The country declares how a clone's ids are +numbered (:class:`CloneIdRule`) and what the clones carry +(:class:`CloneDeclaration`). The module names no country's columns, tables or +constants: it reads every entity, id column and membership column from the +frame's :class:`~microcosm.frame.EntitySchema`. + +:func:`expand_clones` returns ``K`` clones of every entity table, +clone-major. Clone 0 is the frame's own rows, ids unchanged. Clone ``k`` +repeats them, with each entity's ids minted by the declared rule, each +person's membership the id of its unit's clone ``k``, and every other column +copied unchanged. Every row carries its clone ordinal, and, where the country +declares a column for it, its id before the expansion. Every weighted +entity's weights are split equally, the same float in each of a row's clones, +and the mass log gains one record per weighted entity stating the declared +reason. + +Every refusal names entities, columns, dtypes, weight kinds and declared +constants only, never an id, a weight or another value of a row. The declared +rule's own refusals must keep to counts and declared constants as well. + +The core copies no link table and gives every record the same number of +clones. Those wait for a caller that needs them. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from dataclasses import dataclass, field +from types import MappingProxyType + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame, MassChangeRecord, WeightKind, Weights + +__all__ = [ + "CLONE_MASS_RTOL", + "CloneDeclaration", + "CloneIdRule", + "expand_clones", +] + +#: How closely the clones' total weight must equal the frame's, relative to +#: the frame's: the tolerance of the Frame's own mass-conserving replacement. +CLONE_MASS_RTOL = 1e-9 + + +@dataclass(frozen=True) +class CloneIdRule: + """How a country numbers its clones' ids. + + ``bound`` is the most clones the rule can number: clone ordinals run + ``0 .. bound - 1``, so a frame expands into 1 to ``bound`` clones. + ``mint(ids, k)`` returns clone ``k``'s ids from the originals', for + ``1 <= k < bound``. It must return one integer per id, of the ids' own + dtype, as a function of that id alone, keep the originals' order, and + put every clone ``k`` id above every clone ``k - 1`` id. A mint in the + ids' dtype keeps it at every clone count. ``require_below(ids, what)`` + refuses incoming ids the mint has no room for; ``what`` names the ids in + its message. + """ + + bound: int + mint: Callable[[np.ndarray, int], np.ndarray] + require_below: Callable[[np.ndarray, str], None] + + def __post_init__(self) -> None: + if not _is_count(self.bound) or self.bound < 1: + raise ValueError("A clone-id rule needs an integer bound of at least 1.") + if not callable(self.mint) or not callable(self.require_below): + raise TypeError( + "A clone-id rule's mint and require_below must be callable." + ) + + +@dataclass(frozen=True, eq=False) +class CloneDeclaration: + """What a country's clones carry, declared once and the same at every K. + + - ``ids``: how the clones' ids are numbered. + - ``clone_index_columns``: each entity's column for its rows' clone + ordinal, ``0 .. K - 1`` as int64. Every entity of the frame needs one. + - ``weight_kinds``: the weight kinds the expansion accepts, on every + entity the frame weights. Each of a row's ``K`` clones takes ``1 / K`` + of its weight, and the clones keep each entity's kind. + - ``reason``: the reason each mass record states. It is declared here, + not per call, so it is the same at every clone count. + - ``source_id_columns``: for any entity, a column for each row's id + before the expansion (its clone-0 id), the same in all its clones. + + A declaration compares and hashes by identity, as its mappings are + frozen views that do not hash. + """ + + ids: CloneIdRule + clone_index_columns: Mapping[str, str] + weight_kinds: frozenset[WeightKind] + reason: str + source_id_columns: Mapping[str, str] = field(default_factory=dict) + + def __post_init__(self) -> None: + if not isinstance(self.ids, CloneIdRule): + raise TypeError("A clone declaration's ids must be a CloneIdRule.") + index = _column_names(self.clone_index_columns, "clone-index") + source = _column_names(self.source_id_columns, "source-id") + if not index: + raise ValueError("A clone declaration names a clone-index column.") + columns = [*index.values(), *source.values()] + if len(set(columns)) != len(columns): + raise ValueError("A clone declaration's columns must be distinct.") + kinds = frozenset(self.weight_kinds) + if not kinds or not all(isinstance(kind, WeightKind) for kind in kinds): + raise ValueError("A clone declaration accepts at least one WeightKind.") + if not isinstance(self.reason, str) or not self.reason.strip(): + raise ValueError("A clone declaration states its mass record's reason.") + object.__setattr__(self, "clone_index_columns", MappingProxyType(index)) + object.__setattr__(self, "source_id_columns", MappingProxyType(source)) + object.__setattr__(self, "weight_kinds", kinds) + + +def expand_clones(frame: Frame, *, clones: int, declaration: CloneDeclaration) -> Frame: + """``frame`` as ``clones`` clones of every record, clone-major. + + Each entity table becomes ``clones`` copies of itself, one after another. + Clone 0's rows are the frame's own, with their ids. Clone ``k``'s rows + repeat them: each entity's ids are ``declaration.ids.mint(ids, k)``, and a + person's membership in a unit is that unit's clone ``k`` id. Every other + column is copied unchanged. Each row gains its clone ordinal in its + entity's clone-index column, and its id before the expansion in a + declared source-id column, so a row and its clones share that id. + + Every weighted entity's weights become ``numpy.tile(w / clones, + clones)``: one float per row of the frame, the same in each of its + clones, of the entity's weight kind. The strata repeat once per clone + and the metadata is kept. The mass log gains one record per weighted + entity, in the frame's entity order: the entity's totals before and + after, which agree within :data:`CLONE_MASS_RTOL`, a declared factor of + 1.0, and the declared reason. At ``clones == 1`` the rows, ids and + weights are the frame's own; only the declared columns and the records + are added. + + Refused, naming entities, columns, dtypes, weight kinds and declared + constants only: + + - a clone count that is not an integer in ``[1, bound]``; + - a declaration that names an entity the frame lacks, or gives an entity + of the frame no clone-index column; + - a frame that already carries a declared column or a link table; + - weights, on any entity, of a kind the declaration does not accept; + - an id or membership column that is not of a numpy integer dtype, or a + membership column whose dtype is not its unit ids' dtype; + - incoming ids the rule's ``require_below`` refuses; + - a mint that does not return one integer per id in the ids' own dtype, + does not keep the ids' order, or does not put every clone ``k`` id + above every clone ``k - 1`` id; + - clones whose total weight strays from the frame's beyond + :data:`CLONE_MASS_RTOL`. + """ + + rule = declaration.ids + if not _is_count(clones) or not 1 <= clones <= rule.bound: + raise ValueError( + f"A clone count must be an integer in [1, {rule.bound}], the declared " + "bound." + ) + count = int(clones) + _check_frame(frame, declaration) + schema = frame.schema + ids = {entity: _clone_ids(frame, entity, count, rule) for entity in frame.entities} + weights, records = _split_weights(frame, count, declaration.reason) + ordinals = np.arange(count, dtype=np.int64) + tables: dict[str, pd.DataFrame] = {} + for entity in frame.entities: + table = frame.table(entity) + rows = len(table) + out = table.take(np.tile(np.arange(rows), count)).reset_index(drop=True) + out[schema.entity_id_column(entity)] = np.concatenate(ids[entity]) + if entity == schema.person_entity: + for group in schema.group_entities: + column = schema.membership_column(group) + out[column] = _member_ids(table[column], ids[group]) + source = declaration.source_id_columns.get(entity) + if source is not None: + out[source] = np.tile(ids[entity][0], count) + out[declaration.clone_index_columns[entity]] = np.repeat(ordinals, rows) + tables[entity] = out + return Frame( + tables, + schema, + weights, + pd.concat([frame.strata] * count, ignore_index=True), + mass_log=(*frame.mass_log, *records), + metadata=frame.metadata, + ) + + +def _is_count(value: object) -> bool: + return isinstance(value, int | np.integer) and not isinstance(value, bool) + + +def _column_names(columns: Mapping[str, str], what: str) -> dict[str, str]: + """A declared entity-to-column mapping, each name a non-empty string.""" + + if not isinstance(columns, Mapping): + raise TypeError(f"A clone declaration's {what} columns must be a mapping.") + names = dict(columns) + for entity, column in names.items(): + if not ( + isinstance(entity, str) and entity and isinstance(column, str) and column + ): + raise ValueError( + f"A clone declaration's {what} columns map entity names to " + "column names." + ) + return names + + +def _check_frame(frame: Frame, declaration: CloneDeclaration) -> None: + """Refuse a frame the declaration cannot clone, before anything is minted.""" + + entities = frame.entities + named = {*declaration.clone_index_columns, *declaration.source_id_columns} + unknown = sorted(named.difference(entities)) + if unknown: + raise ValueError( + f"The clone declaration names entities the frame lacks: {unknown}." + ) + unindexed = [e for e in entities if e not in declaration.clone_index_columns] + if unindexed: + raise ValueError( + f"The clone declaration gives {unindexed} no clone-index column." + ) + declared = { + *declaration.clone_index_columns.values(), + *declaration.source_id_columns.values(), + } + carried = set().union(*(frame.table(e).columns for e in entities)) + held = sorted(declared & carried) + if held: + raise ValueError(f"The frame already carries the declared column(s) {held}.") + if frame.links: + raise ValueError("The clone expansion copies no link table; the frame has one.") + for entity in frame.weighted_entities: + kind = frame.weights_for(entity).kind + if kind not in declaration.weight_kinds: + accepted = sorted(k.value for k in declaration.weight_kinds) + raise ValueError( + f"The clone declaration accepts {accepted} weights, not " + f"{kind.value!r} weights on {entity!r}." + ) + schema = frame.schema + person = frame.table(schema.person_entity) + columns = [(frame.table(e), schema.entity_id_column(e)) for e in entities] + columns += [(person, schema.membership_column(g)) for g in schema.group_entities] + for table, column in columns: + dtype = table[column].dtype + if not (isinstance(dtype, np.dtype) and dtype.kind in "iu"): + raise TypeError( + f"The clone expansion mints numpy integer ids; {column!r} holds " + f"{dtype}." + ) + # A membership points at its unit's minted ids, so it must share their + # dtype, or a clone count above one would change it. + for group in schema.group_entities: + membership, unit = schema.membership_column(group), schema.id_column(group) + found, wanted = person[membership].dtype, frame.table(group)[unit].dtype + if found != wanted: + raise TypeError( + f"The clone expansion needs {membership!r} in the dtype of " + f"{unit!r}; it holds {found} against {wanted}." + ) + for entity in entities: + column = schema.entity_id_column(entity) + declaration.ids.require_below( + frame.table(entity)[column].to_numpy(copy=True), f"{column} values" + ) + + +def _clone_ids( + frame: Frame, entity: str, count: int, rule: CloneIdRule +) -> list[np.ndarray]: + """``entity``'s ids in each clone: the frame's, then each clone's minted. + + The frame's ids are distinct (and a group's ascend), so a mint that keeps + their order strictly and puts each clone above the one before gives ids + that are distinct across all clones and ascend in every group table. + """ + + column = frame.schema.entity_id_column(entity) + original = frame.table(entity)[column].to_numpy(copy=True) + original.setflags(write=False) + order = np.argsort(original, kind="stable") + clones = [original] + for k in range(1, count): + minted = np.asarray(rule.mint(original, k)) + # The frame's ids are numpy integers, so a mint in their dtype mints + # integers, and concatenating the clones keeps the ids' dtype. + if minted.shape != original.shape or minted.dtype != original.dtype: + raise ValueError( + f"The clone-id rule must mint one integer for each {entity} id, " + "in the ids' own dtype." + ) + ranked = minted[order] + if (ranked[1:] <= ranked[:-1]).any(): + raise ValueError( + f"The clone-id rule must keep the order of the {entity} ids it copies." + ) + if minted.size and int(minted.min()) <= int(clones[-1].max()): + raise ValueError( + f"The clone-id rule must mint every clone's {entity} ids above " + "the clone before's." + ) + clones.append(minted) + return clones + + +def _member_ids(members: pd.Series, unit_ids: list[np.ndarray]) -> np.ndarray: + """A membership column in every clone: clone ``k``'s members point at + their unit's clone ``k`` id. + + A frame's group ids ascend and are exactly the ids its members name, so + each member's position in its unit's table is exact. + """ + + values = members.to_numpy(copy=True) + positions = np.searchsorted(unit_ids[0], values) + return np.concatenate([values, *(clone[positions] for clone in unit_ids[1:])]) + + +def _equal_shares(values: np.ndarray, count: int) -> np.ndarray: + """Each row's weight split into ``count`` equal shares, clone-major.""" + + return np.tile(values / count, count) + + +def _split_weights( + frame: Frame, count: int, reason: str +) -> tuple[dict[str, Weights], tuple[MassChangeRecord, ...]]: + """Every weighted entity's split weights, and the mass record each leaves, + in the frame's entity order.""" + + weights: dict[str, Weights] = {} + records: list[MassChangeRecord] = [] + for entity in frame.weighted_entities: + before = frame.weights_for(entity) + after = Weights(_equal_shares(before.values, count), before.kind) + if not np.isclose(after.total, before.total, rtol=CLONE_MASS_RTOL, atol=0.0): + raise ValueError( + f"The clones' {entity} weights stray from the frame's total mass " + f"beyond the relative tolerance {CLONE_MASS_RTOL}." + ) + weights[entity] = after + records.append( + MassChangeRecord( + entity=entity, + old_total=before.total, + new_total=after.total, + declared_factor=1.0, + reason=reason, + ) + ) + return weights, tuple(records) diff --git a/packages/microcosm-build/src/microcosm/build/copy_agreement.py b/packages/microcosm-build/src/microcosm/build/copy_agreement.py new file mode 100644 index 0000000000..1ed89bcfaa --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/copy_agreement.py @@ -0,0 +1,130 @@ +"""Whether copied rows still equal the rows they were copied from. + +A copying step makes each copy a whole-row copy of its parent and rewrites +only what it declares; a later step that re-draws a copy's columns gives the +copy values of its own. A check on a finished frame can then ask, column by +column, whether every copy of a layer still carries its immediate parent's +values on the columns that layer must not move. This module answers that for +any table, any id column and any copy layer: the caller says which rows are +the layer's copies and which row each was copied from, as its own copy-id +rules decode them, and names the columns. + +It names no country, reads no frame and imports nothing first-party. Its +results are booleans and its texts name columns only: never an id, a value +or a count of rows, so a check that prints them prints nothing of a record. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from types import MappingProxyType + +import numpy as np +import pandas as pd + +__all__ = ["CopyAgreement", "rows_agree_with_parents"] + + +@dataclass(frozen=True) +class CopyAgreement: + """Whether every checked copy row equals its parent, column by column. + + ``columns`` maps each named column to whether every checked row's value + equals its parent's (missing on both counts as equal; a column the table + lacks reads ``False``). ``parents_present`` says whether every checked + row's parent is a row of the table. ``copies_checked`` says whether any + row was flagged as a copy. ``failures`` carries one count-free text per + failing column or condition. + """ + + columns: Mapping[str, bool] = field(default_factory=dict) + parents_present: bool = True + copies_checked: bool = False + failures: tuple[str, ...] = () + + @property + def passed(self) -> bool: + return not self.failures + + +def _same(child: pd.Series, parent: pd.Series) -> np.ndarray: + """Row by row, whether the two values are equal or both missing.""" + + child = child.reset_index(drop=True) + parent = parent.reset_index(drop=True) + child_missing = child.isna().to_numpy(dtype=bool) + parent_missing = parent.isna().to_numpy(dtype=bool) + present = ~child_missing & ~parent_missing + equal = np.zeros(len(child), dtype=bool) + if present.any(): + # Only rows where both values are present are compared, so no + # missing marker reaches the comparison. + equal[present] = child[present].eq(parent[present]).to_numpy(dtype=bool) + return equal | (child_missing & parent_missing) + + +def rows_agree_with_parents( + table: pd.DataFrame, + *, + id_column: str, + parent_ids: Sequence[object] | np.ndarray, + rows: Sequence[bool] | np.ndarray, + columns: Iterable[str], +) -> CopyAgreement: + """Whether each of ``rows`` equals its parent on each of ``columns``. + + ``parent_ids`` gives, for every row of ``table`` in order, the + ``id_column`` value of the row it was copied from; only the rows + ``rows`` flags are read. Each flagged row is compared with the table's + row whose id is its parent's, column by column. A flagged row whose + parent is not in the table fails the check, as do ids that do not name + each row once and a named column the table lacks. + + Refused, by a fixed text: ``parent_ids`` or ``rows`` that do not give + one entry per row, and an ``id_column`` the table lacks. + """ + + names = tuple(str(column) for column in columns) + flagged = np.asarray(rows, dtype=bool) + parents = np.asarray(parent_ids) + if flagged.shape != (len(table),) or parents.shape != (len(table),): + raise ValueError( + "rows_agree_with_parents reads one copy flag and one parent id per " + "table row." + ) + if id_column not in table.columns: + raise ValueError(f"The table has no {id_column!r} column to find parents by.") + failures: list[str] = [] + agree = dict.fromkeys(names, False) + ids = pd.Index(table[id_column].to_numpy()) + if ids.hasnans or not ids.is_unique: + failures.append(f"{id_column}: the ids do not name each row once.") + return CopyAgreement( + columns=MappingProxyType(agree), + parents_present=False, + copies_checked=bool(flagged.any()), + failures=tuple(failures), + ) + positions = ids.get_indexer(parents[flagged]) + found = positions >= 0 + parents_present = bool(found.all()) + if not parents_present: + failures.append(f"{id_column}: a copy row's parent is not in the table.") + child_positions = np.flatnonzero(flagged)[found] + parent_positions = positions[found] + for column in names: + if column not in table.columns: + failures.append(f"{column}: the table has no such column.") + continue + values = table[column] + same = _same(values.iloc[child_positions], values.iloc[parent_positions]) + agree[column] = bool(same.all()) + if not agree[column]: + failures.append(f"{column}: a copy row differs from its parent.") + return CopyAgreement( + columns=MappingProxyType(agree), + parents_present=parents_present, + copies_checked=bool(flagged.any()), + failures=tuple(failures), + ) diff --git a/packages/microcosm-build/src/microcosm/build/graph_atomic_geography.py b/packages/microcosm-build/src/microcosm/build/graph_atomic_geography.py index 5d42252363..cc631c1795 100644 --- a/packages/microcosm-build/src/microcosm/build/graph_atomic_geography.py +++ b/packages/microcosm-build/src/microcosm/build/graph_atomic_geography.py @@ -44,12 +44,19 @@ class _GeographyKernel(KernelBase): def implementation_hash(self) -> str: + from microcosm.frame import materialize as frame_materialize + from microcosm.graph import store as graph_store + + # load_source_bytes takes a content store, whose frame encoding writes + # through frame_materialize (microcosm#931). return source_hash( type(self), geography, canonical_json, keyed_uniform, load_source_bytes, + graph_store, + frame_materialize, dependencies=_DEPENDENCIES, ) diff --git a/packages/microcosm-build/src/microcosm/build/source_lineage.py b/packages/microcosm-build/src/microcosm/build/source_lineage.py index b3b48265cc..b150893956 100644 --- a/packages/microcosm-build/src/microcosm/build/source_lineage.py +++ b/packages/microcosm-build/src/microcosm/build/source_lineage.py @@ -19,18 +19,22 @@ declared copy column either. - :func:`copy_paths` gives every household its ordered copy path, read from the declared columns, never inferred from ids, weights or values. -- :func:`replica_keys` and :func:`collapse_replicas` keep one training row per +- :func:`replica_keys` and :func:`fold_replicas` keep one training row per true source and path without the named branches, with the rows' weights summed, so a fit trained on the frame sees each survey record once however - many geographic clones carry it. A frame without such replicas keeps its - rows and weights value for value. + many geographic clones carry it. The row kept is the replica that comes + first in the caller's order (its lowest id, say). :class:`ReplicaFold` + also says which row each record trains on, which records' replicas differ + and which records share a key; :func:`collapse_replicas` returns its rows + and weights alone. A frame without such replicas keeps its rows and + weights value for value. """ from __future__ import annotations import math from collections.abc import Iterable, Mapping -from dataclasses import dataclass +from dataclasses import dataclass, field import numpy as np import pandas as pd @@ -41,8 +45,10 @@ __all__ = [ "CopyBranch", "CopyLineage", + "ReplicaFold", "collapse_replicas", "copy_paths", + "fold_replicas", "record_sources", "replica_keys", ] @@ -329,30 +335,71 @@ def replica_keys( ) -def collapse_replicas( +@dataclass(frozen=True, eq=False) +class ReplicaFold: + """A fit's training rows, each record's replicas folded in. + + Every field holds one value per record, in row order: + + - ``kept``: the rows the fit trains on, one per variant of each key; + - ``weights``: on a kept row, the selected weights of its variant summed + exactly; every other weight as it was given; + - ``representative``: the row a selected record's variant trains on, and + -1 off the selection; + - ``varied``: whether a selected record's key holds more than one + variant, so that its replicas did not all fold into one row; + - ``key_group``: a code of a selected record's replica key, shared by + every selected record of that key, and -1 off the selection. + + The fields carry per-record weights, so the representation prints none. + """ + + kept: np.ndarray = field(repr=False) + weights: np.ndarray = field(repr=False) + representative: np.ndarray = field(repr=False) + varied: np.ndarray = field(repr=False) + key_group: np.ndarray = field(repr=False) + + +def fold_replicas( keys: np.ndarray, rows: np.ndarray, weights: np.ndarray, *, inputs: pd.DataFrame | None = None, -) -> tuple[np.ndarray, np.ndarray]: - """Keep the first of each key's ``rows``, carrying the key's summed weight. + order: np.ndarray | None = None, +) -> ReplicaFold: + """Fold each key's selected replicas into one row per variant. - ``rows`` marks the records a fit trains on and ``weights`` their weights, - both over every record. The first selected record of each key, in row - order, stays selected with the selected weights of its key summed - exactly (:func:`math.fsum`, so the sum does not depend on row order); its - replicas leave the selection. Weights off the selection are returned - unchanged. When no two selected records share a key, the selection and - weights are returned as they were, value for value. + ``keys`` gives every record its replica key (:func:`replica_keys`), + ``rows`` marks the records a fit trains on and ``weights`` their + weights, all three over every record. The selected records that share a + key are one record's replicas. ``inputs`` holds, one row per record, what the fit reads: its predictors - and targets, or the columns they derive from. Replicas then fold only - where they agree on every one of those columns. Replicas that differ stay - rows of their own: each distinct variant keeps its first record, with - the summed weight of the replicas that share it, so the fit sees every - variant at its share and nothing is dropped (microcosm#931). A missing - value agrees only with a missing value. + and targets, or the columns they derive from. Replicas fold only where + they agree on every one of those columns, a missing value agreeing only + with a missing value. Replicas that agree on them form one variant of + the record, and each variant folds on its own, so the fit sees every + variant at its share and nothing is dropped (microcosm#931). Without + ``inputs`` a key is one variant. + + Each variant trains on its representative: its selected record with the + least ``order``, ties going to the earlier row, or without ``order`` its + first selected record in row order. ``order`` holds one integer per + record, each record's id say, so that the replica a variant trains on + does not depend on the row order. The representative carries its + variant's selected weights summed exactly (:func:`math.fsum`, so the sum + does not depend on row order either), and the variant's other records + leave the selection. Every other weight is returned unchanged. + + Variety is decided per key. When no two selected records share a key, + the selection and weights come back as they were, value for value, each + selected record its own representative and none varied. When keys + repeat but no two selected records share a variant, the selection and + weights come back as they were too, and every record of a key with more + than one variant is varied: replicas that all differ fold nothing, yet + their record varies. """ keys = np.asarray(keys, dtype=object) @@ -360,17 +407,37 @@ def collapse_replicas( values = np.asarray(weights, dtype=np.float64) if not (len(keys) == len(selected) == len(values)): raise ValueError("Replica keys, rows and weights must align.") - positions = np.flatnonzero(selected) - if inputs is None: - codes, uniques = pd.factorize( - pd.Series(keys[positions], dtype=object), sort=False - ) - groups = len(uniques) - else: + ranks = None + if order is not None: + ranks = np.asarray(order) + # Timedeltas subclass numpy's integers but are not ids, so the kind + # is checked, not the subtype. + if ranks.shape != (len(keys),) or ranks.dtype.kind not in "iu": + raise ValueError( + "Replica order must be one integer per record: " + f"{ranks.size} value(s) for {len(keys)} record(s)." + ) + if inputs is not None: if not isinstance(inputs, pd.DataFrame) or len(inputs) != len(keys): raise ValueError("Replica inputs must be a frame with one row per record.") if not inputs.columns.is_unique: raise ValueError("Replica inputs must name each column once.") + positions = np.flatnonzero(selected) + key_codes, key_uniques = pd.factorize( + pd.Series(keys[positions], dtype=object), sort=False + ) + if (key_codes < 0).any(): + raise ValueError("Replica keys must not be null on a selected record.") + key_group = np.full(len(keys), -1, dtype=np.int64) + key_group[positions] = key_codes + representative = np.full(len(keys), -1, dtype=np.int64) + varied = np.zeros(len(keys), dtype=bool) + if len(key_uniques) == len(positions): + representative[positions] = positions + return ReplicaFold(selected, values, representative, varied, key_group) + if inputs is None: + codes, groups = key_codes, len(key_uniques) + else: table = inputs.iloc[positions].reset_index(drop=True) table.columns = [f"input:{index}" for index in range(table.shape[1])] for column in table.columns: @@ -382,17 +449,52 @@ def collapse_replicas( .ngroup() .to_numpy() ) - groups = int(codes.max()) + 1 if len(codes) else 0 + groups = int(codes.max()) + 1 + # A variant lies inside one key, so a key's variants are counted by the + # key each variant code belongs to. + variant_key = np.empty(groups, dtype=np.int64) + variant_key[codes] = key_codes + variants = np.bincount(variant_key, minlength=len(key_uniques)) + varied[positions] = variants[key_codes] > 1 if groups == len(positions): - return selected, values - first = np.full(groups, -1, dtype=np.int64) - members: list[list[float]] = [[] for _ in range(groups)] - for position, code in zip(positions.tolist(), codes.tolist(), strict=True): - if first[code] < 0: - first[code] = position - members[code].append(float(values[position])) - kept = np.zeros(len(selected), dtype=bool) + representative[positions] = positions + return ReplicaFold(selected, values, representative, varied, key_group) + # Each variant's selected records by least order, then row; the first of + # each run is its representative. Every code from 0 to groups - 1 occurs, + # so the runs come in code order. + if ranks is None: + sequence = np.lexsort((positions, codes)) + else: + sequence = np.lexsort((positions, ranks[positions], codes)) + runs = np.flatnonzero(np.diff(codes[sequence], prepend=-1)) + first = positions[sequence[runs]] + representative[positions] = first[codes] + kept = np.zeros(len(keys), dtype=bool) kept[first] = True summed = values.copy() - summed[first] = [math.fsum(group) for group in members] - return kept, summed + summed[first] = [ + math.fsum(variant) + for variant in np.split(values[positions[sequence]], runs[1:]) + ] + return ReplicaFold(kept, summed, representative, varied, key_group) + + +def collapse_replicas( + keys: np.ndarray, + rows: np.ndarray, + weights: np.ndarray, + *, + inputs: pd.DataFrame | None = None, + order: np.ndarray | None = None, +) -> tuple[np.ndarray, np.ndarray]: + """The training rows and weights of :func:`fold_replicas`, and nothing else. + + Returns ``(kept, weights)`` over every record: each variant's + representative stays selected with its variant's summed weight, and + without ``order`` the representative is its first selected record in row + order. A selection in which no two records share a variant comes back as + it was, value for value. + """ + + fold = fold_replicas(keys, rows, weights, inputs=inputs, order=order) + return fold.kept, fold.weights diff --git a/packages/microcosm-build/src/microcosm/build/source_manifest.py b/packages/microcosm-build/src/microcosm/build/source_manifest.py index 9ea9c9ffea..c4239e1a0f 100644 --- a/packages/microcosm-build/src/microcosm/build/source_manifest.py +++ b/packages/microcosm-build/src/microcosm/build/source_manifest.py @@ -81,6 +81,25 @@ "classify_cgt_band_facts_with_reviewed_fence", "classify_hmrc_income_facts_with_reviewed_fences", "clone_records", + # microcosm#931: a frame's geographic clones, and each household's + # geography key from its source and clone path. + "expand_geographic_clones", + "key_households_by_clone_path", + # microcosm#931: each clone's atomic area, its local authority, BRMA + # and fuel-relief flag, the SPI copies' own areas, and private rents + # read off the drawn area's BRMA. + "assign_atomic_area", + "resolve_local_authority", + "assign_area_brma", + "assign_rfdr_flags", + "assign_spi_copy_areas", + "localise_private_rents", + # microcosm#931: England and Wales main residences read off each + # owner's constituency inside the WAS chain. + "localise_main_residence_value", + # microcosm#931: Great Britain council tax priced at each household's + # drawn local authority, just before the WAS chain. + "localise_council_tax", "clean_nts_travel_tables", "clean_was_lisa_donor", "clean_aps_occupation_table", @@ -156,7 +175,6 @@ "gate_zero_weight_strata", "head_carry", "impute_annual_property_purchases", - "assign_regional_home_values", "draw_uc_claim_at_legacy_closure", "impute_bus_use_band", "impute_frs_property_finance_costs", diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 8121f4a3cb..5dde486723 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -389,8 +389,12 @@ # calibrate's shared option checks move both again; no US solve changes. # Its TANF ledger fix moves the protocol again: the site's recorded salt # becomes the takes_up_tanf_if_eligible its draw always used. - "seed_map": "270cf6498f8a2977993f969d0703be88fb930499dab42741a2111f57ffec4266", - "seed_protocol": "a696ad74bfa5743b63c65499e6f5a41ea913a96ca38a5e3b9882069284bfca47", + # microcosm#931 moves both again: it adds the keyed tie order, the keyed + # uniform for a label, the binomial share band and a strata keyword to + # stochastic_assignment. stable_identity_uniforms, the one function the + # US keyed draws call there, is unchanged, so no US draw changes. + "seed_map": "8b4cf2e4ddaec8494a1ef41ddc0892ac454894315e51bb7d244b160568593a87", + "seed_protocol": "5b870f9fed400cad5d8a138f622de5bb60954cc62f9ae4f451e2e46035e87fa9", "source_manifest": "dc3879886d2a69f20fb982a675d7fb322c0e4017afef6471be54b0ee3cbd2af6", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", "tail": "565f0ceeadab69d3e96a886a00dd0305f7425fa3acde24d12ba48db2b37d5fc0", diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json index b70fa1faa7..170cdf43cd 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json @@ -8271,6 +8271,8 @@ "count", "draw", "flag_column", + "id_remapping", + "salt", "seed" ], "properties": { @@ -8283,9 +8285,15 @@ "draw": { "type": "string" }, + "id_remapping": { + "type": "string" + }, "flag_column": { "type": "string" }, + "salt": { + "type": "string" + }, "seed": { "type": "integer" }, @@ -8331,9 +8339,20 @@ "required": [ "name", "selector", - "maximum_zero_weight_rows", "reason" ], + "oneOf": [ + { + "required": [ + "maximum_zero_weight_rows" + ] + }, + { + "required": [ + "maximum_zero_weight_rows_rule" + ] + } + ], "properties": { "name": { "type": "string" @@ -8344,6 +8363,23 @@ "maximum_zero_weight_rows": { "type": "integer" }, + "maximum_zero_weight_rows_rule": { + "type": "object", + "additionalProperties": false, + "required": [ + "rule", + "sigma" + ], + "properties": { + "rule": { + "const": "binomial_share_band" + }, + "sigma": { + "type": "number", + "exclusiveMinimum": 0 + } + } + }, "selector": { "type": "object", "minProperties": 1, @@ -8377,6 +8413,9 @@ "conservation": { "type": "string" }, + "geographic_clone_stratum": { + "type": "string" + }, "pension_age_share": { "type": "number" }, @@ -9960,14 +9999,765 @@ "n_estimators": { "type": "integer" }, - "seed": { - "type": "integer" + "seed": { + "type": "integer" + }, + "gate_calibration": { + "enum": [ + "in_sample", + "cross_fitted" + ] + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "resource", + "factor", + "owner_rows" + ], + "properties": { + "kind": { + "const": "uprate_to_regional_reference" + }, + "resource": { + "type": "string" + }, + "factor": { + "type": "string" + }, + "denominator": { + "type": "string" + }, + "owner_rows": { + "type": "string" + }, + "skip_empty_or_nonpositive_regions": { + "type": "boolean" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "entity", + "copies", + "flag_column", + "mass_split", + "weight_kind_out", + "conservation", + "id_remapping", + "declared_factor", + "reason" + ], + "properties": { + "kind": { + "const": "clone_records" + }, + "entity": { + "type": "string" + }, + "copies": { + "type": "integer", + "minimum": 2 + }, + "flag_column": { + "type": "string" + }, + "original_flag": { + "type": "boolean" + }, + "clone_flag": { + "type": "boolean" + }, + "mass_split": { + "type": "number" + }, + "weight_kind_out": { + "type": "string" + }, + "conservation": { + "type": "string" + }, + "id_remapping": { + "type": "string" + }, + "declared_factor": { + "type": "number" + }, + "reason": { + "type": "string" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "entity", + "clone_index_columns", + "ordinal_bound", + "id_remapping", + "order", + "weight_split", + "weight_kind_out", + "source_columns", + "conservation", + "declared_factor", + "reason" + ], + "properties": { + "kind": { + "const": "expand_geographic_clones" + }, + "entity": { + "type": "string" + }, + "clone_index_columns": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "string" + } + }, + "ordinal_bound": { + "type": "integer", + "minimum": 1 + }, + "id_remapping": { + "type": "string" + }, + "order": { + "type": "string" + }, + "weight_split": { + "type": "string" + }, + "weight_kind_out": { + "type": "string" + }, + "source_columns": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "conservation": { + "type": "string" + }, + "declared_factor": { + "type": "number" + }, + "reason": { + "type": "string" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "source", + "vintage_resource" + ], + "properties": { + "kind": { + "const": "key_households_by_clone_path" + }, + "source": { + "type": "string" + }, + "vintage_resource": { + "type": "string" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "identity", + "sources", + "constraints", + "weight_by", + "draw_inputs", + "fallback_levels", + "layers", + "outputs", + "stream", + "seed" + ], + "properties": { + "kind": { + "const": "assign_atomic_area" + }, + "identity": { + "type": "string" + }, + "sources": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "string" + } + }, + "constraints": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "input", + "support", + "required" + ], + "properties": { + "input": { + "type": "string" + }, + "support": { + "type": "string" + }, + "required": { + "type": "boolean" + } + } + } + }, + "weight_by": { + "type": "object", + "additionalProperties": false, + "required": [ + "input", + "default", + "bands" + ], + "properties": { + "input": { + "type": "string" + }, + "default": { + "type": "string" + }, + "bands": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "draw_inputs": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": [ + "string", + "integer" + ] + } + } + ] + } + }, + "fallback_levels": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "constraints", + "weight_by" + ], + "properties": { + "name": { + "type": "string" + }, + "constraints": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "weight_by": { + "type": "boolean" + } + } + } + }, + "layers": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "outputs": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "stream": { + "type": "string" + }, + "seed": { + "type": "integer", + "minimum": 0 + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "code_column", + "output", + "names_resource", + "vintage" + ], + "properties": { + "kind": { + "const": "resolve_local_authority" + }, + "code_column": { + "type": "string" + }, + "output": { + "type": "string" + }, + "names_resource": { + "type": "string" + }, + "vintage": { + "type": "string" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "output", + "area_system", + "layer_resource", + "method", + "area_codes", + "count_resource", + "count_table_systems", + "cell_columns", + "assignment_grain", + "collapse_to", + "collapse_method", + "seed", + "salts" + ], + "properties": { + "kind": { + "const": "assign_area_brma" + }, + "output": { + "type": "string" + }, + "area_system": { + "type": "string" + }, + "layer_resource": { + "type": "string" + }, + "method": { + "type": "string" + }, + "area_codes": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "string" + } + }, + "count_resource": { + "type": "string" + }, + "count_table_systems": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "cell_columns": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "assignment_grain": { + "type": "string" + }, + "collapse_to": { + "type": "string" + }, + "collapse_method": { + "type": "string" + }, + "seed": { + "type": "integer", + "minimum": 0 + }, + "salts": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "string" + } + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "layer_resource", + "method", + "area_system", + "area_code", + "roster_sources", + "require_flagged" + ], + "properties": { + "kind": { + "const": "assign_rfdr_flags" + }, + "layer_resource": { + "type": "string" + }, + "method": { + "type": "string" + }, + "area_system": { + "type": "string" + }, + "area_code": { + "type": "string" + }, + "roster_sources": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "string" + } + }, + "require_flagged": { + "type": "boolean" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "copies", + "key", + "area_draw", + "local_authority", + "brma", + "rfdr", + "rewrites" + ], + "properties": { + "kind": { + "const": "assign_spi_copy_areas" + }, + "copies": { + "type": "object", + "additionalProperties": false, + "required": [ + "branch", + "paths_require", + "kinds" + ], + "properties": { + "branch": { + "type": "string" + }, + "paths_require": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "kinds": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "string" + } + } + } + }, + "key": { + "type": "object", + "additionalProperties": false, + "required": [ + "column", + "source", + "vintage_resource" + ], + "properties": { + "column": { + "type": "string" + }, + "source": { + "type": "string" + }, + "vintage_resource": { + "type": "string" + } + } + }, + "area_draw": { + "type": "object", + "additionalProperties": false, + "required": [ + "definition_of", + "level", + "stream", + "seed" + ], + "properties": { + "definition_of": { + "type": "string" + }, + "level": { + "type": "string" + }, + "stream": { + "type": "string" + }, + "seed": { + "type": "integer", + "minimum": 0 + } + } + }, + "local_authority": { + "type": "object", + "additionalProperties": false, + "required": [ + "as_stage" + ], + "properties": { + "as_stage": { + "type": "string" + } + } + }, + "brma": { + "type": "object", + "additionalProperties": false, + "required": [ + "as_stage", + "engine_households", + "engine_simulations", + "count_resource", + "count_table_systems", + "cell_columns", + "assignment_grain", + "collapse_to", + "collapse_method", + "seed", + "salts" + ], + "properties": { + "as_stage": { + "type": "string" + }, + "engine_households": { + "type": "string" + }, + "engine_simulations": { + "type": "string" + }, + "count_resource": { + "type": "string" + }, + "count_table_systems": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "cell_columns": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "assignment_grain": { + "type": "string" + }, + "collapse_to": { + "type": "string" + }, + "collapse_method": { + "type": "string" + }, + "seed": { + "type": "integer", + "minimum": 0 + }, + "salts": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "string" + } + } + } + }, + "rfdr": { + "type": "object", + "additionalProperties": false, + "required": [ + "as_stage", + "require_flagged" + ], + "properties": { + "as_stage": { + "type": "string" + }, + "require_flagged": { + "type": "boolean" + } + } + }, + "rewrites": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "tenure_type", + "excluded_regions", + "identity", + "process", + "stream", + "seed", + "categories", + "reference_channel", + "quantiles_resource", + "quantiles_method", + "counts_resource", + "weeks_in_year", + "scale_band", + "sampled_region_renters", + "method" + ], + "properties": { + "kind": { + "const": "localise_private_rents" + }, + "tenure_type": { + "type": "string" + }, + "excluded_regions": { + "type": "array", + "items": { + "type": "string" + } + }, + "identity": { + "type": "string" + }, + "process": { + "type": "string" + }, + "stream": { + "type": "string" + }, + "seed": { + "type": "integer", + "minimum": 0 + }, + "categories": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "reference_channel": { + "type": "string" + }, + "quantiles_resource": { + "type": "string" + }, + "quantiles_method": { + "type": "string" + }, + "counts_resource": { + "type": "string" + }, + "weeks_in_year": { + "type": "integer", + "minimum": 1 + }, + "scale_band": { + "type": "array", + "minItems": 2, + "maxItems": 2, + "items": { + "type": "number" + } + }, + "sampled_region_renters": { + "type": "integer", + "minimum": 1 }, - "gate_calibration": { - "enum": [ - "in_sample", - "cross_fitted" - ] + "method": { + "type": "string" } } }, @@ -9976,25 +10766,83 @@ "additionalProperties": false, "required": [ "kind", - "resource", + "value_column", + "after_segment", + "rows", + "rank", + "stream", + "knots", + "knots_level", + "constituency_column", + "regional_reference", + "value", "factor", - "owner_rows" + "region_map", + "rank_basis_column", + "kept", + "method" ], "properties": { "kind": { - "const": "uprate_to_regional_reference" + "const": "localise_main_residence_value" }, - "resource": { + "value_column": { + "type": "string" + }, + "after_segment": { + "type": "integer", + "minimum": 1 + }, + "rows": { + "type": "string" + }, + "rank": { + "type": "string" + }, + "stream": { + "type": "array", + "minItems": 4, + "maxItems": 4, + "items": { + "type": [ + "string", + "integer" + ] + } + }, + "knots": { + "type": "string" + }, + "knots_level": { + "type": "string" + }, + "constituency_column": { + "type": "string" + }, + "regional_reference": { + "type": "string" + }, + "value": { "type": "string" }, "factor": { "type": "string" }, - "owner_rows": { + "region_map": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "string" + } + }, + "rank_basis_column": { "type": "string" }, - "skip_empty_or_nonpositive_regions": { - "type": "boolean" + "kept": { + "type": "string" + }, + "method": { + "type": "string" } } }, @@ -10003,52 +10851,144 @@ "additionalProperties": false, "required": [ "kind", - "entity", - "copies", - "flag_column", - "mass_split", - "weight_kind_out", - "conservation", - "id_remapping", - "declared_factor", - "reason" + "method", + "band_d_resources", + "band_d_year", + "authority_column", + "reference", + "draw_stage", + "copy_draw_stage", + "support_sources", + "provenance_column", + "ratio_classes", + "formula_classes", + "formula_needs", + "band_ratios", + "single_adult_column", + "single_adult_discount", + "scaled_columns", + "cap", + "excluded_regions", + "support_channel_rows" ], "properties": { "kind": { - "const": "clone_records" + "const": "localise_council_tax" }, - "entity": { + "method": { "type": "string" }, - "copies": { - "type": "integer", - "minimum": 2 + "band_d_resources": { + "type": "object", + "additionalProperties": false, + "required": [ + "england_wales", + "scotland" + ], + "properties": { + "england_wales": { + "type": "string" + }, + "scotland": { + "type": "string" + } + } }, - "flag_column": { + "band_d_year": { "type": "string" }, - "original_flag": { - "type": "boolean" + "authority_column": { + "type": "string" }, - "clone_flag": { - "type": "boolean" + "reference": { + "type": "string" }, - "mass_split": { - "type": "number" + "draw_stage": { + "type": "string" }, - "weight_kind_out": { + "copy_draw_stage": { "type": "string" }, - "conservation": { + "support_sources": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "string" + } + }, + "provenance_column": { "type": "string" }, - "id_remapping": { + "ratio_classes": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "formula_classes": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "formula_needs": { "type": "string" }, - "declared_factor": { - "type": "number" + "band_ratios": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "string" + } + } }, - "reason": { + "single_adult_column": { + "type": "string" + }, + "single_adult_discount": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "scaled_columns": { + "type": "object", + "additionalProperties": false, + "required": [ + "household", + "person" + ], + "properties": { + "household": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "person": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + } + } + }, + "cap": { + "type": "string" + }, + "excluded_regions": { + "type": "array", + "items": { + "type": "string" + } + }, + "support_channel_rows": { "type": "string" } } @@ -10294,6 +11234,8 @@ "wealth_measure", "investable_wealth_columns", "selection", + "seed", + "salt", "maximum_copy_weight", "copy_rule", "id_remapping", @@ -10385,6 +11327,13 @@ "selection": { "type": "string" }, + "seed": { + "type": "integer" + }, + "salt": { + "type": "string", + "minLength": 1 + }, "maximum_copy_weight": { "type": "number" }, @@ -10428,6 +11377,8 @@ "stratum_lower", "copies", "stratum_rule", + "seed", + "salt", "copy_rule", "id_remapping", "flag_column", @@ -10455,6 +11406,13 @@ "stratum_rule": { "type": "string" }, + "seed": { + "type": "integer" + }, + "salt": { + "type": "string", + "minLength": 1 + }, "copy_rule": { "type": "string" }, @@ -10610,6 +11568,7 @@ "seed", "salt", "clone_index", + "id_remapping", "flag_column", "band_column", "carrier_column", @@ -10689,6 +11648,9 @@ "clone_index": { "type": "integer" }, + "id_remapping": { + "type": "string" + }, "flag_column": { "type": "string" }, @@ -11400,67 +12362,6 @@ } } }, - { - "type": "object", - "additionalProperties": false, - "required": [ - "kind", - "knots", - "knots_level", - "region_map", - "rows", - "rank", - "support_stratum_lower", - "stream", - "value", - "property_wealth", - "rank_basis_column" - ], - "properties": { - "kind": { - "const": "assign_regional_home_values" - }, - "knots": { - "type": "string" - }, - "knots_level": { - "type": "string" - }, - "region_map": { - "type": "object", - "additionalProperties": { - "type": "string" - } - }, - "rows": { - "type": "string" - }, - "rank": { - "type": "string" - }, - "support_stratum_lower": { - "type": "number" - }, - "stream": { - "type": "array", - "items": { - "type": [ - "string", - "integer" - ] - } - }, - "value": { - "type": "string" - }, - "property_wealth": { - "type": "string" - }, - "rank_basis_column": { - "type": "string" - } - } - }, { "type": "object", "additionalProperties": false, @@ -12954,6 +13855,7 @@ "order", "walk", "walk_seed", + "tie_salt", "within_band", "top_band", "maximum_top_band_multiplier", @@ -12991,6 +13893,10 @@ "walk_seed": { "type": "integer" }, + "tie_salt": { + "type": "string", + "minLength": 1 + }, "within_band": { "type": "string" }, diff --git a/packages/microcosm-build/src/microcosm/build/stochastic_assignment.py b/packages/microcosm-build/src/microcosm/build/stochastic_assignment.py index 2d73825ed7..61224eff57 100644 --- a/packages/microcosm-build/src/microcosm/build/stochastic_assignment.py +++ b/packages/microcosm-build/src/microcosm/build/stochastic_assignment.py @@ -3,6 +3,7 @@ from __future__ import annotations import hashlib +import math from collections.abc import Mapping, Sequence from typing import Any @@ -12,9 +13,13 @@ __all__ = [ "assign_binary_from_rate", "assign_binary_with_anchored_residual", + "binomial_share_tolerance", "clipped_normal_from_uniforms", "integer_record_keys", + "keyed_tie_order", + "keyed_tie_uniforms", "sample_categorical_from_counts", + "stable_identity_uniform", "stable_identity_uniforms", "weight_balanced_uniforms", ] @@ -46,6 +51,23 @@ def stable_identity_uniforms( ) +def stable_identity_uniform(key: object, *, seed: int, salt: str) -> float: + """One U[0,1) draw keyed by ``seed:salt:key``, for a label, not a record. + + The draw ``stable_identity_uniforms([key], seed=seed, salt=salt)`` gives, + for a scalar a stage keys on a label of its own (a walk's cell, a band's + bounds) rather than on a record id: it is the same whichever other draws + the stage makes, and in whatever order, where the n-th draw of a shared + Generator belongs to whichever call happens to come n-th (microcosm#931). + ``key`` is one value; a sequence would be formatted whole into the key + string, so it is refused. + """ + + if np.ndim(key) != 0: + raise TypeError("key must be a single value, not a sequence.") + return float(stable_identity_uniforms([key], seed=seed, salt=salt)[0]) + + def weight_balanced_uniforms( strata: Sequence[object] | np.ndarray, draws: Sequence[float] | np.ndarray, @@ -114,17 +136,100 @@ def integer_record_keys( The key string formats each id, so the integer ``1001`` and the float ``1001.0`` draw differently, and a merge that turned an id column into floats would silently redraw every record. A repeated id would give two - records the same uniforms. + records the same uniforms. Repeats are found on a sorted copy, where + equal ids sit side by side, without building the unique ids. """ keys = np.asarray(ids) - if keys.ndim != 1 or not np.issubdtype(keys.dtype, np.integer): + # Timedeltas subclass numpy's integers but are not ids, and repeated NaT + # never compare equal, so the kind is checked, not the subtype. + if keys.ndim != 1 or keys.dtype.kind not in "iu": raise TypeError(f"{label} must be a one-dimensional array of integer ids.") - if np.unique(keys).size != keys.size: + ordered = np.sort(keys) + if (ordered[1:] == ordered[:-1]).any(): raise ValueError(f"{label} must name each record once.") return keys +def keyed_tie_uniforms( + ids: Sequence[object] | np.ndarray, *, seed: int, salt: str +) -> np.ndarray: + """The uniforms :func:`keyed_tie_order` breaks exact ties on, one per id. + + ``stable_identity_uniforms`` of ``ids`` checked by + :func:`integer_record_keys`, so each record's uniform depends on its own + id alone, in whichever pool it is walked. A stage that orders the same + records in several walks draws them once here and sorts on them, as + :func:`keyed_tie_order` does (microcosm#931). + """ + + keys = integer_record_keys(ids, label=salt) + return stable_identity_uniforms(keys, seed=seed, salt=salt) + + +def keyed_tie_order( + primary: Sequence[float] | np.ndarray, + ids: Sequence[object] | np.ndarray, + *, + seed: int, + salt: str, +) -> np.ndarray: + """Positions that sort ``primary`` ascending, exact ties in keyed order. + + The order is ``np.lexsort((ids, u, primary))``, ``u`` the records' + :func:`keyed_tie_uniforms`. Records whose ``primary`` values differ keep + the order those values give them, so on distinct values it is + ``np.lexsort((ids, primary))`` bit for bit; records tied exactly on + ``primary`` take the order of their uniforms, then of their ids. Ties + broken on ids alone put the lowest id first every time, so where the + copies or geographic clones of one household tie, the lowest copy + ordinal always came first (microcosm#931). Each uniform is a function + of its record's id, so the order does not depend on the order of the + rows. Pass a negated key to walk it descending. + + Only the records that share their ``primary`` value with another are + drawn, since no other record's uniform can move it; NaNs tie with each + other, as the sort puts them together. + """ + + values = np.asarray(primary, dtype=np.float64) + keys = integer_record_keys(ids, label=salt) + if values.shape != keys.shape: + raise ValueError("primary and ids must be aligned 1-D arrays.") + order = np.lexsort((keys, values)) + ordered = values[order] + same = (ordered[1:] == ordered[:-1]) | ( + np.isnan(ordered[1:]) & np.isnan(ordered[:-1]) + ) + if not same.any(): + return order + tied = np.zeros(order.size, dtype=bool) + tied[1:] |= same + tied[:-1] |= same + rows = order[tied] + uniforms = np.zeros(order.size, dtype=np.float64) + uniforms[rows] = stable_identity_uniforms(keys[rows], seed=seed, salt=salt) + return np.lexsort((keys, uniforms, values)) + + +def binomial_share_tolerance(*, target: float, rows: int, sigma: float) -> float: + """Binomial band for a share realized over ``rows`` identity-keyed draws. + + ``sigma`` standard errors of a share whose rows are each drawn at + ``target``, ``sigma x sqrt(target x (1 - target) / rows)``, floored at one + row's worth and capped at one; one when there are no rows. Moved here + from the UK stage-health gates, which keep their four-sigma band under + their own name (microcosm#931). + """ + + if rows <= 0: + return 1.0 + return min( + 1.0, + max(1.0 / rows, sigma * math.sqrt(target * (1.0 - target) / rows)), + ) + + def assign_binary_from_rate( draws: Sequence[float] | np.ndarray, rate: float, @@ -140,6 +245,8 @@ def assign_binary_with_anchored_residual( rate: float, anchor: Sequence[bool] | np.ndarray | None = None, population: Sequence[bool] | np.ndarray | None = None, + *, + strata: Sequence[int] | np.ndarray | None = None, ) -> np.ndarray: """Assign a flag while forcing reported-recipient anchors to true. @@ -150,6 +257,22 @@ def assign_binary_with_anchored_residual( rows and rows outside it are never drawn. Anchored overshoot is accepted, an anchor outside the population stays true (reported receipt is a fact), and the residual fills only non-anchored rows inside the population. + + ``strata``, one integer per row, takes the count within each stratum + rather than over every row: each stratum runs the arithmetic above on its + own rows alone, with its own target, the need its own anchors leave and + the threshold its own non-anchored rows fill, so their order does not + matter. A stratum's flags are then the flags its rows take on their own, + whatever the other strata hold, and with every row in one stratum they + are the flags without strata. A caller whose strata are copies of one + population, as geographic clones are (microcosm#931), counts each copy as + the population alone is counted, where one count over every copy rounds + ``int(rate * n_units)`` once over all of them. Without an anchor there is + no count, so strata change nothing, though they are still checked: they + must align with ``draws`` and have an integer dtype, which holds no + missing value. Unlike :func:`weight_balanced_uniforms`' strata, the + labels must be integers: a label a merge left missing would match no + stratum, and its rows would keep their anchors alone. """ draws = np.asarray(draws, dtype=np.float64) @@ -160,11 +283,21 @@ def assign_binary_with_anchored_residual( population = np.asarray(population, dtype=bool) if population.shape != draws.shape: raise ValueError("population and draws must align") + if strata is not None: + strata = _stratum_labels(strata, draws.shape) if anchor is None: return population & (draws < rate) anchor = np.asarray(anchor, dtype=bool) if anchor.shape != draws.shape: raise ValueError("anchor and draws must align") + if strata is not None: + result = anchor.copy() + for stratum in np.unique(strata): + rows = strata == stratum + result[rows] = assign_binary_with_anchored_residual( + draws[rows], rate, anchor[rows], population[rows] + ) + return result result = anchor.copy() target = int(rate * int(population.sum())) remaining_needed = max(0, target - int((anchor & population).sum())) @@ -225,3 +358,23 @@ def _validate_rate(rate: Any) -> float: if not 0.0 <= value <= 1.0: raise ValueError(f"rate must be in [0, 1], got {value}.") return value + + +def _stratum_labels( + strata: Sequence[int] | np.ndarray, shape: tuple[int, ...] +) -> np.ndarray: + """``strata`` as one integer label per row, or a refusal. + + The labels must align with the draws and have an integer dtype, which + holds no missing value: a float holding whole numbers is refused, as + :func:`integer_record_keys` refuses float ids, and so is a float, object + or nullable column holding a missing value. The refusals name no count, + label or row. + """ + + labels = np.asarray(strata) + if labels.shape != shape: + raise ValueError("strata and draws must align") + if labels.dtype.kind not in "iu": + raise TypeError("strata must be integers, one per row, none missing") + return labels diff --git a/packages/microcosm-build/src/microcosm/build/transport/column_kernels.py b/packages/microcosm-build/src/microcosm/build/transport/column_kernels.py index fa22aadea7..49d60f44ff 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/column_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/transport/column_kernels.py @@ -50,6 +50,7 @@ from microcosm.frame import concepts as concepts_module from microcosm.frame import input_closure as closure_module from microcosm.frame import unit_construction as units_module +from microcosm.frame.adapters import axiom_input_surface as input_surface_module from microcosm.frame.concept_mapping import ConceptMapping from microcosm.frame.concepts import derive_take_up_draws from microcosm.frame.input_closure import ( @@ -78,9 +79,10 @@ SeedSource, source_hash, ) +from microcosm.graph.canonical import canonical_json from microcosm.graph.population import dtype_for_token -from . import graph_inputs +from . import artifact_types, graph_inputs from . import receipt_contract as receipt_contract_module from .artifact_types import TARGET_SURFACE_TYPE from .graph_inputs import ( @@ -330,6 +332,9 @@ def implementation_hash(self) -> str: return source_hash( type(self), graph_inputs, + # canonical_document_param checks a document parameter's spelling + # through the encoder (microcosm#931). + canonical_json, *self._modules, dependencies=self.capabilities.dependencies, ) @@ -417,7 +422,9 @@ class ConceptsEncodeKernel(_ColumnKernel): """Encode person and household bindings plus declared closure defaults.""" ref = "concepts.encode@1" - _modules = (mapping_module, concepts_module, closure_module) + # InputClosure computes through the Axiom input surface's records + # (microcosm#931). + _modules = (mapping_module, concepts_module, closure_module, input_surface_module) def run(self, context: KernelContext) -> KernelResult: _refuse_take_up(context, self.ref) @@ -526,7 +533,15 @@ class ConceptsEncodeGroupsKernel(_ColumnKernel): """Group bindings execute through G4's encoder and built memberships.""" ref = "concepts.encode_groups@1" - _modules = (mapping_module, units_module, concepts_module, closure_module) + # InputClosure computes through the Axiom input surface's records + # (microcosm#931). + _modules = ( + mapping_module, + units_module, + concepts_module, + closure_module, + input_surface_module, + ) def run(self, context: KernelContext) -> KernelResult: _refuse_take_up(context, self.ref) @@ -784,6 +799,17 @@ def implementation_hash(self) -> str: hierarchy_module, calibration_predicates_module, receipt_contract_module, + # The kernel checks the surface's type (artifact_types), and + # decode_target_surface checks its factor evidence and Chronicle + # feed identity through the next three, rebuilds its registry's + # targets through calibrate.target and decodes its canonical JSON + # (microcosm#931). + artifact_types, + target_kernels.reference_factors_module, + target_kernels.ledger_artifact_module, + target_kernels.chronicle_epoch_module, + target_kernels.target_module, + canonical_json, dependencies=self.capabilities.dependencies, ) diff --git a/packages/microcosm-build/src/microcosm/build/transport/compile_modules.py b/packages/microcosm-build/src/microcosm/build/transport/compile_modules.py index e38932c372..1e375b5c4b 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/compile_modules.py +++ b/packages/microcosm-build/src/microcosm/build/transport/compile_modules.py @@ -16,10 +16,15 @@ ``EXCLUDED_DIRECT_IMPORTS`` names every first-party (``microcosm.*``) module a listed module imports but the tuple leaves out, with the reason. A test pins that the two together cover every direct import of every listed module, -so a new import must be listed or excluded. The exclusions cover only the -listed modules' own imports; a second test walks every module each compile -kernel hashes, its own module included, and records the imports no review -has classified yet (microcosm#1497). +so a new import must be listed or excluded. + +The exclusions cover only the listed modules' own imports, and keep a module +off the list, not out of a kernel's hash. Each transport kernel also hashes, +beside this tuple, every first-party module its own code computes through: +the derived binding rule in test_transport_binding_rule holds each kernel's +hash to that, in place of the per-kernel import record microcosm#1497 kept +(microcosm#931). This module is not hashed, so an edit to a reason re-keys +nothing; an edit to the tuple re-keys the kernels. """ from __future__ import annotations @@ -42,6 +47,7 @@ "microcosm.calibrate.hierarchy", "microcosm.calibrate.geography_constants", "microcosm.build.transport.reference_selection", + "microcosm.graph.canonical", ) _COUNTRY_SPEC_LOADER = ( @@ -51,9 +57,16 @@ "validators do not use" ) _GRAPH_RUNTIME = ( - "the graph runtime (kernel protocol, declarations and the canonical JSON " - "encoder node keys are computed with); no kernel in the repository " - "hashes a microcosm.graph module" + "the graph runtime the listed modules read a node's context and declare " + "artifact types with: the kernel protocol and the declarations. Both are " + "core modules every node runs through, which the derived binding rule " + "lists rather than binds (test_transport_binding_rule): node_key hashes " + "each node's declaration, its artifact types included, and every input " + "and output passes through microcosm.graph.kernel's records, so binding " + "either would re-key every node on any edit to it, and the transport " + "graph tests, which run composed graphs through run_graph, catch an edit " + "that changes a value. The canonical JSON encoder, which the listed " + "modules encode and digest documents with, is listed (microcosm#931)" ) EXCLUDED_DIRECT_IMPORTS: Mapping[str, str] = MappingProxyType( @@ -91,7 +104,6 @@ "graph_inputs.context_frame's entity schema; target compilation " "reads no population" ), - "microcosm.graph.canonical": _GRAPH_RUNTIME, "microcosm.graph.decl": _GRAPH_RUNTIME, "microcosm.graph.kernel": _GRAPH_RUNTIME, } diff --git a/packages/microcosm-build/src/microcosm/build/transport/evidence_contract.py b/packages/microcosm-build/src/microcosm/build/transport/evidence_contract.py index 49127385dc..8cb9c42a44 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/evidence_contract.py +++ b/packages/microcosm-build/src/microcosm/build/transport/evidence_contract.py @@ -168,6 +168,12 @@ "microcosm.frame.adapters.axiom_input_surface", "microcosm.frame.adapters.axiom", "microcosm.frame.schema", + # The kernel types its engines by frame.rules' RulesEngine, and + # reads an AxiomEngine's compiled programs (engine_root_inputs), + # whose class builds its tables through frame.materialize + # (microcosm#931). + "microcosm.frame.rules", + "microcosm.frame.materialize", _CANONICAL, _LEAF, ), @@ -179,6 +185,9 @@ "microcosm.calibrate.registry", "microcosm.calibrate.solve", "microcosm.calibrate.target", + # The decoded problem's targets carry their hierarchy + # (microcosm#931). + "microcosm.calibrate.hierarchy", "microcosm.frame.schema", "microcosm.frame.weights", _CANONICAL, diff --git a/packages/microcosm-build/src/microcosm/build/transport/gate_kernels.py b/packages/microcosm-build/src/microcosm/build/transport/gate_kernels.py index 212a027900..eaef447907 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/gate_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/transport/gate_kernels.py @@ -136,6 +136,7 @@ binding_identity, evidence_contract, graph_inputs, + takeup_kernels, target_kernels, ) from .artifact_types import ( @@ -795,6 +796,11 @@ def implementation_hash(self) -> str: calibrate_artifacts_module, frame_bundle_module, frame_schema_module, + # The evidence decoders: decode_target_surface checks a surface's + # factor evidence through reference_factors, and + # decode_cell_diagnostics is takeup_kernels' (microcosm#931). + target_kernels.reference_factors_module, + takeup_kernels, dependencies=self.capabilities.dependencies, ) return hashlib.sha256( diff --git a/packages/microcosm-build/src/microcosm/build/transport/geography_kernels.py b/packages/microcosm-build/src/microcosm/build/transport/geography_kernels.py index 79edaa0db4..40c7b7879b 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/geography_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/transport/geography_kernels.py @@ -18,6 +18,7 @@ import numpy as np import microcosm.build.atomic_geography as atomic_geography_module +import microcosm.build.graph_atomic_geography as graph_atomic_geography_module import microcosm.calibrate.hierarchy as hierarchy_module import microcosm.calibrate.registry as registry_module from microcosm.build.atomic_geography import _U53, encode_atomic_support @@ -35,7 +36,7 @@ ) from microcosm.graph.canonical import canonical_json -from . import graph_inputs, target_kernels +from . import artifact_types, graph_inputs, target_kernels from .artifact_types import TARGET_SURFACE_TYPE from .graph_inputs import ( canonical_document_param, @@ -287,6 +288,17 @@ def implementation_hash(self) -> str: registry_module, hierarchy_module, canonical_json, + # The kernel checks the surface's type (artifact_types) and types + # its support (graph_atomic_geography); decode_target_surface + # checks the surface's factor evidence and its Chronicle feed + # identity through the next three and rebuilds its registry's + # targets through calibrate.target (microcosm#931). + artifact_types, + graph_atomic_geography_module, + target_kernels.reference_factors_module, + target_kernels.ledger_artifact_module, + target_kernels.chronicle_epoch_module, + target_kernels.target_module, dependencies=self.capabilities.dependencies, ) diff --git a/packages/microcosm-build/src/microcosm/build/transport/measure_kernels.py b/packages/microcosm-build/src/microcosm/build/transport/measure_kernels.py index f93aac85ee..198e825071 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/measure_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/transport/measure_kernels.py @@ -46,6 +46,7 @@ Numeric, source_hash, ) +from microcosm.graph.canonical import canonical_json from microcosm.graph.population import dtype_for_token from . import graph_inputs @@ -289,6 +290,9 @@ def implementation_hash(self) -> str: type(self), graph_inputs, reference_intervals_module, + # canonical_document_param checks the measure document's spelling + # through the encoder (microcosm#931). + canonical_json, dependencies=self.capabilities.dependencies, ) diff --git a/packages/microcosm-build/src/microcosm/build/transport/money_kernels.py b/packages/microcosm-build/src/microcosm/build/transport/money_kernels.py index 392d67036e..4a4739ef03 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/money_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/transport/money_kernels.py @@ -66,6 +66,7 @@ SeedSource, source_hash, ) +from microcosm.graph.canonical import canonical_json from . import graph_inputs from .graph_inputs import ( @@ -193,6 +194,10 @@ def implementation_hash(self) -> str: chronicle_epoch_module, concepts_module, exact_money_module, + # reference_factors digests and replays the factor documents + # through the encoder, and canonical_document_param checks each + # document parameter's spelling through it (microcosm#931). + canonical_json, dependencies=self.capabilities.dependencies, ) @@ -417,6 +422,9 @@ def implementation_hash(self) -> str: type(self), graph_inputs, exact_money_module, + # canonical_document_param checks the money document's spelling + # through the encoder (microcosm#931). + canonical_json, dependencies=self.capabilities.dependencies, ) @@ -510,6 +518,9 @@ def implementation_hash(self) -> str: type(self), graph_inputs, exact_money_module, + # canonical_document_param checks the money document's spelling + # through the encoder (microcosm#931). + canonical_json, dependencies=self.capabilities.dependencies, ) diff --git a/packages/microcosm-build/src/microcosm/build/transport/population_kernels.py b/packages/microcosm-build/src/microcosm/build/transport/population_kernels.py index f8d24121fd..53af1d9729 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/population_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/transport/population_kernels.py @@ -61,6 +61,7 @@ StructuralDelta, source_hash, ) +from microcosm.graph.canonical import canonical_json from microcosm.graph.population import dtype_for_token from . import graph_inputs, target_kernels, unit_declarations @@ -375,6 +376,9 @@ def implementation_hash(self) -> str: transport_module, concepts_module, scaling_module, + # canonical_document_param checks the currency document's + # spelling through the encoder (microcosm#931). + canonical_json, dependencies=self.capabilities.dependencies, ) diff --git a/packages/microcosm-build/src/microcosm/build/transport/reporting_kernels.py b/packages/microcosm-build/src/microcosm/build/transport/reporting_kernels.py index 46d1431b9e..9616401e6f 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/reporting_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/transport/reporting_kernels.py @@ -30,7 +30,7 @@ from microcosm.graph.canonical import canonical_json from microcosm.graph.population import dtype_for_token -from . import graph_inputs +from . import artifact_types, graph_inputs from .artifact_types import INCOME_CELLS_TYPE from .graph_inputs import ( canonical_document_param, @@ -185,6 +185,12 @@ def implementation_hash(self) -> str: sys.modules[__name__], reporting, graph_inputs, + # The shared contract check names the income cells' type + # (artifact_types); the classifier document's spelling is checked + # through the encoder, which the income deciles also encode their + # cells with (microcosm#931). + artifact_types, + canonical_json, dependencies=self.capabilities.dependencies, ) diff --git a/packages/microcosm-build/src/microcosm/build/transport/takeup_kernels.py b/packages/microcosm-build/src/microcosm/build/transport/takeup_kernels.py index b1d110a293..147327bc46 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/takeup_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/transport/takeup_kernels.py @@ -43,7 +43,7 @@ ) from microcosm.graph.canonical import canonical_json -from . import graph_inputs, target_kernels, weight_quality +from . import artifact_types, graph_inputs, target_kernels, weight_quality from .artifact_types import CELL_DIAGNOSTICS_TYPE from .graph_inputs import ( canonical_document_param, @@ -882,7 +882,13 @@ class TakeupBandsKernel(KernelBase): def implementation_hash(self) -> str: return source_hash( - type(self), graph_inputs, dependencies=self.capabilities.dependencies + type(self), + graph_inputs, + # The kernel decodes its scenario tables and encodes its bands as + # canonical JSON, and checks its document parameters' spelling + # through the encoder (microcosm#931). + canonical_json, + dependencies=self.capabilities.dependencies, ) def run(self, context: KernelContext) -> KernelResult: @@ -1001,7 +1007,16 @@ class TakeupCellDiagnosticsKernel(KernelBase): ) def implementation_hash(self) -> str: - return source_hash(type(self), decode_cell_diagnostics, graph_inputs) + return source_hash( + type(self), + decode_cell_diagnostics, + graph_inputs, + # The kernel types its output (artifact_types), and decodes its + # scenario tables and encodes its output as canonical JSON + # (microcosm#931). + artifact_types, + canonical_json, + ) def run(self, context: KernelContext) -> KernelResult: require_params(context, self.ref, required=frozenset({"artifact_scenarios"})) diff --git a/packages/microcosm-build/src/microcosm/build/transport/target_kernels.py b/packages/microcosm-build/src/microcosm/build/transport/target_kernels.py index cdea94844e..002618f5c3 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/target_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/transport/target_kernels.py @@ -50,6 +50,7 @@ import microcosm.build.calibration_columns as calibration_columns_module import microcosm.build.calibration_predicates as calibration_predicates_module +import microcosm.build.chronicle_epoch as chronicle_epoch_module import microcosm.build.ledger_artifact as ledger_artifact_module import microcosm.build.reference_factors as reference_factors_module import microcosm.build.reference_intervals as reference_intervals_module @@ -61,6 +62,7 @@ import microcosm.calibrate.registry as registry_module import microcosm.calibrate.target as target_module import microcosm.frame.bundle as frame_bundle_module +import microcosm.frame.concepts as frame_concepts_module import microcosm.frame.schema as frame_schema_module import microcosm.frame.unit_construction as units_module import microcosm.frame.weights as frame_weights_module @@ -601,6 +603,12 @@ def implementation_hash(self) -> str: reference_intervals_module, reference_factors_module, reference_profiles_module, + # validate_reference_resolution checks the factor evidence's + # Chronicle feed identity through these two, and the bindings and + # columns are digested as canonical JSON (microcosm#931). + ledger_artifact_module, + chronicle_epoch_module, + canonical_json, dependencies=self.capabilities.dependencies, ) @@ -963,6 +971,15 @@ def implementation_hash(self) -> str: reference_profiles_module, reference_selection_module, *_MATRIX_MODULES, + # decode_target_surface checks the surface's factor evidence and + # its Chronicle feed identity through these three; the interval + # diagnostics count source households by frame.concepts' column + # name and are digested as canonical JSON (microcosm#931). + reference_factors_module, + ledger_artifact_module, + chronicle_epoch_module, + frame_concepts_module, + canonical_json, dependencies=self.capabilities.dependencies, ) diff --git a/packages/microcosm-build/src/microcosm/build/transport/terminal_kernels.py b/packages/microcosm-build/src/microcosm/build/transport/terminal_kernels.py index 76b0c51f97..6267781158 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/terminal_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/transport/terminal_kernels.py @@ -43,6 +43,7 @@ import pandas as pd import microcosm.build.artifact_files as artifact_files_module +import microcosm.build.trace as trace_module import microcosm.calibrate._target_loss_attribution as loss_attribution_module import microcosm.calibrate.artifacts as calibrate_artifacts_module import microcosm.calibrate.diagnostics as calibrate_diagnostics_module @@ -231,6 +232,17 @@ def implementation_hash(self) -> str: frame_bundle_module, frame_schema_module, frame_weights_module, + # decode_target_surface checks the surface's factor evidence and + # its Chronicle feed identity through the first three; the + # surface's targets carry their hierarchy, the rebuilt result's + # constraint matrix checks its monetary binding, and the + # diagnostics are encoded as canonical JSON (microcosm#931). + target_kernels.reference_factors_module, + target_kernels.ledger_artifact_module, + target_kernels.chronicle_epoch_module, + target_kernels.hierarchy_module, + target_kernels.monetary_binding_module, + canonical_json, dependencies=self.capabilities.dependencies, ) @@ -559,6 +571,13 @@ def implementation_hash(self) -> str: frame_bundle_module, frame_schema_module, frame_weights_module, + # require_terminal_gate_reports decodes the gate reports into the + # manifest, phase report and result records these three define, + # and the descriptor is digested as canonical JSON (microcosm#931). + gate_kernels.country_spec_module, + gate_kernels.gate_battery_module, + gate_kernels.gates_module, + canonical_json, dependencies=self.capabilities.dependencies, ) @@ -642,6 +661,11 @@ def implementation_hash(self) -> str: artifact_files_module, materialize_module, axiom_adapter_module, + # file_artifact digests the exported file through + # trace.sha256_file, and the report is encoded as canonical JSON + # (microcosm#931). + trace_module, + canonical_json, dependencies=self.capabilities.dependencies, ) @@ -689,6 +713,13 @@ def implementation_hash(self) -> str: graph_inputs, artifact_types, gate_kernels, + # require_terminal_gate_reports decodes the gate reports into the + # manifest, phase report and result records these three define, + # and the receipt is digested as canonical JSON (microcosm#931). + gate_kernels.country_spec_module, + gate_kernels.gate_battery_module, + gate_kernels.gates_module, + canonical_json, dependencies=self.capabilities.dependencies, ) diff --git a/packages/microcosm-build/src/microcosm/build/uk/calibration_measure_exclusions.json b/packages/microcosm-build/src/microcosm/build/uk/calibration_measure_exclusions.json index 3f2139ce0d..041a83f783 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/calibration_measure_exclusions.json +++ b/packages/microcosm-build/src/microcosm/build/uk/calibration_measure_exclusions.json @@ -1368,6 +1368,24 @@ "adjudication": "María's go of 2026-11-12 on B10 of the November event-data plan's PR3 (microcosm#1095): P4 stays a stated limitation; measure Social Security Scotland's ADP and Carer Support Payment caseloads for Scotland, averaged over July to December 2025, on every build and hold them out of the solve (María's signature of 2026-11-12).", "approved_on": "2026-11-12", "expires_on": "2027-02-12" + }, + { + "name": "ons.land.corporate_land_value", + "reason": "Diagnostic by design, not a weight question: policyengine-uk's corporate_land_value gives each household aggregate_corporate_land_value times its share of the weighted corporate_sector_wealth (corporate and private pension wealth), so a released file's weighted total equals that parameter whatever the weights (policyengine-uk's test_corporate_land_value.py), and the row is met or missed by the parameter alone. The solve fits the column the engine measured at the spine's starting weights, so, bound, the row only pins the weighted mass of the corporate and private pension wealth the was_wealth stage draws, which no other row targets, and says nothing about land. Its fact, ONS's preliminary end-2025 national balance sheet E43B, non-financial corporations' land (£2,063,534m), is also not the engine's parameter, which policyengine-uk describes as the land of non-financial and financial corporations. Measured on every evaluation; workstream 2 of microcosm#1521 settles the parameter's source.", + "tracking": "microcosm#1521", + "approved_by": "juaristi22", + "adjudication": "María's ruling of 2026-11-06 on the #931 move (microcosm#931), in line with workstream 2 of microcosm#1521 and the first must item of Vahid's review there (record the corporate row as met by construction once the engine's parameter comes from the target's ONS series, and keep it out of the weight question): hold the corporate and total-economy land rows as measured diagnostics, keep ons.land.household_land_value bound and correct its note (E44N is the households sector alone, not households and NPISH). A calibrate-only rerun on the move's spine is to attribute the change.", + "approved_on": "2026-11-06", + "expires_on": "2027-02-06" + }, + { + "name": "ons.land.land_value", + "reason": "Diagnostic by design, not a weight question: ONS's preliminary end-2025 total-economy land (E44R, £6,872,455m) also counts the land of financial corporations, general government and NPISH, which neither the household row (E44N, households alone, £4,485,171m) nor the corporate row (E43B, non-financial corporations, £2,063,534m) covers. policyengine-uk's land_value is household_land_value plus corporate_land_value, the columns those two rows read, so, bound beside them, the three rows could not all be met: E44R less E44N less E43B is £323,750m, 4.7 percent of the total, which the solve could close only by raising property-weighted and corporate-wealth-weighted mass. Beside the bound household row, the rest of this row is the corporate column, which pins corporate-wealth mass rather than land (ons.land.corporate_land_value). The household row stays bound. Measured on every evaluation; workstream 2 of microcosm#1521 decides whether the engine gets those sectors' land as aggregates.", + "tracking": "microcosm#1521", + "approved_by": "juaristi22", + "adjudication": "María's ruling of 2026-11-06 on the #931 move (microcosm#931), in line with workstream 2 of microcosm#1521 and the first must item of Vahid's review there (record the corporate row as met by construction once the engine's parameter comes from the target's ONS series, and keep it out of the weight question): hold the corporate and total-economy land rows as measured diagnostics, keep ons.land.household_land_value bound and correct its note (E44N is the households sector alone, not households and NPISH). A calibrate-only rerun on the move's spine is to attribute the change.", + "approved_on": "2026-11-06", + "expires_on": "2027-02-06" } ] } diff --git a/packages/microcosm-build/src/microcosm/build/uk/council_tax_band_d_england_wales_2024_25.json b/packages/microcosm-build/src/microcosm/build/uk/council_tax_band_d_england_wales_2024_25.json new file mode 100644 index 0000000000..0a262b214f --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk/council_tax_band_d_england_wales_2024_25.json @@ -0,0 +1,423 @@ +{ + "band_d": { + "E06000001": "2380.35", + "E06000002": "2370.01", + "E06000003": "2309.04", + "E06000004": "2352.85", + "E06000005": "2258.86", + "E06000006": "2136.80", + "E06000007": "2171.10", + "E06000008": "2224.68", + "E06000009": "2277.00", + "E06000010": "2078.73", + "E06000011": "2230.39", + "E06000012": "2276.19", + "E06000013": "2132.84", + "E06000014": "2053.12", + "E06000015": "2107.35", + "E06000016": "2292.51", + "E06000017": "2543.29", + "E06000018": "2529.69", + "E06000019": "2329.51", + "E06000020": "2037.49", + "E06000021": "1978.59", + "E06000022": "2155.39", + "E06000023": "2460.32", + "E06000024": "2165.36", + "E06000025": "2309.89", + "E06000026": "2214.87", + "E06000027": "2242.85", + "E06000030": "2198.04", + "E06000031": "2082.65", + "E06000032": "2210.00", + "E06000033": "2050.00", + "E06000034": "2040.66", + "E06000035": "2108.35", + "E06000036": "2051.30", + "E06000037": "2263.62", + "E06000038": "2367.47", + "E06000039": "2186.63", + "E06000040": "1683.78", + "E06000041": "2262.93", + "E06000042": "2150.46", + "E06000043": "2339.02", + "E06000044": "2075.45", + "E06000045": "2156.99", + "E06000046": "2373.57", + "E06000047": "2431.19", + "E06000049": "2217.19", + "E06000050": "2277.08", + "E06000051": "2203.66", + "E06000052": "2342.54", + "E06000053": "1857.35", + "E06000054": "2330.28", + "E06000055": "2237.26", + "E06000056": "2275.55", + "E06000057": "2355.31", + "E06000058": "2157.20", + "E06000059": "2503.63", + "E06000060": "2286.13", + "E06000061": "2191.34", + "E06000062": "2250.31", + "E06000063": "2283.83", + "E06000064": "2277.47", + "E06000065": "2282.95", + "E06000066": "2267.02", + "E07000008": "2248.95", + "E07000009": "2264.91", + "E07000010": "2331.10", + "E07000011": "2321.51", + "E07000012": "2303.96", + "E07000032": "2191.88", + "E07000033": "2320.84", + "E07000034": "2129.91", + "E07000035": "2224.06", + "E07000036": "2155.12", + "E07000037": "2168.97", + "E07000038": "2246.16", + "E07000039": "2128.54", + "E07000040": "2348.19", + "E07000041": "2270.22", + "E07000042": "2404.17", + "E07000043": "2394.39", + "E07000044": "2372.19", + "E07000045": "2394.70", + "E07000046": "2362.95", + "E07000047": "2457.05", + "E07000061": "2416.45", + "E07000062": "2437.47", + "E07000063": "2503.48", + "E07000064": "2439.42", + "E07000065": "2484.03", + "E07000066": "2159.44", + "E07000067": "2109.51", + "E07000068": "2083.46", + "E07000069": "2142.54", + "E07000070": "2122.14", + "E07000071": "2107.30", + "E07000072": "2100.67", + "E07000073": "2140.47", + "E07000074": "2144.73", + "E07000075": "2175.36", + "E07000076": "2094.28", + "E07000077": "2148.96", + "E07000078": "2156.86", + "E07000079": "2169.77", + "E07000080": "2220.50", + "E07000081": "2145.30", + "E07000082": "2260.22", + "E07000083": "2126.12", + "E07000084": "2045.90", + "E07000085": "2123.67", + "E07000086": "2117.95", + "E07000087": "2063.40", + "E07000088": "2132.91", + "E07000089": "2174.44", + "E07000090": "2110.36", + "E07000091": "2193.24", + "E07000092": "2110.28", + "E07000093": "2089.08", + "E07000094": "2142.63", + "E07000095": "2094.99", + "E07000096": "2187.99", + "E07000098": "2179.19", + "E07000099": "2225.95", + "E07000102": "2199.87", + "E07000103": "2236.30", + "E07000105": "2202.18", + "E07000106": "2214.97", + "E07000107": "2174.67", + "E07000108": "2253.49", + "E07000109": "2205.28", + "E07000110": "2292.93", + "E07000111": "2306.69", + "E07000112": "2326.82", + "E07000113": "2196.40", + "E07000114": "2276.45", + "E07000115": "2266.15", + "E07000116": "2237.66", + "E07000117": "2347.12", + "E07000118": "2233.81", + "E07000119": "2269.96", + "E07000120": "2270.47", + "E07000121": "2297.19", + "E07000122": "2415.07", + "E07000123": "2365.48", + "E07000124": "2196.90", + "E07000125": "2313.21", + "E07000126": "2242.03", + "E07000127": "2253.03", + "E07000128": "2253.37", + "E07000129": "2292.51", + "E07000130": "2219.31", + "E07000131": "2212.48", + "E07000132": "2194.09", + "E07000133": "2241.28", + "E07000134": "2230.65", + "E07000135": "2223.50", + "E07000136": "2165.35", + "E07000137": "2128.67", + "E07000138": "2190.87", + "E07000139": "2186.65", + "E07000140": "2135.40", + "E07000141": "2115.01", + "E07000142": "2208.59", + "E07000143": "2219.32", + "E07000144": "2224.97", + "E07000145": "2200.98", + "E07000146": "2215.69", + "E07000147": "2231.98", + "E07000148": "2285.23", + "E07000149": "2257.63", + "E07000170": "2399.81", + "E07000171": "2423.75", + "E07000172": "2401.09", + "E07000173": "2393.93", + "E07000174": "2385.74", + "E07000175": "2467.51", + "E07000176": "2416.58", + "E07000177": "2351.01", + "E07000178": "2441.91", + "E07000179": "2356.55", + "E07000180": "2341.57", + "E07000181": "2320.36", + "E07000192": "2178.08", + "E07000193": "2151.39", + "E07000194": "2155.20", + "E07000195": "2143.59", + "E07000196": "2115.31", + "E07000197": "2107.72", + "E07000198": "2135.00", + "E07000199": "2107.68", + "E07000200": "2132.39", + "E07000202": "2254.05", + "E07000203": "2111.44", + "E07000207": "2333.65", + "E07000208": "2308.34", + "E07000209": "2319.25", + "E07000210": "2297.25", + "E07000211": "2349.48", + "E07000212": "2272.62", + "E07000213": "2304.56", + "E07000214": "2356.11", + "E07000215": "2358.64", + "E07000216": "2371.10", + "E07000217": "2371.60", + "E07000218": "2319.41", + "E07000219": "2289.49", + "E07000220": "2269.87", + "E07000221": "2263.85", + "E07000222": "2247.66", + "E07000223": "2321.98", + "E07000224": "2261.92", + "E07000225": "2237.64", + "E07000226": "2199.96", + "E07000227": "2212.91", + "E07000228": "2245.92", + "E07000229": "2235.52", + "E07000234": "2195.85", + "E07000235": "2180.73", + "E07000236": "2185.50", + "E07000237": "2131.62", + "E07000238": "2100.65", + "E07000239": "2214.57", + "E07000240": "2198.55", + "E07000241": "2224.88", + "E07000242": "2225.25", + "E07000243": "2176.01", + "E07000244": "2117.61", + "E07000245": "2139.61", + "E08000001": "2147.47", + "E08000002": "2288.80", + "E08000003": "2068.25", + "E08000004": "2331.35", + "E08000005": "2330.24", + "E08000006": "2324.46", + "E08000007": "2346.64", + "E08000008": "2190.93", + "E08000009": "1972.61", + "E08000010": "1923.63", + "E08000011": "2255.55", + "E08000012": "2420.39", + "E08000013": "2174.60", + "E08000014": "2338.68", + "E08000015": "2263.44", + "E08000016": "2134.47", + "E08000017": "2024.11", + "E08000018": "2210.88", + "E08000019": "2273.25", + "E08000021": "2293.59", + "E08000022": "2220.47", + "E08000023": "2197.13", + "E08000024": "1996.60", + "E08000025": "2090.96", + "E08000026": "2295.65", + "E08000027": "1937.46", + "E08000028": "2027.83", + "E08000029": "1984.23", + "E08000030": "2375.30", + "E08000031": "2295.02", + "E08000032": "2055.35", + "E08000033": "2190.39", + "E08000034": "2207.66", + "E08000035": "2066.22", + "E08000036": "2079.80", + "E08000037": "2450.95", + "E09000001": "1217.89", + "E09000002": "2002.75", + "E09000003": "1943.24", + "E09000004": "2155.04", + "E09000005": "2036.05", + "E09000006": "1949.71", + "E09000007": "2010.87", + "E09000008": "2366.91", + "E09000009": "1948.34", + "E09000010": "2065.48", + "E09000011": "1920.53", + "E09000012": "1877.37", + "E09000013": "1386.77", + "E09000014": "2107.32", + "E09000015": "2286.32", + "E09000016": "2207.92", + "E09000017": "1863.91", + "E09000018": "1991.01", + "E09000019": "1920.81", + "E09000020": "1529.85", + "E09000021": "2375.35", + "E09000022": "1865.41", + "E09000023": "2037.98", + "E09000024": "1999.28", + "E09000025": "1724.34", + "E09000026": "2089.87", + "E09000027": "2263.66", + "E09000028": "1792.98", + "E09000029": "2166.18", + "E09000030": "1675.51", + "E09000031": "2173.72", + "E09000032": "968.76", + "E09000033": "975.02", + "W06000001": "1982.089951", + "W06000002": "2160.650013", + "W06000003": "2135.860011", + "W06000004": "2090.10996", + "W06000005": "2062.459993", + "W06000006": "2013.94994", + "W06000008": "2103.89", + "W06000009": "1885.429986", + "W06000010": "2048.030009", + "W06000011": "2018.140652", + "W06000012": "2281.350035", + "W06000013": "2244.549991", + "W06000014": "1973.132616", + "W06000015": "1827.149992", + "W06000016": "2082.080005", + "W06000018": "1813.000002", + "W06000019": "2302.99004", + "W06000020": "1951.87267", + "W06000021": "2110.669981", + "W06000022": "1855.630005", + "W06000023": "2053.830021", + "W06000024": "2329.409986" + }, + "built_on": "2026-11-13", + "description": "Band D council tax for 2024-25 of every billing authority in England and Wales, keyed by April 2023 local authority code: England from MHCLG's Table 8 (area council tax for a dwelling occupied by two adults, at pence), Wales from StatsWales (the total council tax of the billing authority area, unrounded as published). Scotland's 2024-25 Band D is vendored from the pinned Chronicle feed into its own resource; Northern Ireland has domestic rates. Rebuilt by tools/build_uk_council_tax_band_d.py (microcosm#931).", + "fiscal_year": "2024-25", + "kind": "uk_council_tax_band_d_england_wales", + "licence": { + "attribution": "Contains public sector information licensed under the Open Government Licence v3.0. Sources: Ministry of Housing, Communities and Local Government, Council Tax levels set by local authorities in England 2024 to 2025, Table 8; Welsh Government, StatsWales, Composition of average band D council tax by billing authority.", + "licence": "Open Government Licence v3.0" + }, + "local_authority_vintage": "2023_april_lad", + "method": "uk-council-tax-band-d-v1", + "receipts": { + "authorities": { + "england": 296, + "wales": 22 + }, + "england_values_quantised_to_pence": { + "exact_binary": 14, + "tolerance": "1e-9", + "within_tolerance": 282 + }, + "wales_excluded_names": [ + "Total Unitary Authorities" + ] + }, + "resource": "council_tax_band_d_england_wales_2024_25.json", + "schema_version": 1, + "sources": { + "england": { + "basis": "Area council tax for a Band D dwelling occupied by two adults, in pounds, adult social care and parish precepts included (the table's note a). Each amount is stored as a binary float and kept as the pence decimal it lies within 1e-9 of.", + "chronicle_manifest": { + "commit": "80a3482905843d447f8811b500a56dbf3b5f47fa", + "path": "db/data/mhclg/council_tax_levels_england_2024_25_table_8/manifest.yaml", + "repository": "PolicyEngine/chronicle" + }, + "chronicle_package_id": "mhclg-council-tax-levels-england-2024-25-table-8", + "chronicle_source_id": "mhclg", + "chronicle_year": 2024, + "filename": "Table_8_24-25.ods", + "publisher": "Ministry of Housing, Communities and Local Government", + "registered_by": { + "issue": "PolicyEngine/chronicle#412", + "merge_commit": "c51f40d2bdaa66315dd63f334ede5790a8891e27", + "pull_request": "PolicyEngine/chronicle#413" + }, + "registration_kind": "raw_only", + "selector": { + "code_column": "ONS Code", + "note_column": "Notes", + "sheet": "Table_8", + "value_column": "Band D" + }, + "sha256": "249127ae1bff3f37bb335896f801524562de0dd9668c6c9cd64eacd8a8880245", + "size_bytes": 53280, + "table": "Council Tax levels set by local authorities in England 2024 to 2025, Table 8: Area council tax for a dwelling occupied by 2 adults by band 2024-25", + "url": "https://assets.publishing.service.gov.uk/media/65fad6739316f5001164c3ca/Table_8_24-25.ods" + }, + "wales": { + "basis": "Average band D council tax of the billing authority area (the county council element with community councils, plus the police element), in pounds, unrounded as published. Each authority is joined to its April 2023 code by its exact name on the local authority roster.", + "chronicle_manifest": { + "commit": "80a3482905843d447f8811b500a56dbf3b5f47fa", + "path": "db/data/welshgov/band_d_council_tax_composition_1993_94_to_2026_27/manifest.yaml", + "repository": "PolicyEngine/chronicle" + }, + "chronicle_package_id": "welshgov-band-d-council-tax-composition-1993-94-to-2026-27", + "chronicle_source_id": "welshgov", + "chronicle_year": 2026, + "filename": "statswales_band_d_council_tax_composition_by_billing_authority.csv", + "publisher": "Welsh Government (StatsWales)", + "registered_by": { + "issue": "PolicyEngine/chronicle#412", + "merge_commit": "c51f40d2bdaa66315dd63f334ede5790a8891e27", + "pull_request": "PolicyEngine/chronicle#413" + }, + "registration_kind": "raw_only", + "selector": { + "columns": [ + "Data values", + "Data description", + "Row", + "Year", + "Authority", + "Notes" + ], + "excluded_names": [ + "Total Unitary Authorities" + ], + "name_column": "Authority", + "note_column": "Notes", + "rows": { + "Data description": "\u00a3 per band D", + "Row": "Total CT for billing authority area", + "Year": "2024-25" + }, + "value_column": "Data values" + }, + "sha256": "b6ea9848bd0fdf4a6310b62a62d614fd4c325b8f8d2e83a4d89eb85886ce552e", + "size_bytes": 590478, + "table": "Composition of average band D council tax by billing authority (StatsWales)", + "url": "https://api.stats.gov.wales/v1/3ca971eb-7d9b-4b79-89fa-6d7f77431aad/download/csv" + } + } +} diff --git a/packages/microcosm-build/src/microcosm/build/uk/country_package.json b/packages/microcosm-build/src/microcosm/build/uk/country_package.json index c0835ebaeb..51274769fc 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/country_package.json +++ b/packages/microcosm-build/src/microcosm/build/uk/country_package.json @@ -122,6 +122,11 @@ "kind": "legacy_json", "schema_id": "legacy_json" }, + { + "path": "scotgov_band_d_council_tax_2024_25.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, { "path": "dwp_move_to_uc_facts.json", "kind": "legacy_json", @@ -327,6 +332,11 @@ "kind": "legacy_json", "schema_id": "legacy_json" }, + { + "path": "council_tax_band_d_england_wales_2024_25.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, { "path": "price_paid_property_support_bounds.json", "kind": "legacy_json", diff --git a/packages/microcosm-build/src/microcosm/build/uk/gates.json b/packages/microcosm-build/src/microcosm/build/uk/gates.json index 723e4de542..174a117609 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/gates.json +++ b/packages/microcosm-build/src/microcosm/build/uk/gates.json @@ -93,12 +93,15 @@ "stocks_and_shares_isa", "student_loan_balance" ], + "exempt_columns": [ + "main_residence_value", + "property_wealth" + ], "max_clipped_low_rows_by_column": { "cash_isa": 0, "consumer_debt": 0, "corporate_wealth": 0, "gross_financial_wealth": 0, - "main_residence_value": 0, "mortgage_debt": 0, "net_financial_wealth": 0, "non_residential_property_value": 0, @@ -106,7 +109,6 @@ "other_residential_property_value": 0, "owned_land": 0, "private_pension_wealth": 0, - "property_wealth": 0, "savings": 0, "stocks_and_shares_isa": 0, "student_loan_balance": 0 @@ -116,7 +118,6 @@ "consumer_debt": 0, "corporate_wealth": 0, "gross_financial_wealth": 0, - "main_residence_value": 0, "mortgage_debt": 0, "net_financial_wealth": 0, "non_residential_property_value": 0, @@ -124,13 +125,12 @@ "other_residential_property_value": 0, "owned_land": 0, "private_pension_wealth": 0, - "property_wealth": 0, "savings": 0, "stocks_and_shares_isa": 0, "student_loan_balance": 0 } }, - "notes": "Stage-time support-clip health gate over the WAS wealth receipt; raw FRS mapping stages have no imputation semantics and intentionally have no empty health gate. Allowances pinned at the receipted spine-e baseline (2026-08-26): zero rows clipped on every non-exempt column, so any clipping is a real signal against a clean baseline; a missing allowance fails closed rather than skipping the comparison. private_pension_wealth joined the gated columns with microcosm#1063 (it had been an output without an allowance); the derived totals are clipped like every other column, and a clipped total would also break its identity in the wealth_coherence gate." + "notes": "Support-clip check on the was_wealth receipt. was_wealth clips each column in columns, except those in exempt_columns, to the donors' realized support. The gate holds each clipped column's clipped_low_rows and clipped_high_rows to its allowances in max_clipped_low_rows_by_column and max_clipped_high_rows_by_column, each multiplied by the receipt's clone_count where it gives one. main_residence_value and property_wealth are exempt through exempt_columns, since localised home values can pass the donor range; their receipt entries must be marked exempt." }, { "id": "uk_stage_was_wealth_coherence", @@ -684,11 +684,12 @@ "check": "child_benefit_take_up", "maximum_claim_rate_deviation": 0.02, "maximum_age_claim_rate_deviation": 0.05, + "maximum_age_claim_rate_standard_errors": 3, "minimum_age_child_rows": 200, "maximum_opt_out_share_deviation": 0.01, "minimum_eligible_family_units": 1 }, - "notes": "Stage-time gate over the Child Benefit take-up receipt (microcosm#1063): the share of eligible children claimed for is within 0.02 of HMRC's published claim rates by single year of age (Table 15, May 2025) at the frame's age mix, and within 0.05 at each age with at least 200 child rows whose solved rate was not clipped (clipped ages are reported); the opted-out share of claiming families is within 0.01 of the published opted-out share of registered families (August 2025), or below it only where the charged families are exhausted. The terminal take-up signal gate no longer measures would_claim_child_benefit or child_benefit_opts_out, whose rates are shares of eligible children and of claiming families, not of every benefit unit." + "notes": "Stage-time gate over the Child Benefit take-up receipt (microcosm#1063): the share of eligible children claimed for is within 0.02 of HMRC's published claim rates by single year of age (Table 15, May 2025) at the frame's age mix. At each age with at least 200 child rows whose solved rate was not clipped (clipped ages are reported), the share is within a tolerance of that age's published rate p: the larger of 0.05 and 3 standard errors of a share drawn at p over the age's n child rows, 3 * sqrt(p * (1 - p) / n) (maximum_age_claim_rate_standard_errors; María's ruling of 2026-11-10, microcosm#931). Each age's realised rate carries the sampling noise of the stage's keyed draws, which at an age with few child rows can exceed a fixed 0.05; because the draws are keyed, a failure from that noise would repeat on every rebuild. Three standard errors exceed 0.05 only below 3,600 * p * (1 - p) child rows, so an age with more rows keeps 0.05. The opted-out share of claiming families is within 0.01 of the published opted-out share of registered families (August 2025), or below it only where the charged families are exhausted. The terminal take-up signal gate no longer measures would_claim_child_benefit or child_benefit_opts_out, whose rates are shares of eligible children and of claiming families, not of every benefit unit." }, { "id": "uk_stage_cgt_incidence_clone_mass", @@ -716,7 +717,7 @@ "copies": 6, "maximum_relative_mass_deviation": 1e-09 }, - "notes": "Stage-time gate over the high-value home support receipt (microcosm#1240 fix 1). Total household mass is conserved to the declared deviation; the receipt restates the ruled stratum (owner households at or above the 0.95 weighted rank of main_residence_value within their region) and copy count (6); every region with owner households selects at least one; copies created equal five per selected household. The stage assigns no value and makes no draw, so it holds no distributional threshold." + "notes": "Stage-time gate over the high-value home support receipt (microcosm#1240 fix 1). Total household mass is conserved to the declared deviation; the receipt restates the ruled stratum (owner households at or above the 0.95 weighted rank of main_residence_value within their region) and copy count (6); every region with owner households selects at least one; copies created equal five per selected household. Since microcosm#931 the stratum ranks the values was_wealth reads off each England and Wales owner's constituency, which the regional property uprating returns to their knots, and every copy keeps its parent's value, area and geography key (uk_copy_area_agreement checks the area and the key). The stage assigns no value, and its one draw only orders owners tied on value, so it holds no distributional threshold." }, { "id": "uk_stage_cgt_support_split_mass", @@ -874,15 +875,15 @@ "notes": "Stage-time invariants over the #791 relationship receipt: exactly one HEAD per household agreeing with hrpid and hrpnum, contiguous person numbers, no person with two partners, no unmapped relhrp code, family index and enum domains consistent, and the ten-cell household-type partition covering every household exactly once. The grid reciprocity tolerance is the reviewed 2024-25 tape count; a re-issued tape with a different count refuses for review." }, { - "id": "uk_stage_regional_home_values_ranking", + "id": "uk_stage_area_home_values_ranking", "gate": "stage_health", "phase": "transferred", "criticality": "release_blocking", "evidence_absent_blocks": true, "population_fact_check": true, "parameters": { - "stage": "regional_home_values", - "check": "regional_home_values", + "stage": "was_wealth", + "check": "area_home_values", "regions": [ "E12000001", "E12000002", @@ -895,11 +896,185 @@ "E12000009", "W92000004" ], - "maximum_sources_at_cap": 0, + "maximum_sources_at_cap": 9, "knots_basis_year": 2024, + "knots_level": "constituency", "rank_basis": "weighted_midpoint_within_frs_region" }, - "notes": "Stage-time gate over the regional home-values receipt (microcosm#1308, gated with the next policyengine-uk bump on María's ruling of 2026-10-17): every English region and Wales has rows re-ranked on the regional knots, no more rows are rewritten than the stage carries, no source sits at the value cap, the knots are the 2024 basis and the ranking is the weighted midpoint within each FRS region, and the weighted value stays positive before and after the stage. A synthetic smoke build records a failure without blocking on it (population_fact_check): the fixture holds a few regions, not the population the regional knots describe." + "notes": "Stage-time gate over the WAS stage's area home values block, the regional home values ranking of microcosm#1308 (gated on María's ruling of 2026-10-17) read inside the WAS chain since microcosm#931: every English region and Wales has sources whose main residence was read off their assigned constituency's knots, the knots are the 2024 basis at the constituency level, the ranking is the weighted midpoint within each FRS region, and the sources are counted among geographic clone 0's rows, so the declared maximum holds at any clone count. A weighted mid-rank stays below 1, so a source reads the knots' cap only where its constituency's knots reach the cap below the top grid point and its rank is near the top of its FRS region; the maximum is the top of the receipt's 3-9 band, so the gate fails once the receipt prints 10-29 or more. A region the receipt prints <3 is flagged rather than failed (María's ruling of 2026-10-26). A synthetic smoke build records a failure without blocking on it (population_fact_check): the fixture holds a few regions, not the population the knots describe." + }, + { + "id": "uk_atomic_area_mapping_integrity", + "gate": "stage_health", + "phase": "assembled", + "criticality": "release_blocking", + "evidence_absent_blocks": true, + "parameters": { + "stage": "atomic_area_assign", + "check": "atomic_mapping_integrity", + "validation_scope": "atomic_geography_mapping_integrity", + "support_systems": [ + "uk_ew_output_area_2021", + "uk_scotland_output_area_2022", + "uk_ni_data_zone_2021" + ] + }, + "notes": "Stage-time gate over the spine's area draw (microcosm#931). atomic_area_assign validates every drawn ladder through the shared operator's mapping-integrity check and refuses to finish unless it passes; the gate reads the record the stage keeps: an accepted outcome (pass), the declared scope, and SHA-256 digests of the assignment definition and of each declared support system's conditioned support. Evaluated at the assembled boundary, which closes after area_brma, so a record that fails stops the spine there, before any stage after area_brma runs. Its texts name the field, never a count or a value." + }, + { + "id": "uk_spi_copy_area_mapping_integrity", + "gate": "stage_health", + "phase": "transferred", + "criticality": "release_blocking", + "evidence_absent_blocks": true, + "parameters": { + "stage": "spi_copy_areas", + "check": "atomic_mapping_integrity", + "validation_scope": "atomic_geography_mapping_integrity", + "support_systems": [ + "uk_ew_output_area_2021", + "uk_scotland_output_area_2022", + "uk_ni_data_zone_2021" + ], + "accepted_outcomes": [ + "pass", + "not_run" + ] + }, + "notes": "The mapping-integrity record of spi_copy_areas, which draws the area and ladder of every SPI copy and income band donor with the area draw's definition (microcosm#931). On a frame without a copy neither the draw's validation nor the RFDR layer's roster check runs and the stage records not_run, so the gate accepts pass or not_run. Evaluated at the transferred boundary: spi_copy_areas runs after the assembled boundary closes." + }, + { + "id": "uk_copy_area_agreement", + "gate": "stage_health", + "phase": "transferred", + "criticality": "release_blocking", + "evidence_absent_blocks": true, + "parameters": { + "stage": "cgt_residential_split", + "check": "copy_area_agreement", + "columns_by_copy_layer": { + "spi_support_channel": [ + "region", + "household_clone_index" + ], + "high_value_support": [ + "region", + "household_clone_index", + "geography_household_key", + "atomic_area_code", + "atomic_area_system", + "atomic_area_basis", + "oa_code", + "lsoa_code", + "msoa_code", + "local_authority_code", + "ward_code", + "constituency_code", + "region_code", + "itl3_code", + "itl2_code", + "itl1_code", + "local_authority", + "brma", + "in_rural_fuel_duty_relief_area", + "council_tax", + "council_tax_rebate" + ], + "cgt_support_split": [ + "region", + "household_clone_index", + "geography_household_key", + "atomic_area_code", + "atomic_area_system", + "atomic_area_basis", + "oa_code", + "lsoa_code", + "msoa_code", + "local_authority_code", + "ward_code", + "constituency_code", + "region_code", + "itl3_code", + "itl2_code", + "itl1_code", + "local_authority", + "brma", + "in_rural_fuel_duty_relief_area", + "council_tax", + "council_tax_rebate" + ], + "cgt_incidence_clone": [ + "region", + "household_clone_index", + "geography_household_key", + "atomic_area_code", + "atomic_area_system", + "atomic_area_basis", + "oa_code", + "lsoa_code", + "msoa_code", + "local_authority_code", + "ward_code", + "constituency_code", + "region_code", + "itl3_code", + "itl2_code", + "itl1_code", + "local_authority", + "brma", + "in_rural_fuel_duty_relief_area", + "council_tax", + "council_tax_rebate" + ], + "cgt_residential_split": [ + "region", + "household_clone_index", + "geography_household_key", + "atomic_area_code", + "atomic_area_system", + "atomic_area_basis", + "oa_code", + "lsoa_code", + "msoa_code", + "local_authority_code", + "ward_code", + "constituency_code", + "region_code", + "itl3_code", + "itl2_code", + "itl1_code", + "local_authority", + "brma", + "in_rural_fuel_duty_relief_area", + "council_tax", + "council_tax_rebate" + ] + } + }, + "notes": "Every copy the spine makes after spi_copy_areas equals its immediate parent, the household its id names with the copy layer's copy-id field cleared (copy_id_table.parent), on the columns spi_copy_areas rewrites (the geography key, the area and ladder, local_authority, brma and the Rural Fuel Duty Relief flag) and on region and household_clone_index (C-11, C-20, microcosm#931): the high-value support, CGT support split, CGT incidence clone and CGT residential split copies. The SPI support channel's copies and income band donors draw areas of their own, so they are held on region and household_clone_index alone. The check reads the transferred frame; it is declared on cgt_residential_split, the last copy layer, whose receipt the gate requires. Its details are booleans by layer and column, and its texts name the layer and column, never an id, a count or a value." + }, + { + "id": "uk_council_tax_reduction_within_bill", + "gate": "stage_health", + "phase": "transferred", + "criticality": "release_blocking", + "evidence_absent_blocks": true, + "parameters": { + "stage": "council_tax_localise", + "check": "council_tax_reduction_within_bill", + "bill_column": "council_tax", + "household_reduction_columns": [ + "council_tax_rebate" + ], + "person_reduction_columns": [ + "council_tax_benefit_reported" + ], + "excluded_regions": [ + "NORTHERN_IRELAND" + ], + "absolute_tolerance": 1e-09 + }, + "notes": "Outside excluded_regions, each household's council_tax_rebate, and its persons' council_tax_benefit_reported summed, must be at most its council_tax plus absolute_tolerance, with every value read finite and non-negative. council_tax_localise is the last spine stage that writes these columns." }, { "id": "uk_stage_property_purchases_fit", @@ -1268,7 +1443,7 @@ "price_paid_property_support_bounds.json" ] }, - "notes": "Every WAS/LCFS/ETB imputed output covered by support clipping must remain inside committed disclosure-safe donor support bounds. Exact donor ranges are used in-run for clipping; this release gate uses outward-rounded reviewed bounds so unit-record donor minima and maxima are never committed. The nts_bus_travel person outputs (band and annual trips) are bounded by the NTS donor's ranges since microcosm#930. The was_lisa person balance is bounded by the WAS person donor's credible holders since microcosm#1003. Since microcosm#1240 the dense graph rewrites main_residence_value and property_wealth per clone from the Land Registry value knots, winsorised at the knots' declared cap; price_paid_property_support_bounds.json, generated beside the knots, carries that cap as the two columns' bound and is listed last so it overrides the WAS donor bound for them." + "notes": "Every WAS/LCFS/ETB imputed output covered by support clipping must remain inside committed disclosure-safe donor support bounds. Exact donor ranges are used in-run for clipping; this release gate uses outward-rounded reviewed bounds so unit-record donor minima and maxima are never committed. The nts_bus_travel person outputs (band and annual trips) are bounded by the NTS donor's ranges since microcosm#930. The was_lisa person balance is bounded by the WAS person donor's credible holders since microcosm#1003. Since microcosm#931 England and Wales owners' main_residence_value is read off their constituency's Land Registry value knots, winsorised at the knots' declared cap, inside the spine's WAS stage, which the regional property uprating returns them to, and the WAS stage's support clip exempts main_residence_value and property_wealth (uk_stage_was_wealth_support); price_paid_property_support_bounds.json, generated beside the knots for the dense value node of microcosm#1240, carries that cap as main_residence_value's bound and the cap plus the WAS donor bound's headroom over the main residence as property_wealth's, and is listed last so it overrides the WAS donor bound for both." }, { "id": "uk_aggregate_admin", @@ -1358,14 +1533,35 @@ { "id": "uk_brma_enum_domain", "gate": "enum_domain", - "phase": "assembled", + "phase": "transferred", "criticality": "release_blocking", "parameters": { "columns": [ "brma" ] }, - "notes": "The household BRMA assignment must remain inside the PolicyEngine-UK brma enum domain. Evaluated at the assembled boundary, right after frs_brma writes the column: enum membership is a property of the values themselves, so checking it there is identical to checking it at the end and refuses fourteen stages earlier. Take-up signal stays terminal by contrast, because its weighted shares are not the same statistic before and after calibration." + "notes": "The household BRMA assignment must remain inside the PolicyEngine-UK brma enum domain. area_brma writes the column, Great Britain's from each household's assigned area and Northern Ireland's from the count table, and spi_copy_areas, the last spine stage that writes it, gives every SPI copy and income band donor a BRMA of its own (microcosm#931), so the gate is evaluated at the transferred boundary; later copies inherit their parent's, which uk_copy_area_agreement checks. Enum membership is a property of the values themselves, so checking it there is identical to checking it at the end. It replaces uk_local_brma_enum_domain, the local battery's recheck of the dense line's own BRMA rewrite, which retired when the dense line took its areas from the spine. Take-up signal stays terminal by contrast, because its weighted shares are not the same statistic before and after calibration." + }, + { + "id": "uk_local_authority_enum_domain", + "gate": "enum_domain", + "phase": "transferred", + "criticality": "release_blocking", + "parameters": { + "columns": [ + "local_authority" + ] + }, + "notes": "Every household's local_authority must be a member of the installed PolicyEngine-UK LocalAuthority enum (microcosm#953, microcosm#931). The spine's local_authority stage resolves it from each household's assigned local_authority_code, and spi_copy_areas from each SPI copy's and income band donor's own; the spine battery supplies the rules engine the domain is read from. Evaluated at the transferred boundary, after spi_copy_areas, the last spine stage that writes the column; later copies inherit their parent's, which uk_copy_area_agreement checks." + }, + { + "id": "uk_geography_ladder", + "gate": "spine_agreement", + "phase": "transferred", + "criticality": "release_blocking", + "population_fact_check": true, + "parameters": {}, + "notes": "The UK geography-ladder evaluator (uk_geography_ladder_gate) on the spine's transferred frame (C-31, microcosm#931): every ladder column present and its codes well formed, the ITL layers nested, every local_authority the engine key its local_authority_code resolves to, London's share of England and Wales household weight within its bounds, and ward, ITL and constituency coverage. Evaluated after spi_copy_areas gives the SPI copies and income band donors ladders of their own, so their ladders are checked, and the national line's with them. The London share is a population fact, so a synthetic smoke build records a failure without blocking on it (population_fact_check). The local battery reruns the evaluator on the calibrated candidate (uk_local_geography_ladder_post_calibration)." }, { "id": "uk_ons_household_type_enum_domain", @@ -1535,7 +1731,7 @@ ] } }, - "notes": "The nation check of uk_council_tax_band_domain, on the frame the release ships (microcosm#1278 review). The spine entry stops at the transferred boundary, so a later step that writes council_tax_band, such as a full build's band redraw after geography derivation, is checked only here. Its parameters must equal the spine entry's." + "notes": "The nation check of uk_council_tax_band_domain, on the frame the release ships (microcosm#1278 review). The spine entry stops at the transferred boundary, so a step after it that writes council_tax_band would be checked only here; none does since the dense line's band redraw retired and the dense line took its areas from the spine (microcosm#931), and the entry stays as the shipped frame's own check. Its parameters must equal the spine entry's." }, { "id": "uk_calibration_reference_coverage", @@ -1628,19 +1824,6 @@ "parameters": {}, "notes": "The existing UK geography-ladder evaluator is rerun on the locally calibrated candidate weights and adapted through UKGateBinding; no second ladder verdict implementation is introduced." }, - { - "id": "uk_local_brma_enum_domain", - "gate": "enum_domain", - "phase": "terminal", - "criticality": "release_blocking", - "evidence_absent_blocks": true, - "parameters": { - "columns": [ - "brma" - ] - }, - "notes": "The post-geography node uk.full.geography.brma rewrites household brma from the assigned LSOA or output area after the spine's assembled enum check (microcosm#1262). Recheck the local candidate against the PolicyEngine-UK brma enum domain before export: an invalid value or missing domain evidence is structural and stops even a development build. A separate local entry keeps the assembled spine declaration, and the spine checkpoint identity it feeds, unchanged." - }, { "id": "uk_local_area_support", "gate": "area_support", diff --git a/packages/microcosm-build/src/microcosm/build/uk/ledger_fact_vendor_selections.json b/packages/microcosm-build/src/microcosm/build/uk/ledger_fact_vendor_selections.json index d8c7185729..12c018e48f 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/ledger_fact_vendor_selections.json +++ b/packages/microcosm-build/src/microcosm/build/uk/ledger_fact_vendor_selections.json @@ -2223,6 +2223,39 @@ "expected_row_count": 1 } ] + }, + { + "resource": "scotgov_band_d_council_tax_2024_25.json", + "purpose": "The Scottish Government's average Band D council tax for 2024-25 in each of the 32 council areas, from CTAS 2026 Council Tax Assumptions (the record set scotgov.ctas2026.band_d.fy2024; the local_authority level leaves out its Scotland row). Read through uk_runtime.council_tax_band_d, which merges it with England's and Wales's 2024-25 Band D (council_tax_band_d_england_wales_2024_25.json) over the Great Britain codes of local_authority_names.json for council-tax localisation (microcosm#931).", + "consumers": [ + "uk_runtime.council_tax_band_d" + ], + "planned_consumers": [], + "selections": [ + { + "label": "ctas_2026_average_band_d_2024_25", + "selector": { + "source_name": "scotgov", + "source_concept": [ + "scotgov.council_tax.average_band_d" + ], + "period_type": "fiscal_year", + "period_value": [ + "2024" + ], + "geography_level": [ + "local_authority" + ] + }, + "expected_row_count": 32 + } + ], + "licence": { + "licence": "Open Government Licence v3.0", + "attribution": [ + "Contains public sector information licensed under the Open Government Licence v3.0 (Scottish Government, CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27)." + ] + } } ] } diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index 4096c30332..7c40b57740 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -61,7 +61,7 @@ "required_mass_change_reason": "Capital-gains incidence anchor moves the mass of non-liable clone households back to their paired originals until the sub-exempt and loss-making clone mass match the Advani-Summers reporter composition at the redrawn liable mass; every pair's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "d31536927c42b7a2981612e4b96ff024c42129c4042ca85c71ab86e37b962fa0", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, the HMRC Table 3 redrawn gains on the spine and Advani-Summers capital-gains incidence" @@ -84,7 +84,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "d31536927c42b7a2981612e4b96ff024c42129c4042ca85c71ab86e37b962fa0", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence" @@ -107,7 +107,7 @@ "required_mass_change_reason": "Capital-gains residential split divides every household carrying liable gainers into arms weighted by the solved residential probabilities, one arm per subset of residential gainers; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "d31536927c42b7a2981612e4b96ff024c42129c4042ca85c71ab86e37b962fa0", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative) on the amounts the Table 3 redraw placed" @@ -129,7 +129,7 @@ "required_mass_change_reason": "Capital-gains support split divides the wealthiest households of each HMRC Table 3 income band into light copies at equal weight; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "d31536927c42b7a2981612e4b96ff024c42129c4042ca85c71ab86e37b962fa0", "source_vintages": { "source": "HMRC Capital Gains Tax statistics, July 2026 release, Table 3 for 2024-25, via the vendored conditioning facts", "survey": "Family Resources Survey 2024-25, SPI synthetic support and HMRC Capital Gains Tax statistics Table 3 (2024-25 individuals by size of gain and taxable income)" @@ -163,7 +163,7 @@ "hmrc_spi_employment_benefits" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "d31536927c42b7a2981612e4b96ff024c42129c4042ca85c71ab86e37b962fa0", "source_vintages": { "source": "The FRS job record's company car answers through frs_spine's carriers (EXPBEN01, EXPBEN03, CARVAL, FUELTYP, CARCON, CARAMT), WAS round-8 company cars' list prices, and HMRC's 2024-25 company car recipients by CO2 band (Table 2A) and fuel (Table 2C), via the vendored facts", "survey": "Family Resources Survey 2024-25 job record; Wealth and Assets Survey round 8; HMRC Benefits in kind statistics (July 2026)" @@ -171,6 +171,30 @@ "stage": "company_cars", "status": "required_at_build" }, + "council_tax_localise": { + "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", + "base_candidate_tier": "frs", + "effective_mass_requirements": {}, + "mass_change_semantics": "mass_conserving", + "output_weight_kind": "importance", + "outputs": [ + "council_tax_frs_bill" + ], + "required_mass_change_reason": "Council tax localised by each household's local authority on the source spine: household weights pass through unchanged and total household mass is conserved.", + "rewrites": [ + "council_tax", + "council_tax_rebate", + "council_tax_benefit_reported" + ], + "source_manifest": "source_stages.json", + "source_manifest_sha256": "d31536927c42b7a2981612e4b96ff024c42129c4042ca85c71ab86e37b962fa0", + "source_vintages": { + "source": "Each Great Britain local authority's Band D council tax for 2024-25, keyed by April 2023 local authority code, England's from MHCLG's Council Tax levels set by local authorities in England 2024 to 2025 (Table 8) and Wales's from StatsWales (council_tax_band_d_england_wales_2024_25.json), and Scotland's from the Scottish Government's CTAS 2026 Council Tax Assumptions, vendored from the pinned Chronicle feed (scotgov_band_d_council_tax_2024_25.json); the statutory band proportions and single-adult discount (Local Government Finance Act 1992, as amended); and the conditioned atomic-area supports sources.yaml declares, on which each household's draw cell is recomputed.", + "survey": "Family Resources Survey 2024-25" + }, + "stage": "council_tax_localise", + "status": "required_at_build" + }, "etb_services": { "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", @@ -192,7 +216,7 @@ "required_mass_change_reason": "ETB public-services imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "d31536927c42b7a2981612e4b96ff024c42129c4042ca85c71ab86e37b962fa0", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -212,7 +236,7 @@ "required_mass_change_reason": "ETB VAT expenditure-rate imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "d31536927c42b7a2981612e4b96ff024c42129c4042ca85c71ab86e37b962fa0", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -220,6 +244,31 @@ "stage": "etb_vat", "status": "required_at_build" }, + "geographic_expand": { + "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", + "base_candidate_tier": "frs", + "effective_mass_requirements": {}, + "mass_change_semantics": "mass_conserving", + "output_weight_kind": "importance", + "outputs": [ + "person_clone_index", + "benunit_clone_index", + "household_clone_index", + "source_household_id", + "source_year", + "source_household_key" + ], + "required_mass_change_reason": "Geographic clones split each household's design weight equally among its clones, so household mass is conserved.", + "rewrites": [], + "source_manifest": "source_stages.json", + "source_manifest_sha256": "d31536927c42b7a2981612e4b96ff024c42129c4042ca85c71ab86e37b962fa0", + "source_vintages": { + "source": "Microcosm structural operator (microcosm#931); the FRS spine's own households, no publisher input", + "survey": "Family Resources Survey 2024-25" + }, + "stage": "geographic_expand", + "status": "required_at_build" + }, "high_value_home_support": { "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", @@ -234,7 +283,7 @@ "required_mass_change_reason": "High-value home support divides every owner household in the top value stratum of its region into light copies at equal weight; every household's mass and the total household mass are conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "d31536927c42b7a2981612e4b96ff024c42129c4042ca85c71ab86e37b962fa0", "source_vintages": { "source": "Microcosm structural operator (microcosm#1240 fix 1); no publisher input", "survey": "Family Resources Survey 2024-25 and SPI synthetic support, after the WAS wealth draw and the regional property uprating" @@ -255,7 +304,7 @@ "required_mass_change_reason": "CGT asset-type assignment on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "d31536927c42b7a2981612e4b96ff024c42129c4042ca85c71ab86e37b962fa0", "source_vintages": { "source": "https://www.gov.uk/government/statistics/capital-gains-tax-statistics", "survey": "HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative), Table 4.1 (individuals claiming Business Asset Disposal Relief or Investors' Relief by band of qualifying gain, 2024-25, administrative) and Table 7 (disposals, proceeds and gains by asset type, 2023-24, sample-based), vendored from the pinned Chronicle feed" @@ -279,7 +328,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "d31536927c42b7a2981612e4b96ff024c42129c4042ca85c71ab86e37b962fa0", "source_vintages": { "conditioning_resource": "hmrc_cgt_conditioning_facts.json", "conditioning_resource_sha256": "bade5462cb0376e46a163639c63a5237c45577be0f9aa7e75b10c057f4b3ac3c", @@ -369,7 +418,7 @@ "full_frs_tei_band_unavailable" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "d31536927c42b7a2981612e4b96ff024c42129c4042ca85c71ab86e37b962fa0", "source_vintages": { "hmrc_surface": "2023-24", "mapped_build_period": "2024", @@ -396,7 +445,7 @@ "required_mass_change_reason": "ISA subscriptions per adult from HMRC's ISA tables on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "d31536927c42b7a2981612e4b96ff024c42129c4042ca85c71ab86e37b962fa0", "source_vintages": { "source": "HMRC Individual Savings Account statistics Tables 9.4, 9.6 and 9.7 to 9.9, the cash ISA limit reduction impact note and Lifetime ISA Table 1a, via the vendored facts; Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain).", "survey": "HMRC Annual savings statistics (September 2026 release); Wealth and Assets Survey round 8" @@ -468,7 +517,7 @@ "required_mass_change_reason": "LCFS consumption imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "d31536927c42b7a2981612e4b96ff024c42129c4042ca85c71ab86e37b962fa0", "source_vintages": { "source": "UK Data Service SN 9603 Living Costs and Food Survey 2024-25 (2nd edition) household/person tabs and the vendored Chronicle facts for NEED 2024 mean kWh, DESNZ Energy Trends and QEP average prices paid for calendar 2024, road-fuel outturn (DESNZ, HMRC, ONS Consumer Trends) and road fuel by vehicle type (DESNZ), ONS Consumer Trends household spending by COICOP division, the DfT VEH1103 licensed-car stock, NTS bus use, the NTS private cars (UK Data Service SN 5340, End User Licence; the vehicle and household tabs, survey years 2022-2024) for the zero-emission share by income fifth and the household-paid road-fuel mix, and the DfT and devolved bus finance tables and fares indices.", "survey": "Living Costs and Food Survey 2024-25" @@ -494,7 +543,7 @@ "required_mass_change_reason": "NTS bus-travel imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "d31536927c42b7a2981612e4b96ff024c42129c4042ca85c71ab86e37b962fa0", "source_vintages": { "source": "Department for Transport National Travel Survey, UK Data Service SN 5340 (End User Licence, 19th edition, November 2025), DOI 10.5255/UKDA-SN-5340-19; local licensed tab files (household, individual, trip); England residents only from 2013.", "survey": "National Travel Survey 2002-2024" @@ -515,7 +564,7 @@ "required_mass_change_reason": "PIP points stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "d31536927c42b7a2981612e4b96ff024c42129c4042ca85c71ab86e37b962fa0", "source_vintages": { "source": "pip_dl_category and pip_m_category (frs_disability, and hmrc_spi_income_spine for the SPI rows) and the stored age; dwp_pip_points_facts.json, vendored from the pinned feed", "survey": "DWP's Pathways to Work evidence pack, chapter 2, Tables 2.9 and 2.10 (PIP points by age group, England and Wales, end of January 2025), on the spine's stored PIP awards" @@ -523,6 +572,26 @@ "stage": "pip_points", "status": "required_at_build" }, + "private_rent": { + "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", + "base_candidate_tier": "frs", + "effective_mass_requirements": {}, + "mass_change_semantics": "mass_conserving", + "output_weight_kind": "importance", + "outputs": [], + "required_mass_change_reason": "Private rents read off the BRMA lists on the source spine: household weights pass through unchanged and total household mass is conserved.", + "rewrites": [ + "rent" + ], + "source_manifest": "source_stages.json", + "source_manifest_sha256": "d31536927c42b7a2981612e4b96ff024c42129c4042ca85c71ab86e37b962fa0", + "source_vintages": { + "source": "The BRMAs' weekly lists of rents (brma_rent_quantiles.json) and the census private-rented households by region, BRMA and bedrooms (brma_private_rented_households.json).", + "survey": "Family Resources Survey 2024-25" + }, + "stage": "private_rent", + "status": "required_at_build" + }, "private_school_attendance": { "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", @@ -535,9 +604,9 @@ "required_mass_change_reason": "Private-school attendance stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "d31536927c42b7a2981612e4b96ff024c42129c4042ca85c71ab86e37b962fa0", "source_vintages": { - "source": "policyengine-uk attends_private_school (the gov.simulation.private_school_vat attendance rates and factor), evaluated by the pinned engine", + "source": "policyengine-uk attends_private_school (the gov.simulation.private_school_vat attendance rates and factor) over every person in the frame, from the pinned engine's values and parameters", "survey": "policyengine-uk private-school attendance imputation, stored at the spine's weights" }, "stage": "private_school_attendance", @@ -558,7 +627,7 @@ "property_finance_costs" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "d31536927c42b7a2981612e4b96ff024c42129c4042ca85c71ab86e37b962fa0", "source_vintages": { "source": "https://www.gov.uk/government/statistics/property-rental-income-statistics", "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Property Rental Income Statistics 2026 (tax year 2024-25)" @@ -579,7 +648,7 @@ "required_mass_change_reason": "Annual property purchases imputed on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "d31536927c42b7a2981612e4b96ff024c42129c4042ca85c71ab86e37b962fa0", "source_vintages": { "source": "https://www.gov.uk/government/statistics/uk-stamp-tax-statistics", "survey": "HMRC Annual Stamp Tax Statistics 2024 to 2025 (corrected 11 March 2026), Tables 6b, 8a and 8b" @@ -587,29 +656,6 @@ "stage": "property_purchases", "status": "required_at_build" }, - "regional_home_values": { - "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", - "base_candidate_tier": "frs", - "effective_mass_requirements": {}, - "mass_change_semantics": "mass_conserving", - "output_weight_kind": "importance", - "outputs": [ - "main_residence_value_rank_basis" - ], - "required_mass_change_reason": "Regional home values assigned on the source spine: household weights pass through unchanged and total household mass is conserved.", - "rewrites": [ - "main_residence_value", - "property_wealth" - ], - "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", - "source_vintages": { - "source": "uk/high_value_home_value_knots.json, built by tools/build_uk_high_value_home_stock.py from PolicyEngine/chronicle#343's registrations", - "survey": "HM Land Registry Price Paid latest sales uprated with the UK House Price Index, owner-weighted by region (the dense value node's knots, region level)" - }, - "stage": "regional_home_values", - "status": "required_at_build" - }, "regional_property_uprating": { "base_candidate_sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833", "base_candidate_tier": "frs", @@ -626,7 +672,7 @@ "non_residential_property_value" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "d31536927c42b7a2981612e4b96ff024c42129c4042ca85c71ab86e37b962fa0", "source_vintages": { "source": "MHCLG dwellings and the UK House Price Index regional average price averaged over the twelve months of calendar 2024 from the July 2026 full file, the frame's valuation basis, so the engine's annual uprating from 2024 projects the value once (microcosm#1240 fix 5).", "survey": "Public regional property reference" @@ -654,7 +700,7 @@ "employment_income" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "d31536927c42b7a2981612e4b96ff024c42129c4042ca85c71ab86e37b962fa0", "source_vintages": { "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis" @@ -687,7 +733,7 @@ "household_source_id" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "d31536927c42b7a2981612e4b96ff024c42129c4042ca85c71ab86e37b962fa0", "source_vintages": { "source": "Reserved SPI support households for the HMRC Table 2.5 total-income bands from GBP 200,000, stacked at the published band taxpayer count over the donors per band (PolicyEngine/chronicle#280 lane).", "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Income Tax liabilities statistics July 2026 Table 2.5" @@ -710,7 +756,7 @@ "student_loan_balance" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "d31536927c42b7a2981612e4b96ff024c42129c4042ca85c71ab86e37b962fa0", "source_vintages": { "source": "Explore Education Statistics Table 6a, Higher education total", "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England" @@ -731,7 +777,7 @@ "required_mass_change_reason": "Trading allowance claim statuses stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "d31536927c42b7a2981612e4b96ff024c42129c4042ca85c71ab86e37b962fa0", "source_vintages": { "source": "self_employment_income after the SPI income stage, and household_is_spi_synthetic for the full-relief group; HMRC's published gainers of the raise to GBP 3,000, about 400,000 taxpayers with a lower bill by about GBP 380 on average", "survey": "The self-employed's own profit on the final spine, and HMRC's policy paper on the Budget 2026 trading allowance raise (calibration)" @@ -752,7 +798,7 @@ "required_mass_change_reason": "UC limited capability flags stored on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "d31536927c42b7a2981612e4b96ff024c42129c4042ca85c71ab86e37b962fa0", "source_vintages": { "source": "frs_uc_lcw_status (frs_spine) on the UC row's person, and the locked engine's uc_limited_capability_for_WRA, uc_limited_capability_for_work, is_uc_assessed_claimant and is_carer_for_benefits on the frame as it stands", "survey": "Family Resources Survey 2024-25 (BENEFITS.UCLCW, DWP's limited capability status on a linked Universal Credit claim) and policyengine-uk's limited capability proxy, evaluated by the pinned engine" @@ -774,7 +820,7 @@ "required_mass_change_reason": "WAS Lifetime ISA imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "d31536927c42b7a2981612e4b96ff024c42129c4042ca85c71ab86e37b962fa0", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain).", "survey": "Wealth and Assets Survey round 8" @@ -796,6 +842,7 @@ "gross_financial_wealth", "net_financial_wealth", "main_residence_value", + "main_residence_value_rank_basis", "other_residential_property_value", "non_residential_property_value", "savings", @@ -813,9 +860,9 @@ "required_mass_change_reason": "WAS wealth imputation on the source spine: household weights pass through unchanged and total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "c737d82ea2339730fcc18bd368293aacf9c4bcb87a71dfb2c270a44c0e172565", + "source_manifest_sha256": "d31536927c42b7a2981612e4b96ff024c42129c4042ca85c71ab86e37b962fa0", "source_vintages": { - "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", + "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab. England and Wales owners' main residences are read off HM Land Registry Price Paid latest sales uprated with the UK House Price Index, by constituency (uk/high_value_home_value_knots.json, built by tools/build_uk_high_value_home_stock.py from PolicyEngine/chronicle#343's registrations), and the public regional reference (uk/regional_land_values.json) (microcosm#931).", "survey": "Wealth and Assets Survey round 8" }, "stage": "was_wealth", diff --git a/packages/microcosm-build/src/microcosm/build/uk/scotgov_band_d_council_tax_2024_25.json b/packages/microcosm-build/src/microcosm/build/uk/scotgov_band_d_council_tax_2024_25.json new file mode 100644 index 0000000000..a224edc03a --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk/scotgov_band_d_council_tax_2024_25.json @@ -0,0 +1,1429 @@ +{ + "schema_version": 1, + "country": "uk", + "kind": "uk_vendored_ledger_facts", + "resource": "scotgov_band_d_council_tax_2024_25.json", + "purpose": "The Scottish Government's average Band D council tax for 2024-25 in each of the 32 council areas, from CTAS 2026 Council Tax Assumptions (the record set scotgov.ctas2026.band_d.fy2024; the local_authority level leaves out its Scotland row). Read through uk_runtime.council_tax_band_d, which merges it with England's and Wales's 2024-25 Band D (council_tax_band_d_england_wales_2024_25.json) over the Great Britain codes of local_authority_names.json for council-tax localisation (microcosm#931).", + "consumers": [ + "uk_runtime.council_tax_band_d" + ], + "planned_consumers": [], + "policy": "Rows are copied verbatim from the pinned Chronicle consumer feed named in source_fact_feed; every value is a publisher-stated fact. Consumers declare their own arithmetic on these rows and never edit this file by hand: regenerate it with tools/vendor_uk_ledger_facts.py.", + "licence": { + "licence": "Open Government Licence v3.0", + "attribution": [ + "Contains public sector information licensed under the Open Government Licence v3.0 (Scottish Government, CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27)." + ] + }, + "generator": { + "tool": "tools/vendor_uk_ledger_facts.py", + "version": 1 + }, + "source_fact_feed": { + "source_repo": "PolicyEngine/chronicle", + "source_commit": "4ce631f5ad690fb85007193273ee85a873b824d3", + "build": "build-bundle --suite uk -> build-consumer-artifact", + "artifact_schema_version": "policyengine_ledger.consumer_artifact.v2", + "fact_row_count": 2160233, + "facts_sha256": "c294ba2013d0ff61b150589640ec63add8563390057b769447c0241661ad5a1c", + "manifest_sha256": "74e564fba1c4589e15e4bb19138414c46e4db316486b5c0d7ccbcccb211c52df" + }, + "selections": [ + { + "label": "ctas_2026_average_band_d_2024_25", + "selector": { + "source_name": "scotgov", + "source_concept": [ + "scotgov.council_tax.average_band_d" + ], + "period_type": "fiscal_year", + "period_value": [ + "2024" + ], + "geography_level": [ + "local_authority" + ] + }, + "row_count": 32 + } + ], + "row_count": 32, + "rows": [ + { + "aggregate_fact_key": "ledger_aggregate_fact_v2_e5b027466338127c8d996197", + "semantic_fact_key": "ledger_semantic_fact_v2_ce6622454612d5c116ebc9ef", + "source_record_id": "scotgov.ctas2026.band_d.fy2024.s12000005.average_band_d", + "concept": "scotgov.council_tax.average_band_d", + "measure_id": "average_band_d", + "unit": "gbp", + "aggregation": "mean", + "entity": { + "name": "dwelling", + "role": "chargeable_dwelling" + }, + "geography": { + "id": "S12000005", + "level": "local_authority", + "vintage": "ca_2019" + }, + "period": { + "type": "fiscal_year", + "value": 2024 + }, + "period_coverage": { + "start_date": null, + "end_date": null, + "basis": null + }, + "layout": { + "record_set_id": "scotgov.ctas2026.band_d.fy2024", + "groupby_dimension": "geography", + "groupby_value_id": "s12000005", + "table_record_kind": "detail" + }, + "dimensions": {}, + "source": { + "source_name": "scotgov", + "source_table": "CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27", + "source_file": "CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx", + "source_sha256": "4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16", + "vintage": "ctas_2026", + "raw_r2_key": "raw/scotgov/scotgov-band-d-council-tax-rates-2026-27/2026/4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16/CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx" + }, + "value": 1410.96 + }, + { + "aggregate_fact_key": "ledger_aggregate_fact_v2_9fde92ee1d701cf48fdb1a00", + "semantic_fact_key": "ledger_semantic_fact_v2_58eac4215aad04160c211f0a", + "source_record_id": "scotgov.ctas2026.band_d.fy2024.s12000006.average_band_d", + "concept": "scotgov.council_tax.average_band_d", + "measure_id": "average_band_d", + "unit": "gbp", + "aggregation": "mean", + "entity": { + "name": "dwelling", + "role": "chargeable_dwelling" + }, + "geography": { + "id": "S12000006", + "level": "local_authority", + "vintage": "ca_2019" + }, + "period": { + "type": "fiscal_year", + "value": 2024 + }, + "period_coverage": { + "start_date": null, + "end_date": null, + "basis": null + }, + "layout": { + "record_set_id": "scotgov.ctas2026.band_d.fy2024", + "groupby_dimension": "geography", + "groupby_value_id": "s12000006", + "table_record_kind": "detail" + }, + "dimensions": {}, + "source": { + "source_name": "scotgov", + "source_table": "CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27", + "source_file": "CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx", + "source_sha256": "4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16", + "vintage": "ctas_2026", + "raw_r2_key": "raw/scotgov/scotgov-band-d-council-tax-rates-2026-27/2026/4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16/CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx" + }, + "value": 1334.85 + }, + { + "aggregate_fact_key": "ledger_aggregate_fact_v2_b351a8ab07764f59f4cfcd3b", + "semantic_fact_key": "ledger_semantic_fact_v2_dddabc4fa7a2953688feabad", + "source_record_id": "scotgov.ctas2026.band_d.fy2024.s12000008.average_band_d", + "concept": "scotgov.council_tax.average_band_d", + "measure_id": "average_band_d", + "unit": "gbp", + "aggregation": "mean", + "entity": { + "name": "dwelling", + "role": "chargeable_dwelling" + }, + "geography": { + "id": "S12000008", + "level": "local_authority", + "vintage": "ca_2019" + }, + "period": { + "type": "fiscal_year", + "value": 2024 + }, + "period_coverage": { + "start_date": null, + "end_date": null, + "basis": null + }, + "layout": { + "record_set_id": "scotgov.ctas2026.band_d.fy2024", + "groupby_dimension": "geography", + "groupby_value_id": "s12000008", + "table_record_kind": "detail" + }, + "dimensions": {}, + "source": { + "source_name": "scotgov", + "source_table": "CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27", + "source_file": "CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx", + "source_sha256": "4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16", + "vintage": "ctas_2026", + "raw_r2_key": "raw/scotgov/scotgov-band-d-council-tax-rates-2026-27/2026/4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16/CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx" + }, + "value": 1487.44 + }, + { + "aggregate_fact_key": "ledger_aggregate_fact_v2_e5a34949c6a32d60ae0a0e1d", + "semantic_fact_key": "ledger_semantic_fact_v2_60471a28cb0c9f788c0e40e9", + "source_record_id": "scotgov.ctas2026.band_d.fy2024.s12000010.average_band_d", + "concept": "scotgov.council_tax.average_band_d", + "measure_id": "average_band_d", + "unit": "gbp", + "aggregation": "mean", + "entity": { + "name": "dwelling", + "role": "chargeable_dwelling" + }, + "geography": { + "id": "S12000010", + "level": "local_authority", + "vintage": "ca_2019" + }, + "period": { + "type": "fiscal_year", + "value": 2024 + }, + "period_coverage": { + "start_date": null, + "end_date": null, + "basis": null + }, + "layout": { + "record_set_id": "scotgov.ctas2026.band_d.fy2024", + "groupby_dimension": "geography", + "groupby_value_id": "s12000010", + "table_record_kind": "detail" + }, + "dimensions": {}, + "source": { + "source_name": "scotgov", + "source_table": "CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27", + "source_file": "CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx", + "source_sha256": "4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16", + "vintage": "ctas_2026", + "raw_r2_key": "raw/scotgov/scotgov-band-d-council-tax-rates-2026-27/2026/4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16/CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx" + }, + "value": 1435.62 + }, + { + "aggregate_fact_key": "ledger_aggregate_fact_v2_c5692eb4dd27c0d9a6df3da5", + "semantic_fact_key": "ledger_semantic_fact_v2_c18e1672cc3f2d77a01886ec", + "source_record_id": "scotgov.ctas2026.band_d.fy2024.s12000011.average_band_d", + "concept": "scotgov.council_tax.average_band_d", + "measure_id": "average_band_d", + "unit": "gbp", + "aggregation": "mean", + "entity": { + "name": "dwelling", + "role": "chargeable_dwelling" + }, + "geography": { + "id": "S12000011", + "level": "local_authority", + "vintage": "ca_2019" + }, + "period": { + "type": "fiscal_year", + "value": 2024 + }, + "period_coverage": { + "start_date": null, + "end_date": null, + "basis": null + }, + "layout": { + "record_set_id": "scotgov.ctas2026.band_d.fy2024", + "groupby_dimension": "geography", + "groupby_value_id": "s12000011", + "table_record_kind": "detail" + }, + "dimensions": {}, + "source": { + "source_name": "scotgov", + "source_table": "CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27", + "source_file": "CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx", + "source_sha256": "4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16", + "vintage": "ctas_2026", + "raw_r2_key": "raw/scotgov/scotgov-band-d-council-tax-rates-2026-27/2026/4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16/CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx" + }, + "value": 1415.22 + }, + { + "aggregate_fact_key": "ledger_aggregate_fact_v2_b49c066ce3f70db5f3a6512d", + "semantic_fact_key": "ledger_semantic_fact_v2_625252b269bf997000cb86c6", + "source_record_id": "scotgov.ctas2026.band_d.fy2024.s12000013.average_band_d", + "concept": "scotgov.council_tax.average_band_d", + "measure_id": "average_band_d", + "unit": "gbp", + "aggregation": "mean", + "entity": { + "name": "dwelling", + "role": "chargeable_dwelling" + }, + "geography": { + "id": "S12000013", + "level": "local_authority", + "vintage": "ca_2019" + }, + "period": { + "type": "fiscal_year", + "value": 2024 + }, + "period_coverage": { + "start_date": null, + "end_date": null, + "basis": null + }, + "layout": { + "record_set_id": "scotgov.ctas2026.band_d.fy2024", + "groupby_dimension": "geography", + "groupby_value_id": "s12000013", + "table_record_kind": "detail" + }, + "dimensions": {}, + "source": { + "source_name": "scotgov", + "source_table": "CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27", + "source_file": "CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx", + "source_sha256": "4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16", + "vintage": "ctas_2026", + "raw_r2_key": "raw/scotgov/scotgov-band-d-council-tax-rates-2026-27/2026/4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16/CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx" + }, + "value": 1290.75 + }, + { + "aggregate_fact_key": "ledger_aggregate_fact_v2_2ac60f5492d6c668cb20293f", + "semantic_fact_key": "ledger_semantic_fact_v2_862f8aa9f493ad4877af0d8a", + "source_record_id": "scotgov.ctas2026.band_d.fy2024.s12000014.average_band_d", + "concept": "scotgov.council_tax.average_band_d", + "measure_id": "average_band_d", + "unit": "gbp", + "aggregation": "mean", + "entity": { + "name": "dwelling", + "role": "chargeable_dwelling" + }, + "geography": { + "id": "S12000014", + "level": "local_authority", + "vintage": "ca_2019" + }, + "period": { + "type": "fiscal_year", + "value": 2024 + }, + "period_coverage": { + "start_date": null, + "end_date": null, + "basis": null + }, + "layout": { + "record_set_id": "scotgov.ctas2026.band_d.fy2024", + "groupby_dimension": "geography", + "groupby_value_id": "s12000014", + "table_record_kind": "detail" + }, + "dimensions": {}, + "source": { + "source_name": "scotgov", + "source_table": "CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27", + "source_file": "CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx", + "source_sha256": "4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16", + "vintage": "ctas_2026", + "raw_r2_key": "raw/scotgov/scotgov-band-d-council-tax-rates-2026-27/2026/4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16/CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx" + }, + "value": 1363.82 + }, + { + "aggregate_fact_key": "ledger_aggregate_fact_v2_65354727c38acd6aff3a75b3", + "semantic_fact_key": "ledger_semantic_fact_v2_b05659bbfa37505ab94bf62a", + "source_record_id": "scotgov.ctas2026.band_d.fy2024.s12000017.average_band_d", + "concept": "scotgov.council_tax.average_band_d", + "measure_id": "average_band_d", + "unit": "gbp", + "aggregation": "mean", + "entity": { + "name": "dwelling", + "role": "chargeable_dwelling" + }, + "geography": { + "id": "S12000017", + "level": "local_authority", + "vintage": "ca_2019" + }, + "period": { + "type": "fiscal_year", + "value": 2024 + }, + "period_coverage": { + "start_date": null, + "end_date": null, + "basis": null + }, + "layout": { + "record_set_id": "scotgov.ctas2026.band_d.fy2024", + "groupby_dimension": "geography", + "groupby_value_id": "s12000017", + "table_record_kind": "detail" + }, + "dimensions": {}, + "source": { + "source_name": "scotgov", + "source_table": "CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27", + "source_file": "CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx", + "source_sha256": "4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16", + "vintage": "ctas_2026", + "raw_r2_key": "raw/scotgov/scotgov-band-d-council-tax-rates-2026-27/2026/4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16/CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx" + }, + "value": 1427.19 + }, + { + "aggregate_fact_key": "ledger_aggregate_fact_v2_f4cb297d02e2ce4056affbf0", + "semantic_fact_key": "ledger_semantic_fact_v2_ed0b5d12a8908c0f5cd95ce3", + "source_record_id": "scotgov.ctas2026.band_d.fy2024.s12000018.average_band_d", + "concept": "scotgov.council_tax.average_band_d", + "measure_id": "average_band_d", + "unit": "gbp", + "aggregation": "mean", + "entity": { + "name": "dwelling", + "role": "chargeable_dwelling" + }, + "geography": { + "id": "S12000018", + "level": "local_authority", + "vintage": "ca_2019" + }, + "period": { + "type": "fiscal_year", + "value": 2024 + }, + "period_coverage": { + "start_date": null, + "end_date": null, + "basis": null + }, + "layout": { + "record_set_id": "scotgov.ctas2026.band_d.fy2024", + "groupby_dimension": "geography", + "groupby_value_id": "s12000018", + "table_record_kind": "detail" + }, + "dimensions": {}, + "source": { + "source_name": "scotgov", + "source_table": "CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27", + "source_file": "CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx", + "source_sha256": "4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16", + "vintage": "ctas_2026", + "raw_r2_key": "raw/scotgov/scotgov-band-d-council-tax-rates-2026-27/2026/4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16/CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx" + }, + "value": 1547.01 + }, + { + "aggregate_fact_key": "ledger_aggregate_fact_v2_99ba7285f0e7b5355efb8d4d", + "semantic_fact_key": "ledger_semantic_fact_v2_a4e2fb625754100b7c7a009f", + "source_record_id": "scotgov.ctas2026.band_d.fy2024.s12000019.average_band_d", + "concept": "scotgov.council_tax.average_band_d", + "measure_id": "average_band_d", + "unit": "gbp", + "aggregation": "mean", + "entity": { + "name": "dwelling", + "role": "chargeable_dwelling" + }, + "geography": { + "id": "S12000019", + "level": "local_authority", + "vintage": "ca_2019" + }, + "period": { + "type": "fiscal_year", + "value": 2024 + }, + "period_coverage": { + "start_date": null, + "end_date": null, + "basis": null + }, + "layout": { + "record_set_id": "scotgov.ctas2026.band_d.fy2024", + "groupby_dimension": "geography", + "groupby_value_id": "s12000019", + "table_record_kind": "detail" + }, + "dimensions": {}, + "source": { + "source_name": "scotgov", + "source_table": "CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27", + "source_file": "CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx", + "source_sha256": "4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16", + "vintage": "ctas_2026", + "raw_r2_key": "raw/scotgov/scotgov-band-d-council-tax-rates-2026-27/2026/4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16/CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx" + }, + "value": 1514.73 + }, + { + "aggregate_fact_key": "ledger_aggregate_fact_v2_e31ccd67436606f472dad55f", + "semantic_fact_key": "ledger_semantic_fact_v2_3cea117c5de768f1d3dda801", + "source_record_id": "scotgov.ctas2026.band_d.fy2024.s12000020.average_band_d", + "concept": "scotgov.council_tax.average_band_d", + "measure_id": "average_band_d", + "unit": "gbp", + "aggregation": "mean", + "entity": { + "name": "dwelling", + "role": "chargeable_dwelling" + }, + "geography": { + "id": "S12000020", + "level": "local_authority", + "vintage": "ca_2019" + }, + "period": { + "type": "fiscal_year", + "value": 2024 + }, + "period_coverage": { + "start_date": null, + "end_date": null, + "basis": null + }, + "layout": { + "record_set_id": "scotgov.ctas2026.band_d.fy2024", + "groupby_dimension": "geography", + "groupby_value_id": "s12000020", + "table_record_kind": "detail" + }, + "dimensions": {}, + "source": { + "source_name": "scotgov", + "source_table": "CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27", + "source_file": "CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx", + "source_sha256": "4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16", + "vintage": "ctas_2026", + "raw_r2_key": "raw/scotgov/scotgov-band-d-council-tax-rates-2026-27/2026/4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16/CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx" + }, + "value": 1430.69 + }, + { + "aggregate_fact_key": "ledger_aggregate_fact_v2_979bc7e96ae8481c208f6432", + "semantic_fact_key": "ledger_semantic_fact_v2_3264f6c7d32c1509bcd40d61", + "source_record_id": "scotgov.ctas2026.band_d.fy2024.s12000021.average_band_d", + "concept": "scotgov.council_tax.average_band_d", + "measure_id": "average_band_d", + "unit": "gbp", + "aggregation": "mean", + "entity": { + "name": "dwelling", + "role": "chargeable_dwelling" + }, + "geography": { + "id": "S12000021", + "level": "local_authority", + "vintage": "ca_2019" + }, + "period": { + "type": "fiscal_year", + "value": 2024 + }, + "period_coverage": { + "start_date": null, + "end_date": null, + "basis": null + }, + "layout": { + "record_set_id": "scotgov.ctas2026.band_d.fy2024", + "groupby_dimension": "geography", + "groupby_value_id": "s12000021", + "table_record_kind": "detail" + }, + "dimensions": {}, + "source": { + "source_name": "scotgov", + "source_table": "CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27", + "source_file": "CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx", + "source_sha256": "4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16", + "vintage": "ctas_2026", + "raw_r2_key": "raw/scotgov/scotgov-band-d-council-tax-rates-2026-27/2026/4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16/CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx" + }, + "value": 1452.12 + }, + { + "aggregate_fact_key": "ledger_aggregate_fact_v2_a982ef4bc6799a707d6960ad", + "semantic_fact_key": "ledger_semantic_fact_v2_4e17161f32f0cc671e8ea92a", + "source_record_id": "scotgov.ctas2026.band_d.fy2024.s12000023.average_band_d", + "concept": "scotgov.council_tax.average_band_d", + "measure_id": "average_band_d", + "unit": "gbp", + "aggregation": "mean", + "entity": { + "name": "dwelling", + "role": "chargeable_dwelling" + }, + "geography": { + "id": "S12000023", + "level": "local_authority", + "vintage": "ca_2019" + }, + "period": { + "type": "fiscal_year", + "value": 2024 + }, + "period_coverage": { + "start_date": null, + "end_date": null, + "basis": null + }, + "layout": { + "record_set_id": "scotgov.ctas2026.band_d.fy2024", + "groupby_dimension": "geography", + "groupby_value_id": "s12000023", + "table_record_kind": "detail" + }, + "dimensions": {}, + "source": { + "source_name": "scotgov", + "source_table": "CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27", + "source_file": "CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx", + "source_sha256": "4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16", + "vintage": "ctas_2026", + "raw_r2_key": "raw/scotgov/scotgov-band-d-council-tax-rates-2026-27/2026/4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16/CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx" + }, + "value": 1369.21 + }, + { + "aggregate_fact_key": "ledger_aggregate_fact_v2_e18688659086da3b5d48e8c1", + "semantic_fact_key": "ledger_semantic_fact_v2_212d85515430615920429600", + "source_record_id": "scotgov.ctas2026.band_d.fy2024.s12000026.average_band_d", + "concept": "scotgov.council_tax.average_band_d", + "measure_id": "average_band_d", + "unit": "gbp", + "aggregation": "mean", + "entity": { + "name": "dwelling", + "role": "chargeable_dwelling" + }, + "geography": { + "id": "S12000026", + "level": "local_authority", + "vintage": "ca_2019" + }, + "period": { + "type": "fiscal_year", + "value": 2024 + }, + "period_coverage": { + "start_date": null, + "end_date": null, + "basis": null + }, + "layout": { + "record_set_id": "scotgov.ctas2026.band_d.fy2024", + "groupby_dimension": "geography", + "groupby_value_id": "s12000026", + "table_record_kind": "detail" + }, + "dimensions": {}, + "source": { + "source_name": "scotgov", + "source_table": "CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27", + "source_file": "CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx", + "source_sha256": "4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16", + "vintage": "ctas_2026", + "raw_r2_key": "raw/scotgov/scotgov-band-d-council-tax-rates-2026-27/2026/4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16/CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx" + }, + "value": 1356.11 + }, + { + "aggregate_fact_key": "ledger_aggregate_fact_v2_71bc45a01917dae95785e3f9", + "semantic_fact_key": "ledger_semantic_fact_v2_ac69c9a19a4f0ebde0f0d86e", + "source_record_id": "scotgov.ctas2026.band_d.fy2024.s12000027.average_band_d", + "concept": "scotgov.council_tax.average_band_d", + "measure_id": "average_band_d", + "unit": "gbp", + "aggregation": "mean", + "entity": { + "name": "dwelling", + "role": "chargeable_dwelling" + }, + "geography": { + "id": "S12000027", + "level": "local_authority", + "vintage": "ca_2019" + }, + "period": { + "type": "fiscal_year", + "value": 2024 + }, + "period_coverage": { + "start_date": null, + "end_date": null, + "basis": null + }, + "layout": { + "record_set_id": "scotgov.ctas2026.band_d.fy2024", + "groupby_dimension": "geography", + "groupby_value_id": "s12000027", + "table_record_kind": "detail" + }, + "dimensions": {}, + "source": { + "source_name": "scotgov", + "source_table": "CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27", + "source_file": "CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx", + "source_sha256": "4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16", + "vintage": "ctas_2026", + "raw_r2_key": "raw/scotgov/scotgov-band-d-council-tax-rates-2026-27/2026/4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16/CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx" + }, + "value": 1260.61 + }, + { + "aggregate_fact_key": "ledger_aggregate_fact_v2_274f1ec6f3aed02c8c81cfb3", + "semantic_fact_key": "ledger_semantic_fact_v2_924afe20c8841cbab99ce81e", + "source_record_id": "scotgov.ctas2026.band_d.fy2024.s12000028.average_band_d", + "concept": "scotgov.council_tax.average_band_d", + "measure_id": "average_band_d", + "unit": "gbp", + "aggregation": "mean", + "entity": { + "name": "dwelling", + "role": "chargeable_dwelling" + }, + "geography": { + "id": "S12000028", + "level": "local_authority", + "vintage": "ca_2019" + }, + "period": { + "type": "fiscal_year", + "value": 2024 + }, + "period_coverage": { + "start_date": null, + "end_date": null, + "basis": null + }, + "layout": { + "record_set_id": "scotgov.ctas2026.band_d.fy2024", + "groupby_dimension": "geography", + "groupby_value_id": "s12000028", + "table_record_kind": "detail" + }, + "dimensions": {}, + "source": { + "source_name": "scotgov", + "source_table": "CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27", + "source_file": "CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx", + "source_sha256": "4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16", + "vintage": "ctas_2026", + "raw_r2_key": "raw/scotgov/scotgov-band-d-council-tax-rates-2026-27/2026/4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16/CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx" + }, + "value": 1453.16 + }, + { + "aggregate_fact_key": "ledger_aggregate_fact_v2_e60e8a9e1691072a7ad22f7c", + "semantic_fact_key": "ledger_semantic_fact_v2_869968cb3e769a2343fc5355", + "source_record_id": "scotgov.ctas2026.band_d.fy2024.s12000029.average_band_d", + "concept": "scotgov.council_tax.average_band_d", + "measure_id": "average_band_d", + "unit": "gbp", + "aggregation": "mean", + "entity": { + "name": "dwelling", + "role": "chargeable_dwelling" + }, + "geography": { + "id": "S12000029", + "level": "local_authority", + "vintage": "ca_2019" + }, + "period": { + "type": "fiscal_year", + "value": 2024 + }, + "period_coverage": { + "start_date": null, + "end_date": null, + "basis": null + }, + "layout": { + "record_set_id": "scotgov.ctas2026.band_d.fy2024", + "groupby_dimension": "geography", + "groupby_value_id": "s12000029", + "table_record_kind": "detail" + }, + "dimensions": {}, + "source": { + "source_name": "scotgov", + "source_table": "CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27", + "source_file": "CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx", + "source_sha256": "4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16", + "vintage": "ctas_2026", + "raw_r2_key": "raw/scotgov/scotgov-band-d-council-tax-rates-2026-27/2026/4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16/CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx" + }, + "value": 1300.81 + }, + { + "aggregate_fact_key": "ledger_aggregate_fact_v2_d69bd9b0c3df6fbcb123f273", + "semantic_fact_key": "ledger_semantic_fact_v2_c46f39808dcfd405dac7a4aa", + "source_record_id": "scotgov.ctas2026.band_d.fy2024.s12000030.average_band_d", + "concept": "scotgov.council_tax.average_band_d", + "measure_id": "average_band_d", + "unit": "gbp", + "aggregation": "mean", + "entity": { + "name": "dwelling", + "role": "chargeable_dwelling" + }, + "geography": { + "id": "S12000030", + "level": "local_authority", + "vintage": "ca_2019" + }, + "period": { + "type": "fiscal_year", + "value": 2024 + }, + "period_coverage": { + "start_date": null, + "end_date": null, + "basis": null + }, + "layout": { + "record_set_id": "scotgov.ctas2026.band_d.fy2024", + "groupby_dimension": "geography", + "groupby_value_id": "s12000030", + "table_record_kind": "detail" + }, + "dimensions": {}, + "source": { + "source_name": "scotgov", + "source_table": "CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27", + "source_file": "CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx", + "source_sha256": "4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16", + "vintage": "ctas_2026", + "raw_r2_key": "raw/scotgov/scotgov-band-d-council-tax-rates-2026-27/2026/4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16/CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx" + }, + "value": 1481.5 + }, + { + "aggregate_fact_key": "ledger_aggregate_fact_v2_20bf78770e2a51eefad84777", + "semantic_fact_key": "ledger_semantic_fact_v2_b207fde24b573fc931d63bac", + "source_record_id": "scotgov.ctas2026.band_d.fy2024.s12000033.average_band_d", + "concept": "scotgov.council_tax.average_band_d", + "measure_id": "average_band_d", + "unit": "gbp", + "aggregation": "mean", + "entity": { + "name": "dwelling", + "role": "chargeable_dwelling" + }, + "geography": { + "id": "S12000033", + "level": "local_authority", + "vintage": "ca_2019" + }, + "period": { + "type": "fiscal_year", + "value": 2024 + }, + "period_coverage": { + "start_date": null, + "end_date": null, + "basis": null + }, + "layout": { + "record_set_id": "scotgov.ctas2026.band_d.fy2024", + "groupby_dimension": "geography", + "groupby_value_id": "s12000033", + "table_record_kind": "detail" + }, + "dimensions": {}, + "source": { + "source_name": "scotgov", + "source_table": "CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27", + "source_file": "CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx", + "source_sha256": "4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16", + "vintage": "ctas_2026", + "raw_r2_key": "raw/scotgov/scotgov-band-d-council-tax-rates-2026-27/2026/4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16/CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx" + }, + "value": 1489.55 + }, + { + "aggregate_fact_key": "ledger_aggregate_fact_v2_a5ba88c9fee7e6fc2fdd2bcd", + "semantic_fact_key": "ledger_semantic_fact_v2_26d23e1779498a349cb288bf", + "source_record_id": "scotgov.ctas2026.band_d.fy2024.s12000034.average_band_d", + "concept": "scotgov.council_tax.average_band_d", + "measure_id": "average_band_d", + "unit": "gbp", + "aggregation": "mean", + "entity": { + "name": "dwelling", + "role": "chargeable_dwelling" + }, + "geography": { + "id": "S12000034", + "level": "local_authority", + "vintage": "ca_2019" + }, + "period": { + "type": "fiscal_year", + "value": 2024 + }, + "period_coverage": { + "start_date": null, + "end_date": null, + "basis": null + }, + "layout": { + "record_set_id": "scotgov.ctas2026.band_d.fy2024", + "groupby_dimension": "geography", + "groupby_value_id": "s12000034", + "table_record_kind": "detail" + }, + "dimensions": {}, + "source": { + "source_name": "scotgov", + "source_table": "CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27", + "source_file": "CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx", + "source_sha256": "4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16", + "vintage": "ctas_2026", + "raw_r2_key": "raw/scotgov/scotgov-band-d-council-tax-rates-2026-27/2026/4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16/CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx" + }, + "value": 1393.42 + }, + { + "aggregate_fact_key": "ledger_aggregate_fact_v2_71c119c2bd8bcaab95fff07e", + "semantic_fact_key": "ledger_semantic_fact_v2_cd900811a7f08ab24fbd7279", + "source_record_id": "scotgov.ctas2026.band_d.fy2024.s12000035.average_band_d", + "concept": "scotgov.council_tax.average_band_d", + "measure_id": "average_band_d", + "unit": "gbp", + "aggregation": "mean", + "entity": { + "name": "dwelling", + "role": "chargeable_dwelling" + }, + "geography": { + "id": "S12000035", + "level": "local_authority", + "vintage": "ca_2019" + }, + "period": { + "type": "fiscal_year", + "value": 2024 + }, + "period_coverage": { + "start_date": null, + "end_date": null, + "basis": null + }, + "layout": { + "record_set_id": "scotgov.ctas2026.band_d.fy2024", + "groupby_dimension": "geography", + "groupby_value_id": "s12000035", + "table_record_kind": "detail" + }, + "dimensions": {}, + "source": { + "source_name": "scotgov", + "source_table": "CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27", + "source_file": "CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx", + "source_sha256": "4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16", + "vintage": "ctas_2026", + "raw_r2_key": "raw/scotgov/scotgov-band-d-council-tax-rates-2026-27/2026/4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16/CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx" + }, + "value": 1479.2 + }, + { + "aggregate_fact_key": "ledger_aggregate_fact_v2_8444167691d7981838375213", + "semantic_fact_key": "ledger_semantic_fact_v2_83535a9dd310ba291b09999b", + "source_record_id": "scotgov.ctas2026.band_d.fy2024.s12000036.average_band_d", + "concept": "scotgov.council_tax.average_band_d", + "measure_id": "average_band_d", + "unit": "gbp", + "aggregation": "mean", + "entity": { + "name": "dwelling", + "role": "chargeable_dwelling" + }, + "geography": { + "id": "S12000036", + "level": "local_authority", + "vintage": "ca_2019" + }, + "period": { + "type": "fiscal_year", + "value": 2024 + }, + "period_coverage": { + "start_date": null, + "end_date": null, + "basis": null + }, + "layout": { + "record_set_id": "scotgov.ctas2026.band_d.fy2024", + "groupby_dimension": "geography", + "groupby_value_id": "s12000036", + "table_record_kind": "detail" + }, + "dimensions": {}, + "source": { + "source_name": "scotgov", + "source_table": "CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27", + "source_file": "CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx", + "source_sha256": "4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16", + "vintage": "ctas_2026", + "raw_r2_key": "raw/scotgov/scotgov-band-d-council-tax-rates-2026-27/2026/4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16/CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx" + }, + "value": 1447.69 + }, + { + "aggregate_fact_key": "ledger_aggregate_fact_v2_74733a9cabec879c5c8ec7ca", + "semantic_fact_key": "ledger_semantic_fact_v2_c8dbfd54b0acdcf6b5d77838", + "source_record_id": "scotgov.ctas2026.band_d.fy2024.s12000038.average_band_d", + "concept": "scotgov.council_tax.average_band_d", + "measure_id": "average_band_d", + "unit": "gbp", + "aggregation": "mean", + "entity": { + "name": "dwelling", + "role": "chargeable_dwelling" + }, + "geography": { + "id": "S12000038", + "level": "local_authority", + "vintage": "ca_2019" + }, + "period": { + "type": "fiscal_year", + "value": 2024 + }, + "period_coverage": { + "start_date": null, + "end_date": null, + "basis": null + }, + "layout": { + "record_set_id": "scotgov.ctas2026.band_d.fy2024", + "groupby_dimension": "geography", + "groupby_value_id": "s12000038", + "table_record_kind": "detail" + }, + "dimensions": {}, + "source": { + "source_name": "scotgov", + "source_table": "CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27", + "source_file": "CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx", + "source_sha256": "4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16", + "vintage": "ctas_2026", + "raw_r2_key": "raw/scotgov/scotgov-band-d-council-tax-rates-2026-27/2026/4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16/CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx" + }, + "value": 1436.17 + }, + { + "aggregate_fact_key": "ledger_aggregate_fact_v2_05eec422e83a45552e81d90b", + "semantic_fact_key": "ledger_semantic_fact_v2_79740a105b3b3fde8b9173eb", + "source_record_id": "scotgov.ctas2026.band_d.fy2024.s12000039.average_band_d", + "concept": "scotgov.council_tax.average_band_d", + "measure_id": "average_band_d", + "unit": "gbp", + "aggregation": "mean", + "entity": { + "name": "dwelling", + "role": "chargeable_dwelling" + }, + "geography": { + "id": "S12000039", + "level": "local_authority", + "vintage": "ca_2019" + }, + "period": { + "type": "fiscal_year", + "value": 2024 + }, + "period_coverage": { + "start_date": null, + "end_date": null, + "basis": null + }, + "layout": { + "record_set_id": "scotgov.ctas2026.band_d.fy2024", + "groupby_dimension": "geography", + "groupby_value_id": "s12000039", + "table_record_kind": "detail" + }, + "dimensions": {}, + "source": { + "source_name": "scotgov", + "source_table": "CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27", + "source_file": "CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx", + "source_sha256": "4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16", + "vintage": "ctas_2026", + "raw_r2_key": "raw/scotgov/scotgov-band-d-council-tax-rates-2026-27/2026/4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16/CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx" + }, + "value": 1398.98 + }, + { + "aggregate_fact_key": "ledger_aggregate_fact_v2_81e31b5d9ac130f0956a63db", + "semantic_fact_key": "ledger_semantic_fact_v2_310fbc780806e0b8a5cbfc6c", + "source_record_id": "scotgov.ctas2026.band_d.fy2024.s12000040.average_band_d", + "concept": "scotgov.council_tax.average_band_d", + "measure_id": "average_band_d", + "unit": "gbp", + "aggregation": "mean", + "entity": { + "name": "dwelling", + "role": "chargeable_dwelling" + }, + "geography": { + "id": "S12000040", + "level": "local_authority", + "vintage": "ca_2019" + }, + "period": { + "type": "fiscal_year", + "value": 2024 + }, + "period_coverage": { + "start_date": null, + "end_date": null, + "basis": null + }, + "layout": { + "record_set_id": "scotgov.ctas2026.band_d.fy2024", + "groupby_dimension": "geography", + "groupby_value_id": "s12000040", + "table_record_kind": "detail" + }, + "dimensions": {}, + "source": { + "source_name": "scotgov", + "source_table": "CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27", + "source_file": "CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx", + "source_sha256": "4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16", + "vintage": "ctas_2026", + "raw_r2_key": "raw/scotgov/scotgov-band-d-council-tax-rates-2026-27/2026/4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16/CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx" + }, + "value": 1390.96 + }, + { + "aggregate_fact_key": "ledger_aggregate_fact_v2_11c81cf724405027636e0a51", + "semantic_fact_key": "ledger_semantic_fact_v2_3b4123305c2ece5b26ab3b23", + "source_record_id": "scotgov.ctas2026.band_d.fy2024.s12000041.average_band_d", + "concept": "scotgov.council_tax.average_band_d", + "measure_id": "average_band_d", + "unit": "gbp", + "aggregation": "mean", + "entity": { + "name": "dwelling", + "role": "chargeable_dwelling" + }, + "geography": { + "id": "S12000041", + "level": "local_authority", + "vintage": "ca_2019" + }, + "period": { + "type": "fiscal_year", + "value": 2024 + }, + "period_coverage": { + "start_date": null, + "end_date": null, + "basis": null + }, + "layout": { + "record_set_id": "scotgov.ctas2026.band_d.fy2024", + "groupby_dimension": "geography", + "groupby_value_id": "s12000041", + "table_record_kind": "detail" + }, + "dimensions": {}, + "source": { + "source_name": "scotgov", + "source_table": "CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27", + "source_file": "CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx", + "source_sha256": "4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16", + "vintage": "ctas_2026", + "raw_r2_key": "raw/scotgov/scotgov-band-d-council-tax-rates-2026-27/2026/4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16/CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx" + }, + "value": 1316.68 + }, + { + "aggregate_fact_key": "ledger_aggregate_fact_v2_78391a9b41040e9f52833fd9", + "semantic_fact_key": "ledger_semantic_fact_v2_a667eb8dfa63618fba66d88f", + "source_record_id": "scotgov.ctas2026.band_d.fy2024.s12000042.average_band_d", + "concept": "scotgov.council_tax.average_band_d", + "measure_id": "average_band_d", + "unit": "gbp", + "aggregation": "mean", + "entity": { + "name": "dwelling", + "role": "chargeable_dwelling" + }, + "geography": { + "id": "S12000042", + "level": "local_authority", + "vintage": "ca_2019" + }, + "period": { + "type": "fiscal_year", + "value": 2024 + }, + "period_coverage": { + "start_date": null, + "end_date": null, + "basis": null + }, + "layout": { + "record_set_id": "scotgov.ctas2026.band_d.fy2024", + "groupby_dimension": "geography", + "groupby_value_id": "s12000042", + "table_record_kind": "detail" + }, + "dimensions": {}, + "source": { + "source_name": "scotgov", + "source_table": "CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27", + "source_file": "CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx", + "source_sha256": "4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16", + "vintage": "ctas_2026", + "raw_r2_key": "raw/scotgov/scotgov-band-d-council-tax-rates-2026-27/2026/4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16/CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx" + }, + "value": 1486.43 + }, + { + "aggregate_fact_key": "ledger_aggregate_fact_v2_9d9efa3a05e6397c2fc599b6", + "semantic_fact_key": "ledger_semantic_fact_v2_998c67e39a8381e07ca4ebf5", + "source_record_id": "scotgov.ctas2026.band_d.fy2024.s12000045.average_band_d", + "concept": "scotgov.council_tax.average_band_d", + "measure_id": "average_band_d", + "unit": "gbp", + "aggregation": "mean", + "entity": { + "name": "dwelling", + "role": "chargeable_dwelling" + }, + "geography": { + "id": "S12000045", + "level": "local_authority", + "vintage": "ca_2019" + }, + "period": { + "type": "fiscal_year", + "value": 2024 + }, + "period_coverage": { + "start_date": null, + "end_date": null, + "basis": null + }, + "layout": { + "record_set_id": "scotgov.ctas2026.band_d.fy2024", + "groupby_dimension": "geography", + "groupby_value_id": "s12000045", + "table_record_kind": "detail" + }, + "dimensions": {}, + "source": { + "source_name": "scotgov", + "source_table": "CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27", + "source_file": "CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx", + "source_sha256": "4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16", + "vintage": "ctas_2026", + "raw_r2_key": "raw/scotgov/scotgov-band-d-council-tax-rates-2026-27/2026/4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16/CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx" + }, + "value": 1415.66 + }, + { + "aggregate_fact_key": "ledger_aggregate_fact_v2_e967bc101cae0f416da7a47a", + "semantic_fact_key": "ledger_semantic_fact_v2_28e092a0bbc3ecc690d422a7", + "source_record_id": "scotgov.ctas2026.band_d.fy2024.s12000047.average_band_d", + "concept": "scotgov.council_tax.average_band_d", + "measure_id": "average_band_d", + "unit": "gbp", + "aggregation": "mean", + "entity": { + "name": "dwelling", + "role": "chargeable_dwelling" + }, + "geography": { + "id": "S12000047", + "level": "local_authority", + "vintage": "ca_2019" + }, + "period": { + "type": "fiscal_year", + "value": 2024 + }, + "period_coverage": { + "start_date": null, + "end_date": null, + "basis": null + }, + "layout": { + "record_set_id": "scotgov.ctas2026.band_d.fy2024", + "groupby_dimension": "geography", + "groupby_value_id": "s12000047", + "table_record_kind": "detail" + }, + "dimensions": {}, + "source": { + "source_name": "scotgov", + "source_table": "CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27", + "source_file": "CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx", + "source_sha256": "4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16", + "vintage": "ctas_2026", + "raw_r2_key": "raw/scotgov/scotgov-band-d-council-tax-rates-2026-27/2026/4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16/CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx" + }, + "value": 1385.18 + }, + { + "aggregate_fact_key": "ledger_aggregate_fact_v2_0598ae4ac535b6f09abb0ef1", + "semantic_fact_key": "ledger_semantic_fact_v2_8015c33b482cafbf82186906", + "source_record_id": "scotgov.ctas2026.band_d.fy2024.s12000048.average_band_d", + "concept": "scotgov.council_tax.average_band_d", + "measure_id": "average_band_d", + "unit": "gbp", + "aggregation": "mean", + "entity": { + "name": "dwelling", + "role": "chargeable_dwelling" + }, + "geography": { + "id": "S12000048", + "level": "local_authority", + "vintage": "ca_2019" + }, + "period": { + "type": "fiscal_year", + "value": 2024 + }, + "period_coverage": { + "start_date": null, + "end_date": null, + "basis": null + }, + "layout": { + "record_set_id": "scotgov.ctas2026.band_d.fy2024", + "groupby_dimension": "geography", + "groupby_value_id": "s12000048", + "table_record_kind": "detail" + }, + "dimensions": {}, + "source": { + "source_name": "scotgov", + "source_table": "CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27", + "source_file": "CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx", + "source_sha256": "4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16", + "vintage": "ctas_2026", + "raw_r2_key": "raw/scotgov/scotgov-band-d-council-tax-rates-2026-27/2026/4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16/CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx" + }, + "value": 1403.69 + }, + { + "aggregate_fact_key": "ledger_aggregate_fact_v2_b57a9c50541fcfcc1eb5694e", + "semantic_fact_key": "ledger_semantic_fact_v2_a67c60b16f2faf6e4e333193", + "source_record_id": "scotgov.ctas2026.band_d.fy2024.s12000049.average_band_d", + "concept": "scotgov.council_tax.average_band_d", + "measure_id": "average_band_d", + "unit": "gbp", + "aggregation": "mean", + "entity": { + "name": "dwelling", + "role": "chargeable_dwelling" + }, + "geography": { + "id": "S12000049", + "level": "local_authority", + "vintage": "ca_2019" + }, + "period": { + "type": "fiscal_year", + "value": 2024 + }, + "period_coverage": { + "start_date": null, + "end_date": null, + "basis": null + }, + "layout": { + "record_set_id": "scotgov.ctas2026.band_d.fy2024", + "groupby_dimension": "geography", + "groupby_value_id": "s12000049", + "table_record_kind": "detail" + }, + "dimensions": {}, + "source": { + "source_name": "scotgov", + "source_table": "CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27", + "source_file": "CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx", + "source_sha256": "4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16", + "vintage": "ctas_2026", + "raw_r2_key": "raw/scotgov/scotgov-band-d-council-tax-rates-2026-27/2026/4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16/CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx" + }, + "value": 1499 + }, + { + "aggregate_fact_key": "ledger_aggregate_fact_v2_1bb76588162b6aa7db1b8715", + "semantic_fact_key": "ledger_semantic_fact_v2_2b7a7b534ae2876e01d7faa1", + "source_record_id": "scotgov.ctas2026.band_d.fy2024.s12000050.average_band_d", + "concept": "scotgov.council_tax.average_band_d", + "measure_id": "average_band_d", + "unit": "gbp", + "aggregation": "mean", + "entity": { + "name": "dwelling", + "role": "chargeable_dwelling" + }, + "geography": { + "id": "S12000050", + "level": "local_authority", + "vintage": "ca_2019" + }, + "period": { + "type": "fiscal_year", + "value": 2024 + }, + "period_coverage": { + "start_date": null, + "end_date": null, + "basis": null + }, + "layout": { + "record_set_id": "scotgov.ctas2026.band_d.fy2024", + "groupby_dimension": "geography", + "groupby_value_id": "s12000050", + "table_record_kind": "detail" + }, + "dimensions": {}, + "source": { + "source_name": "scotgov", + "source_table": "CTAS 2026 Council Tax Assumptions: Band D council tax rates by year, 1996-97 to 2026-27", + "source_file": "CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx", + "source_sha256": "4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16", + "vintage": "ctas_2026", + "raw_r2_key": "raw/scotgov/scotgov-band-d-council-tax-rates-2026-27/2026/4e56c333178733c63d8729608740fbf7bf3b3d62ae58c3ad9ea297ad8188cf16/CTAS_2026_Band_D_Council_Tax_rates_1996_97_to_2026_27.xlsx" + }, + "value": 1320.78 + } + ] +} diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index 36586bd836..d76d599410 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -372,11 +372,13 @@ "liable_for_share_of_household_rent", "household_id", "region", + "imd_decile", "tenure_type", "accommodation_type", "num_bedrooms", "council_tax_reported", "council_tax_band", + "council_tax_band_observed", "council_tax_rebate", "council_tax_single_adult_raw", "water_and_sewerage_charges", @@ -431,7 +433,7 @@ "frs_company_car_fuel_code", "frs_company_car_contribution" ], - "notes": "Root E2 spine assembly. It carries direct raw mappings only; education-grant aggregate and council-tax reported fields remain raw carriers for E3. Benefit take-up, BRMA/LHA assignment, stochastic flags, and imputations are intentionally absent from this stage, except two identity-keyed draws. One is the UC start-up claim recency (uk-data#527). The other is each person's months_since_last_birthday, which policyengine-uk reads to place a date of birth (the two-child limit, the State Pension scheme, the Pension Credit qualifying age). Each single year of age and sex is spread evenly over the year at the FRS weights, in an order set by a draw on the person's stable id, as the engine spreads it when the column is unset, and the column is stored so every later engine read, the calibration's and the release's included, takes the same months whatever weights or identifiers the file carries (microcosm#1095). The UC start-up period needs the raw claim, interview and job dates only this stage reads, so a UC record without a linked claim date takes its claim recency from a benefit-unit-keyed draw at the linked share. The 2024-25 househol.tab is not sernum-ordered; the declared household sort guards stable identity. care_hours carries the FRS banded care-hours code as weekly hours (lower band edge), and frs_care_hours_in_household the largest of the four in-household care bands (HOURAB, HOURAH, HOURCB, HOURCH) the same way, a stage-internal carrier the Carer's Allowance draw reads and the release drops (microcosm#1418); would_claim_carers_allowance follows reported receipt, so the engine keeps Carer's Allowance on reported claimants while the hours qualify carers for the Universal Credit carer element (#882); after the SPI income chain, carers_allowance_take_up opens the flag to carers the earnings limit alone keeps from the allowance (microcosm#1319). SEINCAM2, the self-employment income the respondent gives, splits by its sign (uk-data#528), so profits go to self_employment_income and losses to trading_loss as a positive amount, the two inputs policyengine-uk reads from 2.131.0 (pe-uk#2085). The engine uprates trading_loss into each projected year as it does self_employment_income, so a loss persists like a profit; the engine documents that a one-off loss would be set to zero in later years, which a one-year survey cannot tell apart. Six of policyengine-uk's eleven owned-company inputs (pe-uk#1965) come from the FRS job tab (microcosm#1095). A director (DIRECTOR 1 or NATURE 1) who owns at least half of the company (DIRPERC 1 to 3, the ruling of 2026-10-16) stands as its sole owner or partner and works in it. The company is their main employment when the job is their first (JOBTYPE 1); it carries on a property business when the main job's SIC division is 68 and a trade otherwise, and its income share is the dividends the job records (DIVIDEND, after tax), the part of the company's profit the survey sees. A company held in another job carries on neither, since ADULT.SIC codes the main job only, which leaves the engine's company-owner rules off for it. The stocks and the intermediaries inputs keep the engine's defaults. frs_company_car_price_band, frs_company_car_fuel_code, frs_company_car_private_fuel and frs_company_car_contribution carry each person's company car from the FRS job tab (CARVAL, FUELTYP, EXPBEN03, and CARAMT where CARCON is 1, on their EXPBEN01 job with the lowest JOBTYPE), stage-internal carriers the company_cars stage reads and the release drops (PolicyEngine/uk-autumn-budget-dashboard-2026#316). frs_uc_lcw_status carries DWP's limited capability for work status on a linked Universal Credit claim (BENEFITS.UCLCW, coded 0 None, 1 LCFW Group and 2 LCFWRA Group in SN 9563's UKDA data dictionary; blank, as UCSTART is, where the claim has no linked record) on the UC row's person, NOT_LINKED for everyone else; the FRS keeps one UC row per claim, couples included, so the status is the claim's. It is a stage-internal carrier the uc_limited_capability stage reads and the release drops (microcosm#1095, Y1). frs_uc_carer_element carries DWP's carer element on a linked Universal Credit claim (BENEFITS.UCCAREEL, which SN 9563's UKDA data dictionary labels the weekly carer element amount and gives no value labels; the 2024-25 file codes it 0 or 1 on every linked claim, read as NONE or CARER on a claim with a linked record, UCSTART present, and NOT_LINKED on any other) on the UC row's person, NOT_LINKED for everyone else; frs_disability reads it only to check the carer link against DWP's record, and the release drops it (A3 (f) of the November event-data plan's PR3, microcosm#1095). frs_care_recipients carries each person's main care recipients from the FRS care tab (CARE.NEEDPER, WHOLOO01 to WHOLOO14 and HOUR01 to HOUR14, coded as HOURTOT is) as a bitmask, with bit n for household member n among the people given the person's largest weekly band and bit 15 for anyone outside the household among them; frs_disability reads it to write cares_for_severely_disabled_person, and the release drops it (A3 of the November event-data plan's PR3, microcosm#1095)." + "notes": "Root E2 spine assembly. It carries direct raw mappings only; education-grant aggregate and council-tax reported fields remain raw carriers for E3. Benefit take-up, BRMA/LHA assignment, stochastic flags, and imputations are intentionally absent from this stage, except two identity-keyed draws. One is the UC start-up claim recency (uk-data#527). The other is each person's months_since_last_birthday, which policyengine-uk reads to place a date of birth (the two-child limit, the State Pension scheme, the Pension Credit qualifying age). Each single year of age and sex is spread evenly over the year at the FRS weights, in an order set by a draw on the person's stable id, as the engine spreads it when the column is unset, and the column is stored so every later engine read, the calibration's and the release's included, takes the same months whatever weights or identifiers the file carries (microcosm#1095). The UC start-up period needs the raw claim, interview and job dates only this stage reads, so a UC record without a linked claim date takes its claim recency from a benefit-unit-keyed draw at the linked share. The 2024-25 househol.tab is not sernum-ordered; the declared household sort guards stable identity. care_hours carries the FRS banded care-hours code as weekly hours (lower band edge), and frs_care_hours_in_household the largest of the four in-household care bands (HOURAB, HOURAH, HOURCB, HOURCH) the same way, a stage-internal carrier the Carer's Allowance draw reads and the release drops (microcosm#1418); would_claim_carers_allowance follows reported receipt, so the engine keeps Carer's Allowance on reported claimants while the hours qualify carers for the Universal Credit carer element (#882); after the SPI income chain, carers_allowance_take_up opens the flag to carers the earnings limit alone keeps from the allowance (microcosm#1319). SEINCAM2, the self-employment income the respondent gives, splits by its sign (uk-data#528), so profits go to self_employment_income and losses to trading_loss as a positive amount, the two inputs policyengine-uk reads from 2.131.0 (pe-uk#2085). The engine uprates trading_loss into each projected year as it does self_employment_income, so a loss persists like a profit; the engine documents that a one-off loss would be set to zero in later years, which a one-year survey cannot tell apart. Six of policyengine-uk's eleven owned-company inputs (pe-uk#1965) come from the FRS job tab (microcosm#1095). A director (DIRECTOR 1 or NATURE 1) who owns at least half of the company (DIRPERC 1 to 3, the ruling of 2026-10-16) stands as its sole owner or partner and works in it. The company is their main employment when the job is their first (JOBTYPE 1); it carries on a property business when the main job's SIC division is 68 and a trade otherwise, and its income share is the dividends the job records (DIVIDEND, after tax), the part of the company's profit the survey sees. A company held in another job carries on neither, since ADULT.SIC codes the main job only, which leaves the engine's company-owner rules off for it. The stocks and the intermediaries inputs keep the engine's defaults. frs_company_car_price_band, frs_company_car_fuel_code, frs_company_car_private_fuel and frs_company_car_contribution carry each person's company car from the FRS job tab (CARVAL, FUELTYP, EXPBEN03, and CARAMT where CARCON is 1, on their EXPBEN01 job with the lowest JOBTYPE), stage-internal carriers the company_cars stage reads and the release drops (PolicyEngine/uk-autumn-budget-dashboard-2026#316). frs_uc_lcw_status carries DWP's limited capability for work status on a linked Universal Credit claim (BENEFITS.UCLCW, coded 0 None, 1 LCFW Group and 2 LCFWRA Group in SN 9563's UKDA data dictionary; blank, as UCSTART is, where the claim has no linked record) on the UC row's person, NOT_LINKED for everyone else; the FRS keeps one UC row per claim, couples included, so the status is the claim's. It is a stage-internal carrier the uc_limited_capability stage reads and the release drops (microcosm#1095, Y1). frs_uc_carer_element carries DWP's carer element on a linked Universal Credit claim (BENEFITS.UCCAREEL, which SN 9563's UKDA data dictionary labels the weekly carer element amount and gives no value labels; the 2024-25 file codes it 0 or 1 on every linked claim, read as NONE or CARER on a claim with a linked record, UCSTART present, and NOT_LINKED on any other) on the UC row's person, NOT_LINKED for everyone else; frs_disability reads it only to check the carer link against DWP's record, and the release drops it (A3 (f) of the November event-data plan's PR3, microcosm#1095). frs_care_recipients carries each person's main care recipients from the FRS care tab (CARE.NEEDPER, WHOLOO01 to WHOLOO14 and HOUR01 to HOUR14, coded as HOURTOT is) as a bitmask, with bit n for household member n among the people given the person's largest weekly band and bit 15 for anyone outside the household among them; frs_disability reads it to write cares_for_severely_disabled_person, and the release drops it (A3 of the November event-data plan's PR3, microcosm#1095). imd_decile, each household's decile on its own nation's index of multiple deprivation (IMDE, IMDW, IMDS or IMDN), 1 to 10, and 0 where none is observed or the region is unknown, and council_tax_band_observed, False in Northern Ireland, for a CTBAND code outside 1 to 9, for a band the household's region does not use and for an unknown region (council_tax_band itself is unchanged), are stage-internal carriers the clone-first spine's area assignment conditions on and the release drops; they stay out of every receipt and of the spine sidecar's nonzero shares, and a household tab without one of the four IMD variables is refused (microcosm#931)." }, { "stage": "age_tail", @@ -845,15 +847,21 @@ "stored": "the Scottish households' council_tax after impute_cell_means, at the spine design weights", "factor": "level / stored, one factor on every Scottish household's council_tax; a frame with no Scottish bill to level keeps its bills as derived", "weights": "spine_design_weights" + }, + { + "formula": "first match: not_levied where raw gvtregno == 13 (Northern Ireland, which pays domestic rates), whatever branch its bill takes; else the branch derive_council_tax takes: missing where the bill is missing (raw ctannual < 0 or NaN, or ctannual == 0 with ctpaid == 1 and ctreb != 1); observed_gross where ctreb == 1 and ctrebamt > 0 (the reported reduction added back); unknown_reduction where ctreb == 1 (the larger of the netted ctannual and the cell mean); else observed_gross (the netted ctannual, kept at zero where ctannual == 0 and ctpaid != 1, as with ctpaid 2 and 3)", + "kind": "derive", + "output": "council_tax_bill_provenance" } ], "outputs": [ - "council_tax" + "council_tax", + "council_tax_bill_provenance" ], "nonnegative_outputs": [ "council_tax" ], - "notes": "Re-reads raw househol.tab because spine council_tax_reported clips missing values. FRS 2024-25 CTANNUAL is the bill after discounts and council tax reduction, and the engine reads council_tax before the reduction, so a household reporting a reduction gets its CTREBAMT added back and the cells pool non-recipients only (uk-data#496/#499, microcosm#1095). Scottish water and sewerage charges are netted before cell means at the shares DWP's CTANNUAL carries (the gross charges in full for non-recipients, 65% for reduction recipients); no-donor cells impute zero. A household where someone pays council tax (CTPAID 1) with no reported reduction and a zero CTANNUAL owes a bill, so it takes its cell mean as a missing bill does (microcosm#1278). The dead ct_mean.replace(-1, ...) branch is intentionally dropped. Last, the Scottish bills are levelled with one factor to the Scottish Government's own measure of the 2024-25 liability before council tax reduction (S2(a) of the November event-data plan, microcosm#1095; María's signature of 2026-11-09): the sum over the 32 council areas of CTAXBASE 2025's Band D equivalent dwellings before council tax reduction at September 2024, the middle of 2024-25, times CTAS 2026's average Band D council tax for 2024-25, over the Scottish households' bills at the spine design weights. Band D equivalents count dwellings after discounts and exemptions, as the bills do. The bound Scottish band stock (scotgov.council_tax_stock) already gives the household mix, so the level moves the bills' amounts and no weight; obr.council_tax_scotland stays a held-out diagnostic. A frame with no Scottish bill to level keeps its bills as derived, and the stage receipt says so. The liability counts every chargeable dwelling, including about 21,600 second homes and 43,500 empty homes liable for council tax (empty more than 6 months) in September 2024, which no FRS household occupies (Scottish Government, Second Homes and Empty Properties in September 2025, from the councils' CTAXBASE returns). So the level is the liability times Scotland's main-residence share at September 2024: one less those dwellings over CTAXBASE 2024's 2.60 million chargeable dwellings, about 97.5% (B11 of the November event-data plan's PR3; Vahid's second review of microcosm#1550; María's signature of 2026-11-11). The share is taken in dwellings, since no published table splits Band D equivalents by class, and the stage receipt records the counts and the share. It nets out less than these dwellings' whole liability: councils can charge second homes and long-term empty homes a premium of up to double the rate, and homes empty 6 months or less, also liable, are not counted." + "notes": "Re-reads raw househol.tab because spine council_tax_reported clips missing values. FRS 2024-25 CTANNUAL is the bill after discounts and council tax reduction, and the engine reads council_tax before the reduction, so a household reporting a reduction gets its CTREBAMT added back and the cells pool non-recipients only (uk-data#496/#499, microcosm#1095). Scottish water and sewerage charges are netted before cell means at the shares DWP's CTANNUAL carries (the gross charges in full for non-recipients, 65% for reduction recipients); no-donor cells impute zero. A household where someone pays council tax (CTPAID 1) with no reported reduction and a zero CTANNUAL owes a bill, so it takes its cell mean as a missing bill does (microcosm#1278). The dead ct_mean.replace(-1, ...) branch is intentionally dropped. It also derives council_tax_bill_provenance, the branch each bill takes (not_levied in Northern Ireland, whatever its branch); it is dropped at the release boundary, and council_tax_localise alone reads it. Last, the Scottish bills are levelled with one factor to the Scottish Government's own measure of the 2024-25 liability before council tax reduction (S2(a) of the November event-data plan, microcosm#1095; María's signature of 2026-11-09): the sum over the 32 council areas of CTAXBASE 2025's Band D equivalent dwellings before council tax reduction at September 2024, the middle of 2024-25, times CTAS 2026's average Band D council tax for 2024-25, over the Scottish households' bills at the spine design weights. Band D equivalents count dwellings after discounts and exemptions, as the bills do. The bound Scottish band stock (scotgov.council_tax_stock) already gives the household mix, so the level moves the bills' amounts and no weight; obr.council_tax_scotland stays a held-out diagnostic. A frame with no Scottish bill to level keeps its bills as derived, and the stage receipt says so. The liability counts every chargeable dwelling, including about 21,600 second homes and 43,500 empty homes liable for council tax (empty more than 6 months) in September 2024, which no FRS household occupies (Scottish Government, Second Homes and Empty Properties in September 2025, from the councils' CTAXBASE returns). So the level is the liability times Scotland's main-residence share at September 2024: one less those dwellings over CTAXBASE 2024's 2.60 million chargeable dwellings, about 97.5% (B11 of the November event-data plan's PR3; Vahid's second review of microcosm#1550; María's signature of 2026-11-11). The share is taken in dwellings, since no published table splits Band D equivalents by class, and the stage receipt records the counts and the share. It nets out less than these dwellings' whole liability: councils can charge second homes and long-term empty homes a premium of up to double the rate, and homes empty 6 months or less, also liable, are not counted." }, { "stage": "frs_disability", @@ -1065,399 +1073,91 @@ "notes": "Runs before BRMA and always runs; the DSA seed evaluates at the release calibration year (uk-data#478) and the modelling-start gate compares against that year, not the frame period. EDUCQUAL grant-split compatibility is re-cited to the FRS 2024-25 SN 9563 dictionary; codes 1-87 are empirically present and code 87 remains undocumented/defaulted." }, { - "stage": "frs_take_up", + "stage": "frs_hmrc_spine_leaves", "survey": "Family Resources Survey 2024-25", - "source": "Family Resources Survey 2024-25 reported receipt anchors plus sourced UK take-up contract rates.", - "grain": "benunit", - "artifacts": [], - "operations": [ - { - "kind": "aggregate_person_to_benunit", - "method": "any_positive", - "consumed_only": true, - "aggregates": { - "child_benefit_reported_anchor": "child_benefit_reported", - "pension_credit_reported_anchor": "pension_credit_reported", - "universal_credit_reported_anchor": "universal_credit_reported" - } - }, - { - "kind": "aggregate_person_to_benunit", - "method": "any_uc_claimant_aged_18_under_pension_credit_age", - "consumed_only": true, - "aggregates": { - "uc_age_eligible": "age" - }, - "role_column": "is_uc_claimant" - }, - { - "kind": "assign_binary_with_anchored_residual", - "output": "would_claim_child_benefit", - "draw": "would_claim_child_benefit", - "rate_key": "child_benefit", - "anchor": "child_benefit_reported_anchor", - "seed": 0 - }, - { - "kind": "assign_binary_from_rate", - "output": "child_benefit_opts_out", - "draw": "child_benefit_opts_out", - "rate_key": "child_benefit_opts_out_rate", - "seed": 0 - }, - { - "kind": "assign_binary_with_anchored_residual", - "output": "would_claim_pc", - "draw": "would_claim_pc", - "rate_key": "pension_credit", - "anchor": "pension_credit_reported_anchor", - "seed": 0 - }, - { - "kind": "assign_binary_with_anchored_residual", - "output": "would_claim_uc", - "draw": "would_claim_uc", - "rate_key": "universal_credit", - "anchor": "universal_credit_reported_anchor", - "population": "uc_age_eligible", - "seed": 0 - }, - { - "kind": "assign_binary_from_rate", - "output": "would_claim_tfc", - "draw": "would_claim_tfc", - "rate_key": "tax_free_childcare", - "seed": 0 - }, - { - "kind": "assign_binary_from_rate", - "output": "would_claim_extended_childcare", - "draw": "would_claim_extended_childcare", - "rate_key": "extended_childcare", - "seed": 0 - }, - { - "kind": "assign_binary_from_rate", - "output": "would_claim_universal_childcare", - "draw": "would_claim_universal_childcare", - "rate_key": "universal_childcare", - "nested_under": "would_claim_extended_childcare", - "rate_derivation": "the contract's universal_childcare rate is the rate among benefit units not claiming the extended hours; would_claim_universal_childcare = would_claim_extended_childcare OR u < rate, so every unit claiming the extended hours claims the universal hours", - "seed": 0 - }, + "source": "Department for Work and Pensions Family Resources Survey 2024-25 raw adult.tab and benefits.tab, caller-supplied local input", + "grain": "person", + "artifacts": [ { - "kind": "assign_binary_from_rate", - "output": "would_claim_targeted_childcare", - "draw": "would_claim_targeted_childcare", - "rate_key": "targeted_childcare", - "seed": 0 + "role": "frs_table", + "table": "adult", + "kind": "licensed_microdata", + "format": "tab", + "vintage": "2024_25", + "locator": "adult.tab", + "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", + "size_bytes": 34885825, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { - "kind": "assign_binary_from_banded_rates", - "output": "would_claim_uc_childcare", - "draw": "would_claim_uc_childcare", - "band_column": "is_married", - "bands": [ - { - "upper_exclusive": 1, - "rate_key": "uc_childcare_single" - }, - { - "rate_key": "uc_childcare_couple" - } - ], - "seed": 0 + "role": "frs_table", + "table": "benefits", + "kind": "licensed_microdata", + "format": "tab", + "vintage": "2024_25", + "locator": "benefits.tab", + "sha256": "f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3", + "size_bytes": 2362329, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { - "kind": "assign_clipped_normal", - "output": "maximum_extended_childcare_hours_usage", - "draw": "maximum_extended_childcare_hours_usage", - "distribution_key": "maximum_extended_childcare_hours_usage", - "seed": 0 + "role": "frs_table", + "table": "job", + "kind": "licensed_microdata", + "format": "tab", + "vintage": "2024_25", + "locator": "job.tab", + "sha256": "eb7faf7ada3a3851cb2afb83e2983f8907ffeec897cfbe01e56cb0dfefa853e2", + "size_bytes": 10518760, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { - "kind": "assign_binary_from_rate", - "output": "max_free_entitlement_hours_used", - "draw": "max_free_entitlement_hours_used", - "rate_key": "extended_childcare_age_0_usage", - "eligibility": "persons of DfE age 0 in benefit units that claim the working-parent hours (would_claim_extended_childcare)", - "assignment_unit": "person", - "rate_derivation": "a child of DfE age 0 in a claiming unit uses the working-parent hours where u < its age's rate and gets max_free_entitlement_hours_used 0 otherwise; every other person keeps the engine's default of 30; one draw per person serves both ages' operations", - "seed": 0 + "role": "frs_table", + "table": "penprov", + "kind": "licensed_microdata", + "format": "tab", + "vintage": "2024_25", + "locator": "penprov.tab", + "sha256": "9e53de0dc969baec000b3cd68387f0f2dfb3f678732e408de175e0a1d6e3fdc1", + "size_bytes": 513614, + "runtime_sha256_required": true, + "tax_year_start": 2024, + "ukds_study_number": 9563, + "doi": "10.5255/UKDA-SN-9563-1" }, { - "kind": "assign_binary_from_rate", - "output": "max_free_entitlement_hours_used", - "draw": "max_free_entitlement_hours_used", - "rate_key": "extended_childcare_age_1_usage", - "eligibility": "persons of DfE age 1 in benefit units that claim the working-parent hours (would_claim_extended_childcare)", - "assignment_unit": "person", - "rate_derivation": "a child of DfE age 1 in a claiming unit uses the working-parent hours where u < its age's rate and gets max_free_entitlement_hours_used 0 otherwise; every other person keeps the engine's default of 30; one draw per person serves both ages' operations", - "seed": 0 - } - ], - "outputs": [ - "would_claim_child_benefit", - "child_benefit_opts_out", - "would_claim_pc", - "would_claim_uc", - "would_claim_tfc", - "would_claim_extended_childcare", - "would_claim_universal_childcare", - "would_claim_targeted_childcare", - "would_claim_uc_childcare", - "maximum_extended_childcare_hours_usage", - "max_free_entitlement_hours_used" - ], - "nonnegative_outputs": [ - "maximum_extended_childcare_hours_usage", - "max_free_entitlement_hours_used" - ], - "notes": "Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100; salts are output variable names. Reported positive receipt anchors are transient and consumed only. The Universal Credit draw's population is the benefit units with a UC claimant or partner (is_uc_claimant) aged 18 or over and under Pension Credit qualifying age (66 for everyone who can reach it in this release), the age and role conditions of policyengine-uk 2.177.7's is_uc_eligible; the engine lockstep test pins it and names the engine cases not mirrored (16- and 17-year-old claimants under the UC Regs 2013 reg 8 exceptions, mixed-age couples on the Pension Credit route, from 2.128.0 pension-age families moved onto Universal Credit when their legacy benefit closes, SI 2014/1230 reg 60A, and from 2.173.2 a unit whose working-age claimants all receive education with no exception, since frs_education sets is_in_full_time_education from the FRS). So the contract rate is a share of those units, an anchor outside them stays true, and any other unit is never drawn; a unit with no claimant or partner of that age keeps its reported Housing Benefit, which a claim it cannot make no longer zeroes (#882, microcosm#867, microcosm#1095). The universal childcare flag is nested under the working-parent one. A unit that claims the extended hours claims the universal hours, and any other unit claims them at the contract's universal_childcare rate, the rate conditional on not claiming the extended hours; the extended draw is unchanged. childcare_copy_take_up later re-draws the childcare flags and hours of the support and CGT copies on their own benefit units. A child of DfE age 0 or 1 (its age on 31 December of the fit's model year, the age DfE counts by) in a unit that claims the working-parent hours uses them at its age's contract rate, extended_childcare_age_0_usage or extended_childcare_age_1_usage (DfE registers far fewer of the youngest eligible children than of those aged 2 to 4, and fewer at 9 to 11 months than at age 1); a child that does not gets max_free_entitlement_hours_used 0, so the engine funds it no hours, and every other person keeps the engine's default of 30." - }, - { - "stage": "frs_person_draws", - "survey": "Family Resources Survey 2024-25", - "source": "Sourced UK take-up contract rates and identity-keyed deterministic draws.", - "grain": "person", - "artifacts": [ - { - "role": "stochastic_contract", - "resource": "take_up_contract.json", + "role": "employer_rate_bands", "kind": "public_aggregate_reference", - "format": "json" + "resource": "ashe_employer_pension_contribution_rates.json", + "format": "json", + "runtime_sha256_required": true } ], "operations": [ { - "kind": "assign_binary_from_rate", - "output": "would_claim_marriage_allowance", - "draw": "would_claim_marriage_allowance", - "rate_key": "marriage_allowance", - "seed": 0 - }, - { - "kind": "assign_binary_from_banded_rates", - "output": "would_claim_scp", - "draw": "would_claim_scp", - "band_column": "age", - "bands": [ - { - "upper_exclusive": 6, - "rate_key": "scp_under_6" - }, - { - "rate_key": "scp_6_plus" - } - ], - "seed": 0 - }, - { - "kind": "assign_uniform_draw", - "output": "attends_private_school_random_draw", - "seed": 0 - }, - { - "kind": "assign_period_constant", - "output": "tax_free_childcare_spend_routed_share", - "contract_key": "tax_free_childcare_spend_routed_share", - "period_source": "build_year", - "dtype": "float64" - }, - { - "kind": "assign_binary_from_rate", - "output": "is_apprentice", - "draw": "is_apprentice", - "rate_key": "apprentice_stock_19_to_24", - "assignment_unit": "person", - "eligibility": "employees (employment_status FT_EMPLOYED or PT_EMPLOYED, from frs_employment) aged 21 to 24 who are not in full-time education (is_in_full_time_education, from frs_education; the FRS codes no apprentice as in full-time education): SSCBA 1992 s.9A already zero-rates every earner under 21 and s.9B ends the apprentice zero rate at 25, so no other age changes employer NICs, and leaving them False keeps is_in_non_advanced_education, minimum_wage, Care to Learn and the 16 to 19 Bursary Fund unchanged; the flagged keep their age-related minimum_wage too, because policyengine-uk 2.162.8 (policyengine-uk#2683) gives the apprentice rate only under 19 or with is_in_first_year_of_apprenticeship, which the build leaves at its default (False), and the stage refuses an engine without that split or the s.9B threshold", - "rate_derivation": "the contract's apprentice_stock_19_to_24 is DfE's England apprentices in learning at the start of the academic year (participation less starts) per head aged 19 to 24, a point-in-time count taken as uniform across those ages and UK-wide; the target is r times the weighted population aged 21 to 24 at the frame's household weights, and eligible employees are flagged in the order of their identity-keyed uniform until their weight is nearest it (uc_take_up's rule), or all of them when they weigh no more than it; the receipt records the target, the flagged weight, the support bands and the relief the flags imply at the installed engine's Class 1 schedule, and uk_stage_frs_person_draws_apprentices gates them, the relief at or below HMRC's NIC - NS1", - "seed": 0 - } - ], - "outputs": [ - "would_claim_marriage_allowance", - "would_claim_scp", - "attends_private_school_random_draw", - "tax_free_childcare_spend_routed_share", - "is_apprentice" - ], - "nonnegative_outputs": [ - "attends_private_school_random_draw", - "tax_free_childcare_spend_routed_share" - ], - "notes": "Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100; higher_earner_tie_break is intentionally not produced. is_apprentice has no FRS source (no apprenticeship variable in the 2022-23 to 2024-25 adult tables; TRAIN2, NI3TRAIN and TRAINEE code government employment schemes) and is drawn to DfE's point-in-time rate for employees aged 21 to 24 only." - }, - { - "stage": "frs_household_draws", - "survey": "Family Resources Survey 2024-25", - "source": "Sourced UK stochastic contract rates and identity-keyed deterministic draws.", - "grain": "household", - "artifacts": [], - "operations": [ - { - "kind": "assign_binary_from_rate", - "output": "household_owns_tv", - "draw": "household_owns_tv", - "rate_key": "tv_ownership_rate", - "seed": 0 - }, - { - "kind": "assign_binary_from_rate", - "output": "would_evade_tv_licence_fee", - "draw": "would_evade_tv_licence_fee", - "rate_key": "tv_licence_evasion_rate", - "seed": 0 - }, - { - "kind": "assign_binary_from_rate", - "output": "main_residential_property_purchased_is_first_home", - "draw": "main_residential_property_purchased_is_first_home", - "rate_key": "first_time_buyer_rate", - "seed": 0 - }, - { - "kind": "assign_binary_from_rate", - "output": "property_purchased", - "draw": "property_purchased", - "rate_key": "property_purchase_rate", - "seed": 0 - } - ], - "outputs": [ - "household_owns_tv", - "would_evade_tv_licence_fee", - "main_residential_property_purchased_is_first_home", - "property_purchased" - ], - "notes": "Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100. TV evasion remains an independent draw, matching the incumbent reference share." - }, - { - "stage": "frs_brma", - "survey": "Family Resources Survey 2024-25", - "source": "Census private-rented households by region, BRMA and bedrooms (ONS and NISRA 2021, NRS 2022) and policyengine-uk LHA_category predictor.", - "grain": "household", - "artifacts": [ - { - "role": "count_resource", - "resource": "brma_private_rented_households.json", - "kind": "public_aggregated_counts", - "format": "json" - } - ], - "operations": [ - { - "kind": "materialize_rules_engine_predictors", - "predictors": [ - "LHA_category" - ], - "consumed_only": true - }, - { - "kind": "sample_categorical_from_count_table", - "output": "brma", - "count_resource": "brma_private_rented_households.json", - "cell_columns": [ - "region", - "LHA_category" - ], - "assignment_grain": "benunit", - "collapse_to": "household", - "collapse_method": "uniform_member_pick", - "seed": 0 - } - ], - "outputs": [ - "brma" - ], - "notes": "Identity-keyed seed 0 streams replace the incumbent BRMA seed 0 sequential generator. Household collapse uses salt brma:household_pick." - }, - { - "stage": "frs_hmrc_spine_leaves", - "survey": "Family Resources Survey 2024-25", - "source": "Department for Work and Pensions Family Resources Survey 2024-25 raw adult.tab and benefits.tab, caller-supplied local input", - "grain": "person", - "artifacts": [ - { - "role": "frs_table", - "table": "adult", - "kind": "licensed_microdata", - "format": "tab", - "vintage": "2024_25", - "locator": "adult.tab", - "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", - "size_bytes": 34885825, - "runtime_sha256_required": true, - "tax_year_start": 2024, - "ukds_study_number": 9563, - "doi": "10.5255/UKDA-SN-9563-1" - }, - { - "role": "frs_table", - "table": "benefits", - "kind": "licensed_microdata", - "format": "tab", - "vintage": "2024_25", - "locator": "benefits.tab", - "sha256": "f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3", - "size_bytes": 2362329, - "runtime_sha256_required": true, - "tax_year_start": 2024, - "ukds_study_number": 9563, - "doi": "10.5255/UKDA-SN-9563-1" - }, - { - "role": "frs_table", - "table": "job", - "kind": "licensed_microdata", - "format": "tab", - "vintage": "2024_25", - "locator": "job.tab", - "sha256": "eb7faf7ada3a3851cb2afb83e2983f8907ffeec897cfbe01e56cb0dfefa853e2", - "size_bytes": 10518760, - "runtime_sha256_required": true, - "tax_year_start": 2024, - "ukds_study_number": 9563, - "doi": "10.5255/UKDA-SN-9563-1" - }, - { - "role": "frs_table", - "table": "penprov", - "kind": "licensed_microdata", - "format": "tab", - "vintage": "2024_25", - "locator": "penprov.tab", - "sha256": "9e53de0dc969baec000b3cd68387f0f2dfb3f678732e408de175e0a1d6e3fdc1", - "size_bytes": 513614, - "runtime_sha256_required": true, - "tax_year_start": 2024, - "ukds_study_number": 9563, - "doi": "10.5255/UKDA-SN-9563-1" - }, - { - "role": "employer_rate_bands", - "kind": "public_aggregate_reference", - "resource": "ashe_employer_pension_contribution_rates.json", - "format": "json", - "runtime_sha256_required": true - } - ], - "operations": [ - { - "kind": "retain_adjudicated_frs_hmrc_leaves", - "population": "uk_frs_raw_spine", - "source_vintage": "2024-25", - "mapped_build_period": 2024, - "annualization": "weekly raw FRS amounts * 52", - "status": "adjudicated_partial_replay", - "retained_full_constituents": { - "hmrc_spi_pay": { - "spi_concept": "PAY", - "scope": "full", - "raw_sources": [ - "ADULT.INEARNS" - ], - "formula": "max(0, ADULT.INEARNS) * 52" + "kind": "retain_adjudicated_frs_hmrc_leaves", + "population": "uk_frs_raw_spine", + "source_vintage": "2024-25", + "mapped_build_period": 2024, + "annualization": "weekly raw FRS amounts * 52", + "status": "adjudicated_partial_replay", + "retained_full_constituents": { + "hmrc_spi_pay": { + "spi_concept": "PAY", + "scope": "full", + "raw_sources": [ + "ADULT.INEARNS" + ], + "formula": "max(0, ADULT.INEARNS) * 52" }, "hmrc_spi_unemployment_benefit_income": { "spi_concept": "UBISJA", @@ -1669,86 +1369,693 @@ "notes": "Describes each FRS adult's 12 months before the interview beside the current income columns, which it never changes, so the HMRC income-band measures can read annual income while poverty, benefits and the published engine tax keep current income. HMRC counts income received over a tax year: a person who started, left or changed a job, or had a gap, sits in another HMRC band than current pay puts them in, or in none. Outputs: the share of the year paid at today's employee pay and annual pay from other spells (chiefly a recent leaver's last job), the same pair for self-employment, and the shares of the year a private pension started on retiring and contributory JSA, ESA and Incapacity Benefit started on leaving work were in payment; an annual amount is the current amount times its share plus the other spells. The State Pension needs none: policyengine-uk's months_since_state_pension_age gives its share. Months come from CURRJOBM and JOB.WRKPREV for people in work (a person in work is never a leaver), the SAMESIT/SDEMP calendar only where it records a change, and LSTYR, LSTWRK1/2 or RETIRE1 for people out of work who worked in the last 12 months (NUMJOB2 and their last-job JOB rows); a leaving date 12 months or more before the interview gives none. A leaver's pay is the last job's usual gross pay (JOB.UGROSS, asked of people out of work who last worked in the previous 12 months, in respect of that job), scaled to the DV concept by the weighted job-level UGRSPAY/UGROSS ratio on current employees who report both and uprated by the pinned OBR average earnings index for the months since leaving; otherwise a donor's weekly pay, drawn by GROSS4 weight from current employees in their first 24 months in the job within age band x sex x full- or part-time cells (sex dropped below 30 donors), on an identity-keyed uniform (seed 0, salt employment_income_other_spells). Self-employment keeps the whole year where SEJBLONG is one or more, (SEJBMTHS + 1)/12 under a year and at most the calendar's self-employed months where self-employment is the activity today (the calendar records the main activity only, so a side business beside a job keeps its whole year); a ceased self-employed last job pays SEINCAMT (else PROFIT1, a loss as zero) for its SELWKS weeks; SEINCAM2 is already uprated to the interview by the FRS, so it is not uprated again. The receipt gives each source's support as a band of distinct households, ratios only where three or more households carry them, and the exact zeros its stage gate reads." }, { - "stage": "spi_support_channel", - "survey": "Family Resources Survey 2024-25", - "source": "Synthetic SPI support channel sampled uniformly without replacement from the raw FRS spine before cloning.", - "grain": "household", "artifacts": [], + "grain": "household", + "nonnegative_outputs": [ + "person_clone_index", + "benunit_clone_index", + "household_clone_index", + "source_household_id", + "source_year" + ], + "notes": "Gives every FRS household K geographic clones before any other copy step (microcosm#931), so that each clone can later take an area of its own while together they stand for the survey household once. Clone 0 is the frame's own rows and ids; clone k repeats every person, benefit unit and household with each id minted in the copy-id table's geographic_support field at ordinal k (id_remapping), and every table carries its rows' clone, 0 to K - 1, in its clone index column. Each clone carries an equal share of its household's design weight, the same float in every clone, and the frame's mass log gains one record whose reason names no clone count. Every household carries its source_household_id (its clone-0 id), source_year (the frame's year) and source_household_key, so the stages after it count true sources in survey households. K is a parameter of the stage's graph node, never of this declaration, so the declaration and its contract are the same at every K. The stage runs after frs_hmrc_spine_leaves and frs_annual_income_spells, which join the raw FRS tabs on raw person ids, and before the area stages (atomic_area_assign, local_authority and rfdr), frs_take_up, frs_person_draws, frs_household_draws and area_brma, which run on every clone. Its evidence gives K, the copy-id field and its radix, the distinct source households rounded by the shared disclosure rules, and the weight split and kind; no row count, id or weight.", "operations": [ { - "kind": "stack_zero_weight_donors", - "count": 10000, - "seed": 42, - "flag_column": "household_is_spi_synthetic", - "channels": { - "base": "frs", - "synthetic": "spi" + "clone_index_columns": { + "benunit": "benunit_clone_index", + "household": "household_clone_index", + "person": "person_clone_index" }, - "draw": "uniform_without_replacement" - }, - { - "kind": "gate_zero_weight_strata", - "gate": "uk_zero_weight_strata_gate", - "declarations": [ - { - "name": "e7_spi_synthetic_preclone", - "selector": { - "household_is_spi_synthetic": true - }, - "maximum_zero_weight_rows": 10000, - "reason": "E7 stacks exactly 10,000 zero-weight SPI-synthetic households on the raw pre-clone UK FRS spine." - } - ] - }, - { - "kind": "allocate_zero_weight_prior_mass", - "share": 0.5, - "strata": [ - "region" - ], - "pension_age_share": 0.2, - "pension_age_stratum": "any_member_at_or_over_state_pension_age", - "weight_kind_out": "importance", - "conservation": "exact_total" + "conservation": "household_total_to_rounding", + "declared_factor": 1, + "entity": "household", + "id_remapping": "copy-id table field geographic_support; clone k takes every entity id + k x the field's radix (10**9)", + "kind": "expand_geographic_clones", + "order": "clone_major", + "ordinal_bound": 100, + "reason": "Geographic clones split each household's design weight equally among its clones, so household mass is conserved.", + "source_columns": [ + "source_household_id", + "source_year", + "source_household_key" + ], + "weight_kind_out": "design", + "weight_split": "equal" } ], "outputs": [ - "household_is_spi_synthetic", - "person_support_channel", - "person_support_clone_index", - "person_source_id", - "benunit_support_channel", - "benunit_support_clone_index", - "benunit_source_id", - "household_support_channel", - "household_support_clone_index", - "household_source_id", + "person_clone_index", + "benunit_clone_index", + "household_clone_index", "source_household_id", "source_year", "source_household_key" ], - "nonnegative_outputs": [ - "person_support_clone_index", - "person_source_id", - "benunit_support_clone_index", - "benunit_source_id", - "household_support_clone_index", - "household_source_id", - "source_household_id", - "source_year" - ], - "notes": "Stacks the pre-clone SPI support channel, gates the 10,000 zero-weight synthetic households under a stage-local declaration, and allocates prior mass with exact national conservation within each region, split by whether the household has a member at or over State Pension age (the engine's at the build instant): the SPI channel takes 50 percent of the stratum mass of households without one and pension_age_share of those with one. The channel's households are drawn unweighted with one weight per stratum and every adult in them carries a taxpayer's income, so a region-only allocation gave the channel more pension-age mass than the FRS design gives pension-age households (microcosm#1069)." + "rewrites": [], + "source": "Microcosm structural operator (microcosm#931); the FRS spine's own households, no publisher input", + "stage": "geographic_expand", + "survey": "Family Resources Survey 2024-25" }, { - "stage": "spi_income_band_donors", - "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Income Tax liabilities statistics July 2026 Table 2.5", - "source": "Reserved SPI support households for the HMRC Table 2.5 total-income bands from GBP 200,000, stacked at the published band taxpayer count over the donors per band (PolicyEngine/chronicle#280 lane).", - "grain": "household", "artifacts": [ { - "role": "qrf_donor", - "kind": "private_microdata", - "format": "tab_delimited", + "format": "json", + "kind": "public_aggregate_reference", + "resource": "frs_release.json", + "role": "frs_release", + "runtime_sha256_required": true + } + ], + "grain": "household", + "notes": "Keys every household for the area assignment (microcosm#931): geography_household_key, from the identity source (source), the FRS vintage frs_release.json pins for the household's source_year, its source_household_id and its clone path, read from the lineage columns geographic_expand writes, never from ids, weights or values: no branch for clone 0 and the geographic_support branch at ordinal k for clone k. So clone 0 keys as a one-clone spine keys the same base household, and growing K never changes an existing household's key. A source year the release pins no vintage for, and keys that repeat, are refused by count. spi_copy_areas gives each SPI copy and income band donor a key of its own, over its own copy path, and the later copy steps copy a household's key with the household. Its evidence gives the distinct source households rounded by the shared disclosure rules, that the keys are unique, the source, the vintages and the columns it reads; no key is printed.", + "operations": [ + { + "kind": "key_households_by_clone_path", + "source": "frs", + "vintage_resource": "frs_release.json" + } + ], + "outputs": [ + "geography_household_key" + ], + "rewrites": [], + "source": "Microcosm structural operator (microcosm#931); the FRS vintage frs_release.json pins, no publisher input", + "stage": "geographic_identity", + "survey": "Family Resources Survey 2024-25" + }, + { + "artifacts": [], + "grain": "household", + "notes": "Draws each geographic clone's atomic area within its FRS region (microcosm#931) with the shared atomic-area operator, on the three conditioned supports the operation names: a 2021 output area in England and Wales, a 2022 output area in Scotland and a 2021 data zone in Northern Ireland. The draw is conditioned on the household's IMD decile, its imd_decile as \"1\" to \"10\" and missing where it is 0, and, where council_tax_band_observed is true, on its council-tax band, by which England, Wales and Scotland weight each area's households; Northern Ireland's supports carry no band, so it draws by households. A household whose cell holds no area falls back a level at a time, L0 (decile, by band), L1 (decile, by households), L2 (by band) and L3 (by households), each within the region, as the shared fallback-level chooser picks it, and a level's dropped input is missing, so one definition serves every level. The inputs are a narrow, transient frame keyed on geography_household_key, so each clone's draw depends on its key, its own inputs, the supports and the stream alone, and growing K never moves an existing clone's area. A copy of the SPI support channel would draw at L3, its decile and band missing by rule; none exists yet here, and spi_copy_areas draws the copies' areas. The stage writes the three assignment columns and the ten ladder columns in the graph's string dtype, and refuses when the shared operator's mapping-integrity validation fails. Its evidence gives K, the seed, the stream, the definition's and supports' digests, the validation outcome and the distinct source households by system, by level and by fallback reason, rounded by the shared disclosure rules; no key, area code or count of a cell.", + "operations": [ + { + "constraints": [ + { + "input": "region", + "required": true, + "support": "frs_region" + }, + { + "input": "atomic_draw_imd_decile", + "required": false, + "support": "imd_decile" + } + ], + "draw_inputs": { + "atomic_draw_council_tax_band": { + "column": "council_tax_band", + "observed": "council_tax_band_observed" + }, + "atomic_draw_imd_decile": { + "column": "imd_decile", + "unobserved": 0 + }, + "missing_on_copies_of": "spi_support_channel" + }, + "fallback_levels": [ + { + "constraints": [ + "region", + "atomic_draw_imd_decile" + ], + "name": "L0", + "weight_by": true + }, + { + "constraints": [ + "region", + "atomic_draw_imd_decile" + ], + "name": "L1", + "weight_by": false + }, + { + "constraints": [ + "region" + ], + "name": "L2", + "weight_by": true + }, + { + "constraints": [ + "region" + ], + "name": "L3", + "weight_by": false + } + ], + "identity": "geography_household_key", + "kind": "assign_atomic_area", + "layers": [ + "oa_code", + "lsoa_code", + "msoa_code", + "local_authority_code", + "ward_code", + "constituency_code", + "region_code", + "itl3_code", + "itl2_code", + "itl1_code" + ], + "outputs": [ + "atomic_area_code", + "atomic_area_system", + "atomic_area_basis" + ], + "seed": 0, + "sources": { + "uk_ew_output_area_2021": "uk_ew_output_area_2021_conditioned_support", + "uk_ni_data_zone_2021": "uk_ni_data_zone_2021_conditioned_support", + "uk_scotland_output_area_2022": "uk_scotland_output_area_2022_conditioned_support" + }, + "stream": "uk-spine-atomic-area-v1", + "weight_by": { + "bands": { + "uk_ew_output_area_2021": [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I" + ], + "uk_ni_data_zone_2021": [], + "uk_scotland_output_area_2022": [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H" + ] + }, + "default": "households", + "input": "atomic_draw_council_tax_band" + } + } + ], + "outputs": [ + "atomic_area_code", + "atomic_area_system", + "atomic_area_basis", + "oa_code", + "lsoa_code", + "msoa_code", + "local_authority_code", + "ward_code", + "constituency_code", + "region_code", + "itl3_code", + "itl2_code", + "itl1_code" + ], + "rewrites": [], + "source": "Microcosm structural operator (microcosm#931): the shared atomic-area operator over the three conditioned atomic-area supports sources.yaml declares, each system's areas with their IMD deciles and, in England, Wales and Scotland, their households by council-tax band", + "stage": "atomic_area_assign", + "survey": "Family Resources Survey 2024-25" + }, + { + "artifacts": [ + { + "format": "json", + "kind": "public_aggregate_reference", + "resource": "local_authority_names.json", + "role": "local_authority_names", + "runtime_sha256_required": true + } + ], + "grain": "household", + "notes": "Resolves each household's engine local_authority from the local_authority_code its area draw gives it (microcosm#931), through the committed April 2023 local authority names roster (local_authority_names.json), fail-closed: a code off the roster refuses the stage, and the engine input is never defaulted (microcosm#953). The column is written in the graph's string dtype. Its evidence gives K, the roster's vintage and digest and the distinct source households, rounded by the shared disclosure rules.", + "operations": [ + { + "code_column": "local_authority_code", + "kind": "resolve_local_authority", + "names_resource": "local_authority_names.json", + "output": "local_authority", + "vintage": "2023_april_lad" + } + ], + "outputs": [ + "local_authority" + ], + "rewrites": [], + "source": "The committed ONS April 2023 local authority district names roster (local_authority_names.json) and policyengine-uk's local_authority members", + "stage": "local_authority", + "survey": "Family Resources Survey 2024-25" + }, + { + "artifacts": [ + { + "format": "json", + "kind": "public_aggregate_reference", + "resource": "rfdr_area_layer.json", + "role": "rfdr_area_layer", + "runtime_sha256_required": true + } + ], + "grain": "household", + "notes": "Sets each household's in_rural_fuel_duty_relief_area from its drawn output area through the committed Rural Fuel Duty Relief area layer (rfdr_area_layer.json), as a boolean (microcosm#931). Every output area the layer flags must be an area of its system's conditioned support (roster_sources), so the layer and the supports share one code vintage; a system none of whose households is flagged is reported, as a boolean, and not refused (require_flagged), since a spine, a sampled one above all, can miss every flagged area by chance. Northern Ireland has no area in the scheme, so its households are never flagged. Its evidence gives K, the layer's and the supports' digests and the method, the roster check, those booleans and, among clone-0 rows, the distinct flagged source households by system, rounded by the shared disclosure rules.", + "operations": [ + { + "area_code": "atomic_area_code", + "area_system": "atomic_area_system", + "kind": "assign_rfdr_flags", + "layer_resource": "rfdr_area_layer.json", + "method": "uk-rfdr-area-layer-v1", + "require_flagged": false, + "roster_sources": { + "uk_ew_output_area_2021": "uk_ew_output_area_2021_conditioned_support", + "uk_scotland_output_area_2022": "uk_scotland_output_area_2022_conditioned_support" + } + } + ], + "outputs": [ + "in_rural_fuel_duty_relief_area" + ], + "rewrites": [], + "source": "HMRC Notice 2001's Rural Fuel Duty Relief areas, as the committed RFDR area layer (rfdr_area_layer.json) places them in 2021 England and Wales and 2022 Scotland output areas", + "stage": "rfdr", + "survey": "Family Resources Survey 2024-25" + }, + { + "stage": "frs_take_up", + "survey": "Family Resources Survey 2024-25", + "source": "Family Resources Survey 2024-25 reported receipt anchors plus sourced UK take-up contract rates.", + "grain": "benunit", + "artifacts": [], + "operations": [ + { + "kind": "aggregate_person_to_benunit", + "method": "any_positive", + "consumed_only": true, + "aggregates": { + "child_benefit_reported_anchor": "child_benefit_reported", + "pension_credit_reported_anchor": "pension_credit_reported", + "universal_credit_reported_anchor": "universal_credit_reported" + } + }, + { + "kind": "aggregate_person_to_benunit", + "method": "any_uc_claimant_aged_18_under_pension_credit_age", + "consumed_only": true, + "aggregates": { + "uc_age_eligible": "age" + }, + "role_column": "is_uc_claimant" + }, + { + "kind": "assign_binary_with_anchored_residual", + "output": "would_claim_child_benefit", + "draw": "would_claim_child_benefit", + "rate_key": "child_benefit", + "anchor": "child_benefit_reported_anchor", + "seed": 0 + }, + { + "kind": "assign_binary_from_rate", + "output": "child_benefit_opts_out", + "draw": "child_benefit_opts_out", + "rate_key": "child_benefit_opts_out_rate", + "seed": 0 + }, + { + "kind": "assign_binary_with_anchored_residual", + "output": "would_claim_pc", + "draw": "would_claim_pc", + "rate_key": "pension_credit", + "anchor": "pension_credit_reported_anchor", + "seed": 0 + }, + { + "kind": "assign_binary_with_anchored_residual", + "output": "would_claim_uc", + "draw": "would_claim_uc", + "rate_key": "universal_credit", + "anchor": "universal_credit_reported_anchor", + "population": "uc_age_eligible", + "seed": 0 + }, + { + "kind": "assign_binary_from_rate", + "output": "would_claim_tfc", + "draw": "would_claim_tfc", + "rate_key": "tax_free_childcare", + "seed": 0 + }, + { + "kind": "assign_binary_from_rate", + "output": "would_claim_extended_childcare", + "draw": "would_claim_extended_childcare", + "rate_key": "extended_childcare", + "seed": 0 + }, + { + "kind": "assign_binary_from_rate", + "output": "would_claim_universal_childcare", + "draw": "would_claim_universal_childcare", + "rate_key": "universal_childcare", + "nested_under": "would_claim_extended_childcare", + "rate_derivation": "the contract's universal_childcare rate is the rate among benefit units not claiming the extended hours; would_claim_universal_childcare = would_claim_extended_childcare OR u < rate, so every unit claiming the extended hours claims the universal hours", + "seed": 0 + }, + { + "kind": "assign_binary_from_rate", + "output": "would_claim_targeted_childcare", + "draw": "would_claim_targeted_childcare", + "rate_key": "targeted_childcare", + "seed": 0 + }, + { + "kind": "assign_binary_from_banded_rates", + "output": "would_claim_uc_childcare", + "draw": "would_claim_uc_childcare", + "band_column": "is_married", + "bands": [ + { + "upper_exclusive": 1, + "rate_key": "uc_childcare_single" + }, + { + "rate_key": "uc_childcare_couple" + } + ], + "seed": 0 + }, + { + "kind": "assign_clipped_normal", + "output": "maximum_extended_childcare_hours_usage", + "draw": "maximum_extended_childcare_hours_usage", + "distribution_key": "maximum_extended_childcare_hours_usage", + "seed": 0 + }, + { + "kind": "assign_binary_from_rate", + "output": "max_free_entitlement_hours_used", + "draw": "max_free_entitlement_hours_used", + "rate_key": "extended_childcare_age_0_usage", + "eligibility": "persons of DfE age 0 in benefit units that claim the working-parent hours (would_claim_extended_childcare)", + "assignment_unit": "person", + "rate_derivation": "a child of DfE age 0 in a claiming unit uses the working-parent hours where u < its age's rate and gets max_free_entitlement_hours_used 0 otherwise; every other person keeps the engine's default of 30; one draw per person serves both ages' operations", + "seed": 0 + }, + { + "kind": "assign_binary_from_rate", + "output": "max_free_entitlement_hours_used", + "draw": "max_free_entitlement_hours_used", + "rate_key": "extended_childcare_age_1_usage", + "eligibility": "persons of DfE age 1 in benefit units that claim the working-parent hours (would_claim_extended_childcare)", + "assignment_unit": "person", + "rate_derivation": "a child of DfE age 1 in a claiming unit uses the working-parent hours where u < its age's rate and gets max_free_entitlement_hours_used 0 otherwise; every other person keeps the engine's default of 30; one draw per person serves both ages' operations", + "seed": 0 + } + ], + "outputs": [ + "would_claim_child_benefit", + "child_benefit_opts_out", + "would_claim_pc", + "would_claim_uc", + "would_claim_tfc", + "would_claim_extended_childcare", + "would_claim_universal_childcare", + "would_claim_targeted_childcare", + "would_claim_uc_childcare", + "maximum_extended_childcare_hours_usage", + "max_free_entitlement_hours_used" + ], + "nonnegative_outputs": [ + "maximum_extended_childcare_hours_usage", + "max_free_entitlement_hours_used" + ], + "notes": "Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100; salts are output variable names. Reported positive receipt anchors are transient and consumed only. The three anchored draws, would_claim_child_benefit, would_claim_pc and would_claim_uc, count their contract targets within each geographic clone (household_clone_index, microcosm#931), so each clone fills its own residual to int(rate * n_units) over its own units, clone 0 takes a one-clone spine's flags at any K, and a spine without the clone index counts over every unit. The Universal Credit draw's population is the benefit units with a UC claimant or partner (is_uc_claimant) aged 18 or over and under Pension Credit qualifying age (66 for everyone who can reach it in this release), the age and role conditions of policyengine-uk 2.177.7's is_uc_eligible; the engine lockstep test pins it and names the engine cases not mirrored (16- and 17-year-old claimants under the UC Regs 2013 reg 8 exceptions, mixed-age couples on the Pension Credit route, from 2.128.0 pension-age families moved onto Universal Credit when their legacy benefit closes, SI 2014/1230 reg 60A, and from 2.173.2 a unit whose working-age claimants all receive education with no exception, since frs_education sets is_in_full_time_education from the FRS). So the contract rate is a share of those units, an anchor outside them stays true, and any other unit is never drawn; a unit with no claimant or partner of that age keeps its reported Housing Benefit, which a claim it cannot make no longer zeroes (#882, microcosm#867, microcosm#1095). The universal childcare flag is nested under the working-parent one. A unit that claims the extended hours claims the universal hours, and any other unit claims them at the contract's universal_childcare rate, the rate conditional on not claiming the extended hours; the extended draw is unchanged. childcare_copy_take_up later re-draws the childcare flags and hours of the support and CGT copies on their own benefit units. A child of DfE age 0 or 1 (its age on 31 December of the fit's model year, the age DfE counts by) in a unit that claims the working-parent hours uses them at its age's contract rate, extended_childcare_age_0_usage or extended_childcare_age_1_usage (DfE registers far fewer of the youngest eligible children than of those aged 2 to 4, and fewer at 9 to 11 months than at age 1); a child that does not gets max_free_entitlement_hours_used 0, so the engine funds it no hours, and every other person keeps the engine's default of 30." + }, + { + "stage": "frs_person_draws", + "survey": "Family Resources Survey 2024-25", + "source": "Sourced UK take-up contract rates and identity-keyed deterministic draws.", + "grain": "person", + "artifacts": [ + { + "role": "stochastic_contract", + "resource": "take_up_contract.json", + "kind": "public_aggregate_reference", + "format": "json" + } + ], + "operations": [ + { + "kind": "assign_binary_from_rate", + "output": "would_claim_marriage_allowance", + "draw": "would_claim_marriage_allowance", + "rate_key": "marriage_allowance", + "seed": 0 + }, + { + "kind": "assign_binary_from_banded_rates", + "output": "would_claim_scp", + "draw": "would_claim_scp", + "band_column": "age", + "bands": [ + { + "upper_exclusive": 6, + "rate_key": "scp_under_6" + }, + { + "rate_key": "scp_6_plus" + } + ], + "seed": 0 + }, + { + "kind": "assign_uniform_draw", + "output": "attends_private_school_random_draw", + "seed": 0 + }, + { + "kind": "assign_period_constant", + "output": "tax_free_childcare_spend_routed_share", + "contract_key": "tax_free_childcare_spend_routed_share", + "period_source": "build_year", + "dtype": "float64" + }, + { + "kind": "assign_binary_from_rate", + "output": "is_apprentice", + "draw": "is_apprentice", + "rate_key": "apprentice_stock_19_to_24", + "assignment_unit": "person", + "eligibility": "employees (employment_status FT_EMPLOYED or PT_EMPLOYED, from frs_employment) aged 21 to 24 who are not in full-time education (is_in_full_time_education, from frs_education; the FRS codes no apprentice as in full-time education): SSCBA 1992 s.9A already zero-rates every earner under 21 and s.9B ends the apprentice zero rate at 25, so no other age changes employer NICs, and leaving them False keeps is_in_non_advanced_education, minimum_wage, Care to Learn and the 16 to 19 Bursary Fund unchanged; the flagged keep their age-related minimum_wage too, because policyengine-uk 2.162.8 (policyengine-uk#2683) gives the apprentice rate only under 19 or with is_in_first_year_of_apprenticeship, which the build leaves at its default (False), and the stage refuses an engine without that split or the s.9B threshold", + "rate_derivation": "the contract's apprentice_stock_19_to_24 is DfE's England apprentices in learning at the start of the academic year (participation less starts) per head aged 19 to 24, a point-in-time count taken as uniform across those ages and UK-wide; the target is r times the weighted population aged 21 to 24 at the frame's household weights, counted within each geographic clone (household_clone_index, microcosm#931): each clone's eligible employees are flagged in the order of their identity-keyed uniform until their weight is nearest its own target (uc_take_up's rule), or all of them when they weigh no more than it; the receipt records the target, the flagged weight, the support bands and the relief the flags imply at the installed engine's Class 1 schedule, and uk_stage_frs_person_draws_apprentices gates them, the relief at or below HMRC's NIC - NS1", + "seed": 0 + } + ], + "outputs": [ + "would_claim_marriage_allowance", + "would_claim_scp", + "attends_private_school_random_draw", + "tax_free_childcare_spend_routed_share", + "is_apprentice" + ], + "nonnegative_outputs": [ + "attends_private_school_random_draw", + "tax_free_childcare_spend_routed_share" + ], + "notes": "Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100; higher_earner_tie_break is intentionally not produced. is_apprentice has no FRS source (no apprenticeship variable in the 2022-23 to 2024-25 adult tables; TRAIN2, NI3TRAIN and TRAINEE code government employment schemes) and is drawn to DfE's point-in-time rate for employees aged 21 to 24 only." + }, + { + "stage": "frs_household_draws", + "survey": "Family Resources Survey 2024-25", + "source": "Sourced UK stochastic contract rates and identity-keyed deterministic draws.", + "grain": "household", + "artifacts": [], + "operations": [ + { + "kind": "assign_binary_from_rate", + "output": "household_owns_tv", + "draw": "household_owns_tv", + "rate_key": "tv_ownership_rate", + "seed": 0 + }, + { + "kind": "assign_binary_from_rate", + "output": "would_evade_tv_licence_fee", + "draw": "would_evade_tv_licence_fee", + "rate_key": "tv_licence_evasion_rate", + "seed": 0 + }, + { + "kind": "assign_binary_from_rate", + "output": "main_residential_property_purchased_is_first_home", + "draw": "main_residential_property_purchased_is_first_home", + "rate_key": "first_time_buyer_rate", + "seed": 0 + }, + { + "kind": "assign_binary_from_rate", + "output": "property_purchased", + "draw": "property_purchased", + "rate_key": "property_purchase_rate", + "seed": 0 + } + ], + "outputs": [ + "household_owns_tv", + "would_evade_tv_licence_fee", + "main_residential_property_purchased_is_first_home", + "property_purchased" + ], + "notes": "Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100. TV evasion remains an independent draw, matching the incumbent reference share." + }, + { + "artifacts": [ + { + "format": "json", + "kind": "public_aggregate_reference", + "resource": "brma_area_layer.json", + "role": "brma_area_layer", + "runtime_sha256_required": true + }, + { + "format": "json", + "kind": "public_aggregate_reference", + "resource": "brma_private_rented_households.json", + "role": "count_resource", + "runtime_sha256_required": true + } + ], + "grain": "household", + "notes": "Writes each household's brma, in the place frs_brma held, after the draws (microcosm#931): LHA_category reads frs_take_up's would_claim_uc. Great Britain households read it off their drawn LSOA (England and Wales) or output area (Scotland) through the committed area-to-BRMA layer (brma_area_layer.json), fail-closed on an area the layer does not hold or places in another FRS region. Northern Ireland has no open BRMA geography: one engine call over the whole frame materialises LHA_category, and its benefit units draw from the census count table by region and LHA category (count_table_systems) with frs_brma's identity-keyed seed 0 streams, salt brma for the benefit units and brma:household_pick for the household collapse, so each takes the BRMA frs_brma drew for it. brma stays an object column. Its evidence gives K, the layer's and the count table's digests, the seed and salts, and the distinct source households each Great Britain grain reads and those Northern Ireland draws, rounded by the shared disclosure rules.", + "operations": [ + { + "consumed_only": true, + "kind": "materialize_rules_engine_predictors", + "predictors": [ + "LHA_category" + ] + }, + { + "area_codes": { + "uk_ew_output_area_2021": "lsoa_code", + "uk_scotland_output_area_2022": "oa_code" + }, + "area_system": "atomic_area_system", + "assignment_grain": "benunit", + "cell_columns": [ + "region", + "LHA_category" + ], + "collapse_method": "uniform_member_pick", + "collapse_to": "household", + "count_resource": "brma_private_rented_households.json", + "count_table_systems": [ + "uk_ni_data_zone_2021" + ], + "kind": "assign_area_brma", + "layer_resource": "brma_area_layer.json", + "method": "uk-brma-area-layer-v1", + "output": "brma", + "salts": { + "benunit": "brma", + "household_pick": "brma:household_pick" + }, + "seed": 0 + } + ], + "outputs": [ + "brma" + ], + "rewrites": [], + "source": "Great Britain: the committed area-to-BRMA layer (brma_area_layer.json). Northern Ireland: census private-rented households by region, BRMA and bedrooms (ONS and NISRA 2021, NRS 2022) and the policyengine-uk LHA_category predictor.", + "stage": "area_brma", + "survey": "Family Resources Survey 2024-25" + }, + { + "stage": "spi_support_channel", + "survey": "Family Resources Survey 2024-25", + "source": "Synthetic SPI support channel drawn from every geographic clone of the raw FRS spine's households as a keyed Bernoulli share, each household on its own identity-keyed uniform.", + "grain": "household", + "artifacts": [], + "operations": [ + { + "kind": "stack_zero_weight_donors", + "count": 10000, + "seed": 42, + "flag_column": "household_is_spi_synthetic", + "channels": { + "base": "frs", + "synthetic": "spi" + }, + "draw": "keyed_bernoulli_share: stable_identity_uniforms(household_id, seed, salt) < min(1, count x sample fraction / distinct source households)", + "id_remapping": "copy-id table field spi_support_channel; the SPI copy takes every entity id + 1 x the field's radix (10**11)", + "salt": "spi_support_channel:inclusion" + }, + { + "kind": "gate_zero_weight_strata", + "gate": "uk_zero_weight_strata_gate", + "declarations": [ + { + "name": "e7_spi_synthetic_share", + "selector": { + "household_is_spi_synthetic": true + }, + "reason": "E7 stacks a keyed Bernoulli share of the raw UK FRS spine's households as zero-weight SPI-synthetic households, so their count is binomial; it may exceed the share of the household rows entering the stage by at most the repo's four-sigma binomial band, floored at one row.", + "maximum_zero_weight_rows_rule": { + "rule": "binomial_share_band", + "sigma": 4 + } + } + ] + }, + { + "kind": "allocate_zero_weight_prior_mass", + "share": 0.5, + "strata": [ + "region" + ], + "pension_age_share": 0.2, + "pension_age_stratum": "any_member_at_or_over_state_pension_age", + "weight_kind_out": "importance", + "conservation": "exact_total", + "geographic_clone_stratum": "household_clone_index" + } + ], + "outputs": [ + "household_is_spi_synthetic", + "person_support_channel", + "person_support_clone_index", + "person_source_id", + "benunit_support_channel", + "benunit_support_clone_index", + "benunit_source_id", + "household_support_channel", + "household_support_clone_index", + "household_source_id" + ], + "nonnegative_outputs": [ + "person_support_clone_index", + "person_source_id", + "benunit_support_clone_index", + "benunit_source_id", + "household_support_clone_index", + "household_source_id" + ], + "notes": "Stacks the SPI support channel as a keyed Bernoulli share (microcosm#931): a household is copied when its uniform, keyed on its own household id under the stage's seed and salt, falls below min(1, 10,000 times the sample fraction over the distinct source households entering the stage). Each household's uniform depends on its own id and the stage's seed and salt alone, and whether it is copied also on that probability. The denominator counts a source once however many geographic clones carry it, so every clone draws at the same probability on its own ids, and each clone's copy count is binomial with mean min(10,000 times the sample fraction, its source households). A stage-local declaration gates the zero-weight synthetic households at the share of the household rows entering the stage plus the repo's four-sigma binomial band. The stage then allocates prior mass with exact national conservation within each region, and within each geographic clone where the frame carries household_clone_index, split by whether the household has a member at or over State Pension age (the engine's at the build instant): the SPI channel takes 50 percent of the stratum mass of households without one and pension_age_share of those with one. A stratum with base mass and no SPI copy refuses the stage at every clone count: a geographic clone's stratum that the share left without a copy is never given its base mass back, which would leave that clone short of its SPI share, so it refuses as a one-clone spine's stratum would (microcosm#931). The channel's households are drawn unweighted with one weight per stratum and every adult in them carries a taxpayer's income, so a region-only allocation gave the channel more pension-age mass than the FRS design gives pension-age households (microcosm#1069)." + }, + { + "stage": "spi_income_band_donors", + "survey": "Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Income Tax liabilities statistics July 2026 Table 2.5", + "source": "Reserved SPI support households for the HMRC Table 2.5 total-income bands from GBP 200,000, stacked at the published band taxpayer count over the donors per band (PolicyEngine/chronicle#280 lane).", + "grain": "household", + "artifacts": [ + { + "role": "qrf_donor", + "kind": "private_microdata", + "format": "tab_delimited", "survey": "Survey of Personal Incomes Public Use Tape 2022-23", "vintage": "2022-23", "tax_year_start": 2022, @@ -1785,28 +2092,29 @@ "minimum_donors_per_band": 120, "maximum_donor_weight": 250.0, "count_rule": "max(minimum_donors_per_band, ceil(published band taxpayers / maximum_donor_weight))", - "weight_rule": "published band taxpayers / band donor count, equal within a band", + "weight_rule": "published band taxpayers / (geographic clone count x one clone's band donor count), equal within a band", "expected_band_count": 4, "candidate_population": "raw FRS base channel claimants and partners (is_uc_claimant) aged 16 and over", "candidate_order": "person_id ascending breaks equal draw keys", - "draw": "weighted_without_replacement_identity_keyed: Efraimidis-Spirakis keys log(u) / (household weight x propensity) on stable_identity_uniforms(person_id, seed, salt:band); the largest keys are seated, one carrier per household, the highest band first", + "draw": "weighted_without_replacement_identity_keyed within each geographic clone (a spine without household_clone_index is one clone): Efraimidis-Spirakis keys log(u) / (household weight x propensity) on stable_identity_uniforms(person_id, seed, salt:band); the largest keys are seated, one carrier per household, the highest band first", "propensity": "SPI 2022-23 FACT-weighted band share by region, sex and age band", "tape_preparation": "as hmrc_spi_income_spine prepares the tape: its stage-1 seed and draw_spi_donor_ages_by_population", "tape_preparation_seed": 42, "seed": 3, "salt": "spi_income_band_donor_draw", "clone_index": 2, + "id_remapping": "copy-id table field spi_support_channel; a donor takes every entity id + 2 x the field's radix (10**11), and its source is read from its source-id columns, never from the id", "flag_column": "household_is_spi_income_band_donor", "band_column": "spi_income_band_donor_lower_bound", "carrier_column": "person_is_spi_income_band_carrier", - "funding": "stratum reallocation: every incumbent household of the stratum a donor's source household belongs to is scaled by 1 - donor mass / stratum mass", + "funding": "stratum reallocation per geographic clone: every incumbent household of the stratum a donor's source household belongs to, in the donor's clone, is scaled by 1 - donor mass / stratum mass", "funding_strata": [ "region" ], "maximum_reallocation_share": 0.5, "maximum_donor_household_share": 0.15, - "seating_scale": "min(1, maximum_donor_household_share x eligible base households / expected donor count); band counts floored, at least one per band", - "mass_scale": "min(1, min over funding strata of maximum_reallocation_share x stratum mass / seated donor mass); every band's weight scaled alike", + "seating_scale": "min(1, maximum_donor_household_share x eligible base source households / expected donor count of one geographic clone); band counts floored, at least one per band", + "mass_scale": "min(1, min over the funding strata of every geographic clone of maximum_reallocation_share x stratum mass / seated donor mass); one scale for every band and clone", "never_zero_weight": true, "weight_kind_out": "importance", "conservation": "exact_total", @@ -1814,27 +2122,172 @@ "reason": "Reallocate the published HMRC Table 2.5 taxpayer mass of each total-income band from GBP 200,000 from the households of each donor's region to its income band donor support rows; every region's household mass and the total national mass are conserved." } ], - "outputs": [ - "household_is_spi_income_band_donor", - "spi_income_band_donor_lower_bound", - "person_is_spi_income_band_carrier" - ], + "outputs": [ + "household_is_spi_income_band_donor", + "spi_income_band_donor_lower_bound", + "person_is_spi_income_band_carrier" + ], + "rewrites": [ + "household_is_spi_synthetic", + "person_support_channel", + "person_support_clone_index", + "person_source_id", + "benunit_support_channel", + "benunit_support_clone_index", + "benunit_source_id", + "household_support_channel", + "household_support_clone_index", + "household_source_id" + ], + "nonnegative_outputs": [ + "spi_income_band_donor_lower_bound" + ], + "notes": "Reserves SPI-channel households for every Table 2.5 band from GBP 200,000 so the top of the income distribution has support before calibration. A band seats enough donors that none starts above the maximum weight and never fewer than the minimum, at equal weights of the build tax year's published taxpayers over the band's count. Carriers are FRS adults seated by identity-keyed draws on household weight times the tape's own band propensity given region, sex and age band; their households are copied whole at clone index 2 with the synthetic flag. The mass the donors carry is taken in proportion from the households of each donor's region, so the national total and every region's mass are unchanged (microcosm#1063). On a clone-first spine each geographic clone seats its own donors by the same walk, and every clone takes the same seats, sized against the eligible households counted in distinct source households. Each donor weighs its band's weight over the clone count, and each clone's regions fund their own donors at one mass scale for every clone; a spine without household_clone_index is one clone (microcosm#931). The income stage gives every carrier a band-conditional draw from the tape (resample_band_donor_leaves)." + }, + { + "artifacts": [ + { + "format": "json", + "kind": "public_aggregate_reference", + "resource": "frs_release.json", + "role": "frs_release", + "runtime_sha256_required": true + }, + { + "format": "json", + "kind": "public_aggregate_reference", + "resource": "local_authority_names.json", + "role": "local_authority_names", + "runtime_sha256_required": true + }, + { + "format": "json", + "kind": "public_aggregate_reference", + "resource": "brma_area_layer.json", + "role": "brma_area_layer", + "runtime_sha256_required": true + }, + { + "format": "json", + "kind": "public_aggregate_reference", + "resource": "brma_private_rented_households.json", + "role": "count_resource", + "runtime_sha256_required": true + }, + { + "format": "json", + "kind": "public_aggregate_reference", + "resource": "rfdr_area_layer.json", + "role": "rfdr_area_layer", + "runtime_sha256_required": true + } + ], + "grain": "household", + "notes": "Gives every row the SPI support channel copied, an SPI copy (spi_support_channel) or an income band donor (spi_income_band_donors), a key and a place of its own, where each copy had its parent's (microcosm#931). The key is geography_household_key over the copy's own path, read on the support channel's and the geographic clones' branches as geographic_identity reads a path, with the FRS vintage frs_release.json pins for its source_year; the area is drawn at L3, its region by households, with atomic_area_assign's definition, stream and seed, since a copy's decile and band inputs are missing by rule; then its local authority (as local_authority), its BRMA (as area_brma) and its fuel-relief flag (as rfdr). A Great Britain copy reads its BRMA off its area through the layer. Northern Ireland's copies draw theirs from the census count table on their own benefit units with area_brma's seed and salts, after one engine call over their households alone materialises their LHA_category, one simulation over every clone's copies (engine_simulations), so each copy takes a draw of its own (María's ruling of 2026-11-11). Every other row is left as it was, and each rewrite keeps its column's dtype; the copies later stages make inherit their parent's key and area. Its evidence gives K, the seed, the stream, the digests, the validation and roster outcomes and, among clone-0 rows, the distinct source households copied, by kind and system, and those whose copies are flagged, by Great Britain system, rounded by the shared disclosure rules; where one system of a kind rests on one or two source households, every system of that kind is withheld, zeros included, since at one clone the support channel's and the band donors' own copy counts would give it back by difference; no key or area code.", + "operations": [ + { + "consumed_only": true, + "kind": "materialize_rules_engine_predictors", + "predictors": [ + "LHA_category" + ] + }, + { + "area_draw": { + "definition_of": "atomic_area_assign", + "level": "L3", + "seed": 0, + "stream": "uk-spine-atomic-area-v1" + }, + "brma": { + "as_stage": "area_brma", + "assignment_grain": "benunit", + "cell_columns": [ + "region", + "LHA_category" + ], + "collapse_method": "uniform_member_pick", + "collapse_to": "household", + "count_resource": "brma_private_rented_households.json", + "count_table_systems": [ + "uk_ni_data_zone_2021" + ], + "engine_households": "northern_ireland_copies", + "engine_simulations": "one", + "salts": { + "benunit": "brma", + "household_pick": "brma:household_pick" + }, + "seed": 0 + }, + "copies": { + "branch": "spi_support_channel", + "kinds": { + "1": "spi_support_copy", + "2": "income_band_donor" + }, + "paths_require": [ + "spi_support_channel", + "geographic_support" + ] + }, + "key": { + "column": "geography_household_key", + "source": "frs", + "vintage_resource": "frs_release.json" + }, + "kind": "assign_spi_copy_areas", + "local_authority": { + "as_stage": "local_authority" + }, + "rewrites": [ + "geography_household_key", + "atomic_area_code", + "atomic_area_system", + "atomic_area_basis", + "oa_code", + "lsoa_code", + "msoa_code", + "local_authority_code", + "ward_code", + "constituency_code", + "region_code", + "itl3_code", + "itl2_code", + "itl1_code", + "local_authority", + "brma", + "in_rural_fuel_duty_relief_area" + ], + "rfdr": { + "as_stage": "rfdr", + "require_flagged": false + } + } + ], + "outputs": [], "rewrites": [ - "household_is_spi_synthetic", - "person_support_channel", - "person_support_clone_index", - "person_source_id", - "benunit_support_channel", - "benunit_support_clone_index", - "benunit_source_id", - "household_support_channel", - "household_support_clone_index", - "household_source_id" - ], - "nonnegative_outputs": [ - "spi_income_band_donor_lower_bound" - ], - "notes": "Reserves SPI-channel households for every Table 2.5 band from GBP 200,000 so the top of the income distribution has support before calibration. A band seats enough donors that none starts above the maximum weight and never fewer than the minimum, at equal weights of the build tax year's published taxpayers over the band's count. Carriers are FRS adults seated by identity-keyed draws on household weight times the tape's own band propensity given region, sex and age band; their households are copied whole at clone index 2 with the synthetic flag. The mass the donors carry is taken in proportion from the households of each donor's region, so the national total and every region's mass are unchanged (microcosm#1063). The income stage gives every carrier a band-conditional draw from the tape (resample_band_donor_leaves)." + "geography_household_key", + "atomic_area_code", + "atomic_area_system", + "atomic_area_basis", + "oa_code", + "lsoa_code", + "msoa_code", + "local_authority_code", + "ward_code", + "constituency_code", + "region_code", + "itl3_code", + "itl2_code", + "itl1_code", + "local_authority", + "brma", + "in_rural_fuel_duty_relief_area" + ], + "source": "Microcosm structural operator (microcosm#931): atomic_area_assign's draw and the local_authority, area_brma and rfdr lookups, re-run on the rows the SPI support channel copied", + "stage": "spi_copy_areas", + "survey": "Family Resources Survey 2024-25" }, { "stage": "hmrc_spi_income_spine", @@ -3221,10 +3674,40 @@ { "stage": "spi_housing_shell", "survey": "Family Resources Survey 2024-25", - "source": "Family Resources Survey 2024-25 base households' housing (tenure, dwelling type, bedrooms, council tax, housing costs and the housing-linked benefit reports), imputed onto the SPI support households from their own region, composition and post-SPI incomes.", + "source": "Family Resources Survey 2024-25 base households' housing (tenure, dwelling type, bedrooms, council tax, housing costs and the housing-linked benefit reports), imputed onto the SPI support households from their own region, composition and post-SPI incomes; for the council tax of Great Britain's SPI support households, each local authority's Band D council tax for 2024-25, keyed by April 2023 local authority code, England's from MHCLG's Council Tax levels set by local authorities in England 2024 to 2025 (Table 8) and Wales's from StatsWales (council_tax_band_d_england_wales_2024_25.json), and Scotland's from the Scottish Government's CTAS 2026 Council Tax Assumptions, vendored from the pinned Chronicle feed (scotgov_band_d_council_tax_2024_25.json); the statutory band proportions and single-adult discount (Local Government Finance Act 1992, as amended); and the conditioned atomic-area supports sources.yaml declares, on which each such household's reference Band D is taken over its region's areas, weighted by households.", "grain": "household", - "artifacts": [], + "artifacts": [ + { + "format": "json", + "kind": "public_aggregate_reference", + "resource": "council_tax_band_d_england_wales_2024_25.json", + "role": "council_tax_band_d", + "runtime_sha256_required": true + }, + { + "format": "json", + "kind": "public_aggregate_reference", + "resource": "scotgov_band_d_council_tax_2024_25.json", + "role": "scotland_band_d_facts", + "runtime_sha256_required": true + } + ], "operations": [ + { + "artifact_role": "scotland_band_d_facts", + "fail_on_mismatch": true, + "feed_pin": "chronicle_feed.json", + "kind": "verify_vendored_fact_resource", + "mapped_build_period": 2024, + "period_mapping": "fiscal_year_2024_25_is_the_build_period", + "record_sets": [ + "scotgov.ctas2026.band_d.fy2024" + ], + "require_before_source_read": true, + "resource": "scotgov_band_d_council_tax_2024_25.json", + "runtime_sha256_required": true, + "source_vintage": "2024-25 (ScotGov CTAS 2026)" + }, { "kind": "impute_spi_housing_shell", "training_rows": "base_frs_channel_positive_weight", @@ -3287,6 +3770,11 @@ "n_estimators": 100, "seed": 0, "gate_calibration": "cross_fitted" + }, + { + "formula": "After the band draw and the chained amounts, on the Great Britain rows the SPI support channel copied (SPI copies and income band donors): the 2024-25 statute (council_tax_band_d.uk_statutory_council_tax) at the copy's reference Band D, the exact expected 2024-25 Band D of the cell spi_copy_areas drew its area from (its region, by households, as council_tax_localise takes it), for its nation and drawn band, less the single-adult discount where council_tax_single_adult_raw is 1. Northern Ireland's rows keep the chained draw. The bill reads neither the drawn area nor its local authority, so council_tax_localise localises it once.", + "kind": "derive", + "output": "council_tax" } ], "outputs": [], @@ -3309,12 +3797,214 @@ "housing_benefit_reported", "council_tax_benefit_reported" ], - "notes": "Runs after the SPI income chain and before the income-conditioned donor stages, the household counterpart of the SPI stage-2 FRS-only fill. The support copies and income band donors are whole FRS households, so without this stage each kept its FRS parent's housing, drawn for the parent's income. Trained on the FRS base households (positive household weight) and drawn for every SPI-channel household from engine-free predictors (region, household type, adults, children, single-adult status, reference-person age and household income components). Tenure, dwelling type, bedrooms and council-tax band are drawn in that order by weighted multiclass classifiers with an inverse-CDF draw; the housing amounts and the head benefit unit's Housing Benefit and the reference person's council tax benefit by one chained regime-gated QRF conditioned on the drawn categories, clipped to the training range. The QRF's sign gates are cross-fitted (gate_calibration: cross_fitted; microcosm#1359), as lcfs_consumption's are: each is calibrated on the donors its fold gates left out, so a sign is drawn at its donor rate on households the gates were not fitted to. Every draw reads uniforms keyed on household_id. An amount is zero wherever the training households have no positive value for its tenure or region; mortgage payments need a mortgaged tenure (true of every FRS household); Housing Benefit needs a rented tenure (the engine pays it to renting reporters only); council tax benefit is capped at council tax and the household council_tax_rebate equals it, both on the reference person as the FRS holds them. Region, single-adult status, BRMA (region and the recipient's own LHA category), the flat-rate household draws and composition are kept; FRS base rows are byte-unchanged." + "notes": "Runs after the SPI income chain and before the income-conditioned donor stages, the household counterpart of the SPI stage-2 FRS-only fill. The support copies and income band donors are whole FRS households, so without this stage each kept its FRS parent's housing, drawn for the parent's income. Trained on the FRS base households (positive household weight) and drawn for every SPI-channel household from engine-free predictors (region, household type, adults, children, single-adult status, reference-person age and household income components). Tenure, dwelling type, bedrooms and council-tax band are drawn in that order by weighted multiclass classifiers with an inverse-CDF draw; the housing amounts and the head benefit unit's Housing Benefit and the reference person's council tax benefit by one chained regime-gated QRF conditioned on the drawn categories, clipped to the training range. The QRF's sign gates are cross-fitted (gate_calibration: cross_fitted; microcosm#1359), as lcfs_consumption's are: each is calibrated on the donors its fold gates left out, so a sign is drawn at its donor rate on households the gates were not fitted to. Every draw reads uniforms keyed on household_id. An amount is zero wherever the training households have no positive value for its tenure or region; mortgage payments need a mortgaged tenure (true of every FRS household); Housing Benefit needs a rented tenure (the engine pays it to renting reporters only); council tax benefit is capped at council tax and the household council_tax_rebate equals it, both on the reference person as the FRS holds them. Region, single-adult status, BRMA (region and the recipient's own LHA category), the flat-rate household draws and composition are kept; FRS base rows are byte-unchanged. Council tax stays a target of the chained QRF on every row, so every band and every other chained amount is the draw's. After the learned zeros and the tenure rules, a Great Britain row's drawn council tax is replaced by the 2024-25 statute at its reference Band D (the stage's derive of council_tax): the expected 2024-25 Band D of the cell spi_copy_areas draws the row's area from, its region by households, recomputed on the conditioned atomic-area supports as council_tax_localise recomputes it, for the row's nation and drawn band, less the single-adult discount where council_tax_single_adult_raw is 1. A Northern Ireland row keeps the chained draw (María's ruling of 2026-11-14). The bill reads neither the drawn area nor its local authority, so council_tax_localise localises it once. The council tax benefit is capped at the bill written. Loading the committed Band D holds Scotland's vendored resource to the pinned Chronicle feed, and before it reads a row the stage checks that its Scottish declaration maps the facts to the frame's period, as council_tax_localise does; it refuses unless the bill's checks hold on what it writes, and its receipt records the bill's method, inputs and checks under council_tax, with no number." + }, + { + "artifacts": [ + { + "format": "json", + "kind": "public_aggregate_reference", + "resource": "brma_rent_quantiles.json", + "role": "rent_quantiles", + "runtime_sha256_required": true + }, + { + "format": "json", + "kind": "public_aggregate_reference", + "resource": "brma_private_rented_households.json", + "role": "count_resource", + "runtime_sha256_required": true + } + ], + "grain": "household", + "notes": "Reads each Great Britain private renter's rent (tenure_type RENT_PRIVATELY, a positive rent, outside Northern Ireland) off the list of rents of the BRMA its drawn area sits in (brma_rent_quantiles.json), as the dense line's private-rent node did (microcosm#1262, microcosm#931). The renter keeps its rank among the FRS renters of its FRS region and bedroom category, the rank reads the list of its BRMA and category, and each region is scaled so that, on the census private-renter mix of its BRMAs (brma_private_rented_households.json), its mean rent is the FRS level. Each row ranks as a household of its own, on its geography_household_key, and its tie-break uniform is keyed on that key on the stage's own stream at its declared seed, so two FRS renters at one rent, or two clones of one renter, take ranks of their own, and clone 0's tie-breaks are the same at every K. A region's FRS renters are counted in true source households, so a renter's clones and copies count once: a sampled spine ranks and scales a region with fewer FRS renters than sampled_region_renters against Great Britain's, and a full-scale spine ranks every region on its own and refuses one with none. Only rent is rewritten: reported Housing Benefit, Universal Credit and mortgage payments stay as reported, the weights pass through, and the stage records its household mass receipt. Its evidence gives K, the seed, the stream, the resources' digests, the method, the strictness, the numbers of (region, bedroom category) cells ranked against their region or against Great Britain and, among clone-0 rows, the distinct source households re-drawn, rounded by the shared disclosure rules.", + "operations": [ + { + "categories": [ + "B", + "C", + "D", + "E" + ], + "counts_resource": "brma_private_rented_households.json", + "excluded_regions": [ + "NORTHERN_IRELAND" + ], + "identity": "geography_household_key", + "kind": "localise_private_rents", + "method": "uk-private-rent-localisation-v1", + "process": "rent_rank", + "quantiles_method": "uk-brma-rent-quantiles-v1", + "quantiles_resource": "brma_rent_quantiles.json", + "reference_channel": "frs", + "sampled_region_renters": 10, + "scale_band": [ + 0.25, + 4 + ], + "seed": 0, + "stream": "uk-spine-private-rent-v1", + "tenure_type": "RENT_PRIVATELY", + "weeks_in_year": 52 + }, + { + "declared_factor": 1, + "entity": "household", + "gate_coupling": "The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this stage-specific reason.", + "kind": "record_mass_conservation_receipt", + "reason": "Private rents read off the BRMA lists on the source spine: household weights pass through unchanged and total household mass is conserved." + } + ], + "outputs": [], + "rewrites": [ + "rent" + ], + "source": "The BRMAs' weekly lists of rents (brma_rent_quantiles.json) and the census private-rented households by region, BRMA and bedrooms (brma_private_rented_households.json).", + "stage": "private_rent", + "survey": "Family Resources Survey 2024-25" + }, + { + "artifacts": [ + { + "format": "json", + "kind": "public_aggregate_reference", + "resource": "council_tax_band_d_england_wales_2024_25.json", + "role": "council_tax_band_d", + "runtime_sha256_required": true + }, + { + "format": "json", + "kind": "public_aggregate_reference", + "resource": "scotgov_band_d_council_tax_2024_25.json", + "role": "scotland_band_d_facts", + "runtime_sha256_required": true + } + ], + "grain": "household", + "nonnegative_outputs": [ + "council_tax_frs_bill" + ], + "notes": "Prices each Great Britain household's council tax at the local authority the spine drew for it, after private_rent and just before the WAS chain (María's rulings of 2026-11-06, 2026-11-13 and 2026-11-14, microcosm#931). Before it rewrites anything, it keeps the paid bill, max(council_tax - council_tax_rebate, 0), as council_tax_frs_bill on every row; was_wealth's council-tax predictor, which was_lisa builds through was_wealth, reads that column, so the area does not enter it, and the release boundary drops it. A household's ratio is its authority's (local_authority_code) 2024-25 Band D over the expected Band D of its own draw cell, recomputed exactly from the inputs atomic_area_assign drew it on, each of the cell's areas weighted as the draw weights it; the rows the SPI support channel copied, SPI copies and income band donors, take the cell spi_copy_areas drew them from, their region by households. A row of the FRS channel whose bill frs_council_tax observed gross (council_tax_bill_provenance observed_gross) takes the ratio; a missing bill, or one netted of a reduction of unknown size (missing, unknown_reduction), takes the 2024-25 statute where council_tax_band_observed holds (the authority's Band D times its band's statutory proportion, less the single-adult discount where council_tax_single_adult_raw is 1) and the ratio on frs_council_tax's bill where it does not. The rebate and the one person's reported council tax reduction are scaled by the same ratio and capped at the new bill. Northern Ireland, which pays domestic rates, keeps every row as it is, with no cap. A Great Britain row whose drawn area is not an area of its recomputed cell is refused. The WAS chain's HBAI net income reads the localised bill and reduction. The weights pass through, and the stage records its household mass receipt. Its evidence gives K, the Band D resources' and the supports' digests, the method and its checks as booleans, with the distinct source households on each path (the FRS channel's over every clone, the support channel's among clone-0 rows) and, among clone-0 rows, those whose reductions were capped and Northern Ireland's whose reductions exceed their bill, counted through receipt_counts.", + "operations": [ + { + "artifact_role": "scotland_band_d_facts", + "fail_on_mismatch": true, + "feed_pin": "chronicle_feed.json", + "kind": "verify_vendored_fact_resource", + "mapped_build_period": 2024, + "period_mapping": "fiscal_year_2024_25_is_the_build_period", + "record_sets": [ + "scotgov.ctas2026.band_d.fy2024" + ], + "require_before_source_read": true, + "resource": "scotgov_band_d_council_tax_2024_25.json", + "runtime_sha256_required": true, + "source_vintage": "2024-25 (ScotGov CTAS 2026)" + }, + { + "formula": "max(council_tax - council_tax_rebate, 0) on every row, from the bill and rebate before this stage rewrites them: the paid bill was_wealth's council-tax predictor reads, so the area does not enter that predictor (ruling 2)", + "kind": "derive", + "output": "council_tax_frs_bill" + }, + { + "authority_column": "local_authority_code", + "band_d_resources": { + "england_wales": "council_tax_band_d_england_wales_2024_25.json", + "scotland": "scotgov_band_d_council_tax_2024_25.json" + }, + "band_d_year": "2024-25", + "band_ratios": { + "ENGLAND": { + "A": "6/9", + "B": "7/9", + "C": "8/9", + "D": "9/9", + "E": "11/9", + "F": "13/9", + "G": "15/9", + "H": "18/9" + }, + "SCOTLAND": { + "A": "240/360", + "B": "280/360", + "C": "320/360", + "D": "360/360", + "E": "473/360", + "F": "585/360", + "G": "705/360", + "H": "882/360" + }, + "WALES": { + "A": "6/9", + "B": "7/9", + "C": "8/9", + "D": "9/9", + "E": "11/9", + "F": "13/9", + "G": "15/9", + "H": "18/9", + "I": "21/9" + } + }, + "cap": "reduction_and_rebate_at_bill", + "copy_draw_stage": "spi_copy_areas", + "draw_stage": "atomic_area_assign", + "excluded_regions": [ + "NORTHERN_IRELAND" + ], + "formula_classes": [ + "missing", + "unknown_reduction" + ], + "formula_needs": "council_tax_band_observed", + "kind": "localise_council_tax", + "method": "uk-council-tax-localise-v1", + "provenance_column": "council_tax_bill_provenance", + "ratio_classes": [ + "observed_gross" + ], + "reference": "expected_band_d_of_draw_cell", + "scaled_columns": { + "household": [ + "council_tax_rebate" + ], + "person": [ + "council_tax_benefit_reported" + ] + }, + "single_adult_column": "council_tax_single_adult_raw", + "single_adult_discount": 0.25, + "support_channel_rows": "ratio_at_copy_draw_level", + "support_sources": { + "uk_ew_output_area_2021": "uk_ew_output_area_2021_conditioned_support", + "uk_ni_data_zone_2021": "uk_ni_data_zone_2021_conditioned_support", + "uk_scotland_output_area_2022": "uk_scotland_output_area_2022_conditioned_support" + } + }, + { + "declared_factor": 1, + "entity": "household", + "gate_coupling": "The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this stage-specific reason.", + "kind": "record_mass_conservation_receipt", + "reason": "Council tax localised by each household's local authority on the source spine: household weights pass through unchanged and total household mass is conserved." + } + ], + "outputs": [ + "council_tax_frs_bill" + ], + "rewrites": [ + "council_tax", + "council_tax_rebate", + "council_tax_benefit_reported" + ], + "source": "Each Great Britain local authority's Band D council tax for 2024-25, keyed by April 2023 local authority code, England's from MHCLG's Council Tax levels set by local authorities in England 2024 to 2025 (Table 8) and Wales's from StatsWales (council_tax_band_d_england_wales_2024_25.json), and Scotland's from the Scottish Government's CTAS 2026 Council Tax Assumptions, vendored from the pinned Chronicle feed (scotgov_band_d_council_tax_2024_25.json); the statutory band proportions and single-adult discount (Local Government Finance Act 1992, as amended); and the conditioned atomic-area supports sources.yaml declares, on which each household's draw cell is recomputed.", + "stage": "council_tax_localise", + "survey": "Family Resources Survey 2024-25" }, { "stage": "was_wealth", "survey": "Wealth and Assets Survey round 8", - "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", + "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab. England and Wales owners' main residences are read off HM Land Registry Price Paid latest sales uprated with the UK House Price Index, by constituency (uk/high_value_home_value_knots.json, built by tools/build_uk_high_value_home_stock.py from PolicyEngine/chronicle#343's registrations), and the public regional reference (uk/regional_land_values.json) (microcosm#931).", "grain": "household", "base_candidate": { "filename": "populace_uk_2023.h5", @@ -3334,6 +4024,20 @@ "access": "private_local_input", "locator": "caller-supplied local input", "runtime_sha256_required": true + }, + { + "format": "json", + "kind": "public_aggregate_reference", + "resource": "high_value_home_value_knots.json", + "role": "value_knots", + "runtime_sha256_required": true + }, + { + "format": "json", + "kind": "public_aggregate_reference", + "resource": "regional_land_values.json", + "role": "regional_reference", + "runtime_sha256_required": true } ], "operations": [ @@ -3361,7 +4065,8 @@ "num_adults": "NumAdultR8", "num_children": "NumChildR8", "council_tax": "DVCTaxAmtAnnualR8", - "rental_income": "DVNetRentAmtAnnualR8_aggr" + "rental_income": "DVNetRentAmtAnnualR8_aggr", + "hbai_household_net_income": "DVTotinc_bhcR8" }, "derived": { "private_pension_wealth": "totalpenr8_aggr - dvvaldbt_scaper8_aggr", @@ -3385,7 +4090,7 @@ { "kind": "materialize_rules_engine_predictors", "predictors": [ - "household_net_income", + "hbai_household_net_income_before_reset_to_zero", "private_pension_income", "employment_income", "self_employment_income", @@ -3399,7 +4104,7 @@ "seed": 0, "n_estimators": 100, "predictors": [ - "household_net_income", + "hbai_household_net_income", "num_adults", "num_children", "private_pension_income", @@ -3510,6 +4215,40 @@ "secured_debt_within": "other_mortgage", "gate_calibration": "cross_fitted" }, + { + "after_segment": 2, + "constituency_column": "constituency_code", + "factor": "avg_house_price / unweighted_mean(drawn main_residence_value | region, FRS-base owners, >0), the regional property uprating's factor on the draw", + "kept": "Scotland, Northern Ireland, renters and owners without a positive draw keep the draw", + "kind": "localise_main_residence_value", + "knots": "high_value_home_value_knots.json", + "knots_level": "constituency", + "method": "uk-spine-area-home-values-v1", + "rank": "weighted mid-rank of the drawn main_residence_value among the owner households of the FRS region with a positive draw, both support channels, at the stage's household weights, ties broken by a draw keyed on the household id", + "rank_basis_column": "main_residence_value_rank_basis", + "region_map": { + "EAST_MIDLANDS": "E12000004", + "EAST_OF_ENGLAND": "E12000006", + "LONDON": "E12000007", + "NORTH_EAST": "E12000001", + "NORTH_WEST": "E12000002", + "SOUTH_EAST": "E12000008", + "SOUTH_WEST": "E12000009", + "WALES": "W92000004", + "WEST_MIDLANDS": "E12000005", + "YORKSHIRE": "E12000003" + }, + "regional_reference": "regional_land_values.json", + "rows": "owner households (tenure owned outright or with a mortgage) with a positive drawn main_residence_value in the FRS regions of England and Wales", + "stream": [ + "sha256-u53-v1", + "uk-spine-area-home-values-v1", + 0, + 0 + ], + "value": "np.interp(rank, grid, the assigned constituency's knots) divided by the FRS region's factor, a home at the knots' cap taking the largest quotient the factor multiplies back to the cap or below", + "value_column": "main_residence_value" + }, { "kind": "fold_into", "output": "corporate_wealth", @@ -3523,7 +4262,11 @@ }, { "kind": "support_clip", - "range": "donor_realized" + "range": "donor_realized", + "exempt": [ + "main_residence_value", + "property_wealth" + ] }, { "kind": "allocate_within_group_waterfall", @@ -3555,6 +4298,7 @@ "gross_financial_wealth", "net_financial_wealth", "main_residence_value", + "main_residence_value_rank_basis", "other_residential_property_value", "non_residential_property_value", "savings", @@ -3576,6 +4320,7 @@ "private_pension_wealth", "gross_financial_wealth", "main_residence_value", + "main_residence_value_rank_basis", "other_residential_property_value", "non_residential_property_value", "savings", @@ -3590,7 +4335,7 @@ "other_residential_property_secured_debt", "non_residential_property_secured_debt" ], - "notes": "Ports incumbent WAS round-8 wealth imputation with signed E5 differences: exact lower-case column matching replaces the fuzzy r/w fallback; cash ISA uses DVCISAVR8_aggr and stocks-and-shares ISA uses DVIISAVR8_aggr; private pension wealth (WAS total private pension wealth less the current defined-benefit component; DB-type components at the SCAPE rate, money-purchase components at reported fund values) is drawn as its own output private_pension_wealth and no longer folded into corporate_wealth, which keeps only the share-like holdings - pension rights are disregarded capital in every means test (UC Regs 2013 Sch 10 para 10 and the HB/JSA/ESA/IS/SPC parallels; uk-data#452) - and the downstream chain segment conditions on both components; each chain segment starts its own fit and draw RNG streams from a child seed derived deterministically from the declared seed (start_chain respawns from the model seed on every call, so one shared model coupled the k-th target of every segment); corporate_wealth folds stocks-and-shares ISA after drawing corporate_wealth_excl_isa; recipient Northern Ireland regions are mapped to Wales for prediction only; student_loan_balance is allocated by household id rather than the incumbent positional off-by-one. Its QRF sign gates are cross-fitted (gate_calibration: cross_fitted; microcosm#1359), as lcfs_consumption's are: each is calibrated on the donors its fold gates left out, so a sign is drawn at its donor rate on households the gates were not fitted to. Tenure enters every chain segment as one four-way category built the same way on both sides (microcosm#1063): the donor's from Ten1R8 and DVPriRntR8 (owned outright; buying with a mortgage or part rent, part mortgage; rented or rent-free from a private landlord; rented from a council or housing association), the recipient's from the frame's tenure_type (council and housing association together), so the stage reads no engine tenure formula. Two columns are structural in tenure and are fitted and drawn inside their stratum only, zero by rule elsewhere: main_residence_value on the owner tenures (every WAS owner has a positive value and no renter has one) and main_residence_mortgage (TotMortR8) on the mortgaged tenure. mortgage_debt is HMortGR8, all of the household's mortgages, and is derived as the main-residence mortgage plus the mortgages on other property (other_mortgage, HMortGR8 less TotMortR8 on the donor: buy-to-let, second homes and the rest), which are held on every tenure and drawn without a stratum, so an outright owner's or a renter's buy-to-let keeps its mortgage beside the property it is secured on (OthMortR8_sum is read for the receipt only); the donor's conditioning copy carries the same rule, so consumer_debt is fitted on values recipients can hold. Totals are not drawn: property_wealth is the sum of land, the main residence, other houses, buildings and a drawn remainder (buy-to-let, overseas and other property), gross_financial_wealth is savings, the cash ISA, the share-like holdings and a drawn remainder, and net_financial_wealth is gross less consumer debt and the student loan balance - identities the pinned tab satisfies on every row and the cleaning step refuses a donor that breaks. A sum of donor-valued components that leaves the donor's range for the total gives up the excess from the remainder first, and the receipt counts the rows. Eleven chain segments, each with its own child seed: land; the main residence; the other property components; pension and share-like wealth; the financial components, vehicles and the student loan balance; the main-residence mortgage; the other mortgages; consumer debt; then three appended after them, so the first eight draw as they did before them (microcosm#1095, the WAS property draw), each fitted on the donors and drawn for the recipients holding the column holder_strata names. The first is the directly held fraction of the share-like holdings outside ISAs, from which the stage writes directly_held_shares (WAS UK shares plus employee shares and options) and unit_and_investment_trusts (WAS collective investments, which the survey asks about in one question; uk-data#501), the inputs policyengine-uk reads inside corporate_wealth (pe-uk#1974): the split is written on the final corporate_wealth less the stocks-and-shares ISA, so the two and the ISA sum to corporate_wealth and the engine's share_like_capital stays corporate_wealth; a household with such holdings but no drawn fraction (a clip that lowered the ISA alone) holds them directly (undrawn_fraction), and the receipt counts it. The other two are the debt secured on other residential property (WAS DVHseDebtR8_sum + DVBLtDebtR8_sum) and on non-residential property (DVBldDebtR8_sum), which policyengine-uk deducts after the sale costs in valuing Universal Credit property capital (UC Regs 2013 reg 49(1)(b)): each is zero where the final value of its asset is zero, and the two are parts of the mortgages on other property, so where their sum passes other_mortgage both are scaled down to it (secured_debt_within); the receipt counts the rows each rule touched, and the wealth_coherence gate holds the split and the debts as identities. HFINWNTR8_exSLC_Sum is net financial wealth excluding student loans, so gross less it is the non-student-loan financial liabilities; UKDS SN 7215 documentation is the codebook. Engine predictors materialize at their native entity and person/benunit values are summed to household, reproducing the incumbent map_to=household semantics; region is one-hot encoded jointly across donor and recipient (the incumbent's dummy encoding), with unmapped donor GOR codes becoming all-zero dummy rows. The WAS and FRS predictor definitions are not fully like-for-like and are ported as-is; raw WAS missing values are blanket-filled with zero; UKDS negative sentinel codes (-9/-8/-7/-6) are recoded to zero for the nonnegative-domain columns the licensed audit found carrying them (vcarnr8: 2 rows; HBedRmR8: 95.8 percent - the bedrooms question is effectively unasked in the WAS household file, predictor-quality revisit registered on microcosm#145) - a signed difference vs the incumbent, which trains on raw sentinels; DVPriRntR8's -9 is structural not-applicable so the private-renter reading is unchanged; genuinely negative domains are never recoded. The recipient council_tax predictor is the bill the household pays, council_tax less council_tax_rebate, the quantity the WAS donor records, since the spine's council_tax is the liability before council tax reduction (uk-data#496/#499, microcosm#1095). The donor council_tax is DVCTaxAmtAnnualR8, the annual amount paid (CTAmtR8 is one instalment), its children are NumChildR8 (NumCh18R8 overlaps NumAdultR8), and the recipient adults and children are the household's FRS claimants and partners and its other members (is_uc_claimant), which together make the household size on both sides; rental_income is the donor's net rent DVNetRentAmtAnnualR8_aggr against the household's property_income (uk-data#486/#495, microcosm#1095)." + "notes": "Ports incumbent WAS round-8 wealth imputation with signed E5 differences: exact lower-case column matching replaces the fuzzy r/w fallback; cash ISA uses DVCISAVR8_aggr and stocks-and-shares ISA uses DVIISAVR8_aggr; private pension wealth (WAS total private pension wealth less the current defined-benefit component; DB-type components at the SCAPE rate, money-purchase components at reported fund values) is drawn as its own output private_pension_wealth and no longer folded into corporate_wealth, which keeps only the share-like holdings - pension rights are disregarded capital in every means test (UC Regs 2013 Sch 10 para 10 and the HB/JSA/ESA/IS/SPC parallels; uk-data#452) - and the downstream chain segment conditions on both components; each chain segment starts its own fit and draw RNG streams from a child seed derived deterministically from the declared seed (start_chain respawns from the model seed on every call, so one shared model coupled the k-th target of every segment); corporate_wealth folds stocks-and-shares ISA after drawing corporate_wealth_excl_isa; recipient Northern Ireland regions are mapped to Wales for prediction only; student_loan_balance is allocated by household id rather than the incumbent positional off-by-one. Its QRF sign gates are cross-fitted (gate_calibration: cross_fitted; microcosm#1359), as lcfs_consumption's are: each is calibrated on the donors its fold gates left out, so a sign is drawn at its donor rate on households the gates were not fitted to. Tenure enters every chain segment as one four-way category built the same way on both sides (microcosm#1063): the donor's from Ten1R8 and DVPriRntR8 (owned outright; buying with a mortgage or part rent, part mortgage; rented or rent-free from a private landlord; rented from a council or housing association), the recipient's from the frame's tenure_type (council and housing association together), so the stage reads no engine tenure formula. Two columns are structural in tenure and are fitted and drawn inside their stratum only, zero by rule elsewhere: main_residence_value on the owner tenures (every WAS owner has a positive value and no renter has one) and main_residence_mortgage (TotMortR8) on the mortgaged tenure. mortgage_debt is HMortGR8, all of the household's mortgages, and is derived as the main-residence mortgage plus the mortgages on other property (other_mortgage, HMortGR8 less TotMortR8 on the donor: buy-to-let, second homes and the rest), which are held on every tenure and drawn without a stratum, so an outright owner's or a renter's buy-to-let keeps its mortgage beside the property it is secured on (OthMortR8_sum is read for the receipt only); the donor's conditioning copy carries the same rule, so consumer_debt is fitted on values recipients can hold. Totals are not drawn: property_wealth is the sum of land, the main residence, other houses, buildings and a drawn remainder (buy-to-let, overseas and other property), gross_financial_wealth is savings, the cash ISA, the share-like holdings and a drawn remainder, and net_financial_wealth is gross less consumer debt and the student loan balance - identities the pinned tab satisfies on every row and the cleaning step refuses a donor that breaks. A sum of donor-valued components that leaves the donor's range for the total gives up the excess from the remainder first, and the receipt counts the rows. Eleven chain segments, each with its own child seed: land; the main residence; the other property components; pension and share-like wealth; the financial components, vehicles and the student loan balance; the main-residence mortgage; the other mortgages; consumer debt; then three appended after them, so the first eight draw as they did before them (microcosm#1095, the WAS property draw), each fitted on the donors and drawn for the recipients holding the column holder_strata names. The first is the directly held fraction of the share-like holdings outside ISAs, from which the stage writes directly_held_shares (WAS UK shares plus employee shares and options) and unit_and_investment_trusts (WAS collective investments, which the survey asks about in one question; uk-data#501), the inputs policyengine-uk reads inside corporate_wealth (pe-uk#1974): the split is written on the final corporate_wealth less the stocks-and-shares ISA, so the two and the ISA sum to corporate_wealth and the engine's share_like_capital stays corporate_wealth; a household with such holdings but no drawn fraction (a clip that lowered the ISA alone) holds them directly (undrawn_fraction), and the receipt counts it. The other two are the debt secured on other residential property (WAS DVHseDebtR8_sum + DVBLtDebtR8_sum) and on non-residential property (DVBldDebtR8_sum), which policyengine-uk deducts after the sale costs in valuing Universal Credit property capital (UC Regs 2013 reg 49(1)(b)): each is zero where the final value of its asset is zero, and the two are parts of the mortgages on other property, so where their sum passes other_mortgage both are scaled down to it (secured_debt_within); the receipt counts the rows each rule touched, and the wealth_coherence gate holds the split and the debts as identities. HFINWNTR8_exSLC_Sum is net financial wealth excluding student loans, so gross less it is the non-student-loan financial liabilities; UKDS SN 7215 documentation is the codebook. Engine predictors materialize at their native entity and person/benunit values are summed to household, reproducing the incumbent map_to=household semantics; region is one-hot encoded jointly across donor and recipient (the incumbent's dummy encoding), with unmapped donor GOR codes becoming all-zero dummy rows. The household income predictor, hbai_household_net_income, matches the donor's DVTotinc_bhcR8, net regular household income before housing costs less council tax, to the engine's closest concept, policyengine-uk's HBAI net income before housing costs and before the reset to zero (hbai_household_net_income_before_reset_to_zero, which the LCFS, ETB and NTS stages also read; microcosm#1164). The engine's household_net_income it replaces also takes off stamp duty, the TV licence and the business rates and corporate stamp duty that shareholding spreads over the households the engine simulates, and adds the imputed value of NHS, education and transport spending (microcosm#931). The WAS and FRS predictor definitions, that income among them, are not fully like-for-like and are ported as-is: the recipient's income also counts HBAI's in-kind items (free school meals, school milk and fruit, the free TV licence's value, Healthy Start vouchers and the Tax-Free Childcare top-up) and the Warm Home Discount, which WAS's regular income is not documented to include, and how WAS treats pension contributions, maintenance paid, student loan repayments and negative incomes is not verified; raw WAS missing values are blanket-filled with zero; UKDS negative sentinel codes (-9/-8/-7/-6) are recoded to zero for the nonnegative-domain columns the licensed audit found carrying them (vcarnr8: 2 rows; HBedRmR8: 95.8 percent - the bedrooms question is effectively unasked in the WAS household file, predictor-quality revisit registered on microcosm#145) - a signed difference vs the incumbent, which trains on raw sentinels; DVPriRntR8's -9 is structural not-applicable so the private-renter reading is unchanged; genuinely negative domains are never recoded. The recipient council_tax predictor is the bill the household pays, council_tax less council_tax_rebate, the quantity the WAS donor records, since the spine's council_tax is the liability before council tax reduction (uk-data#496/#499, microcosm#1095). The donor council_tax is DVCTaxAmtAnnualR8, the annual amount paid (CTAmtR8 is one instalment), its children are NumChildR8 (NumCh18R8 overlaps NumAdultR8), and the recipient adults and children are the household's FRS claimants and partners and its other members (is_uc_claimant), which together make the household size on both sides; rental_income is the donor's net rent DVNetRentAmtAnnualR8_aggr against the household's property_income (uk-data#486/#495, microcosm#1095)." }, { "stage": "was_lisa", @@ -3709,7 +4454,7 @@ "employment_income" ], "household_predictors": [ - "household_net_income", + "hbai_household_net_income", "num_adults", "num_children", "gross_financial_wealth", @@ -3726,9 +4471,7 @@ { "kind": "materialize_rules_engine_predictors", "predictors": [ - "household_net_income", - "num_adults", - "num_children", + "hbai_household_net_income_before_reset_to_zero", "private_pension_income", "employment_income", "self_employment_income", @@ -3755,7 +4498,7 @@ ], "log1p_predictors": [ "employment_income", - "household_net_income", + "hbai_household_net_income", "gross_financial_wealth", "cash_isa", "stocks_and_shares_isa", @@ -3799,7 +4542,7 @@ "age_band", "is_female", "employment_income", - "household_net_income", + "hbai_household_net_income", "gross_financial_wealth", "savings", "cash_isa", @@ -3843,7 +4586,7 @@ "lifetime_isa_balance", "household_lifetime_isa_balance" ], - "notes": "Imputes Lifetime ISA holdings per adult from the WAS round-8 person tab (microcosm#1003). The donor is every non-dependent adult of the person tab joined many-to-one on CASER8 to the cleaned household tab (the was_wealth donor, same pins); the released value DVFLISAvR8 (the ONS-completed FLISAVR8_i) must sum to the household aggregate DVFLISAVR8_aggr on every household or the stage refuses the tab, and a missing or negative value is refused rather than read as a zero balance. Response classes (observed, banded with the ONS value inside the reported band, ONS-imputed value or holding, rule-impossible) are receipted and never used as predictors. The declared credibility rule recodes holders recorded in an age band starting at 45 or above to non-holders and keeps holders above the balance ceiling as holders outside the balance fit (basis on the clean operation). Ownership is a weighted logistic regression with a ridge penalty on the slopes and an unpenalised intercept, so the donor's weighted ownership share is reproduced in-sample, on the age group, the log of person earnings, household net income, gross financial wealth, cash ISA, stocks-and-shares ISA and savings, private renting, sex and the number of children; the house regime gate (gradient boosting) is miscalibrated for an outcome held by under one per cent of adults with about 120 donor holders. Recipients aged 18 or over hold when a uniform keyed on person_id falls below their probability; no upper age limit is imposed. The balance is the regime-gated QRF fitted on the credible holders and drawn at the owner's keyed quantile (predict_positive_from_uniforms), so it is positive exactly when the person holds. Household predictors reuse the was_wealth definitions; tenure is the private-renter flag on both sides (DVPriRntR8 == 1; tenure_type RENT_PRIVATELY), not the engine is_renting. WAS counts LISAs inside gross financial wealth, so a household total above its gross_financial_wealth draw is scaled down pro rata (receipted) and ownership is re-derived from the capped balance; was_wealth columns are never rewritten. Balances are April 2020 to March 2022 pounds from a Great Britain donor and are not uprated; Northern Ireland recipients are predicted from the GB model. WAS records holdings, not annual contributions, and the stage derives no contribution or bonus. HMRC's 2024-25 Lifetime ISA accounts subscribed to (Table 9.4) anchor the holders (microcosm#1415): the ownership logit shifts up on ages 18 to 47, the ages that can hold and pay into one in 2024-25 (opened at 18 to 39 since April 2017), until the holders at those ages who survive the financial-wealth cap equal HMRC's payers, the fewest holders consistent with them; the same keyed uniforms decide ownership, so the anchor only adds holders, and a target it cannot reach is named, not failed. Where the cap binds, an added holder scales the household's balances down pro rata, existing holders' included. Three later stages read the whole frame (uc_reporter_redraw, uc_take_up, pension_credit_take_up), and the Lifetime ISA counts as Universal Credit and Pension Credit capital, so the added holders can move their draws; the pair build counts the benefit units whose entitlement moves (María's ruling of 29 October 2026)." + "notes": "Imputes Lifetime ISA holdings per adult from the WAS round-8 person tab (microcosm#1003). The donor is every non-dependent adult of the person tab joined many-to-one on CASER8 to the cleaned household tab (the was_wealth donor, same pins); the released value DVFLISAvR8 (the ONS-completed FLISAVR8_i) must sum to the household aggregate DVFLISAVR8_aggr on every household or the stage refuses the tab, and a missing or negative value is refused rather than read as a zero balance. Response classes (observed, banded with the ONS value inside the reported band, ONS-imputed value or holding, rule-impossible) are receipted and never used as predictors. The declared credibility rule recodes holders recorded in an age band starting at 45 or above to non-holders and keeps holders above the balance ceiling as holders outside the balance fit (basis on the clean operation). Ownership is a weighted logistic regression with a ridge penalty on the slopes and an unpenalised intercept, so the donor's weighted ownership share is reproduced in-sample, on the age group, the log of person earnings, household net income (the was_wealth predictor: DVTotinc_bhcR8 on the donor and, on the recipient, the engine's closest concept, policyengine-uk's HBAI net income before the reset to zero, not fully like-for-like; microcosm#931), gross financial wealth, cash ISA, stocks-and-shares ISA and savings, private renting, sex and the number of children; the house regime gate (gradient boosting) is miscalibrated for an outcome held by under one per cent of adults with about 120 donor holders. Recipients aged 18 or over hold when a uniform keyed on person_id falls below their probability; no upper age limit is imposed. The balance is the regime-gated QRF fitted on the credible holders and drawn at the owner's keyed quantile (predict_positive_from_uniforms), so it is positive exactly when the person holds. Household predictors reuse the was_wealth definitions; tenure is the private-renter flag on both sides (DVPriRntR8 == 1; tenure_type RENT_PRIVATELY), not the engine is_renting. WAS counts LISAs inside gross financial wealth, so a household total above its gross_financial_wealth draw is scaled down pro rata (receipted) and ownership is re-derived from the capped balance; was_wealth columns are never rewritten. Balances are April 2020 to March 2022 pounds from a Great Britain donor and are not uprated; Northern Ireland recipients are predicted from the GB model. WAS records holdings, not annual contributions, and the stage derives no contribution or bonus. HMRC's 2024-25 Lifetime ISA accounts subscribed to (Table 9.4) anchor the holders (microcosm#1415): the ownership logit shifts up on ages 18 to 47, the ages that can hold and pay into one in 2024-25 (opened at 18 to 39 since April 2017), until the holders at those ages who survive the financial-wealth cap equal HMRC's payers, the fewest holders consistent with them; the same keyed uniforms decide ownership, so the anchor only adds holders, and a target it cannot reach is named, not failed. Where the cap binds, an added holder scales the household's balances down pro rata, existing holders' included. Three later stages read the whole frame (uc_reporter_redraw, uc_take_up, pension_credit_take_up), and the Lifetime ISA counts as Universal Credit and Pension Credit capital, so the added holders can move their draws; the pair build counts the benefit units whose entitlement moves (María's ruling of 29 October 2026)." }, { "stage": "regional_property_uprating", @@ -3868,9 +4611,10 @@ { "kind": "uprate_to_regional_reference", "resource": "regional_land_values.json", - "factor": "avg_house_price / unweighted_mean(main_residence_value | region, >0)", + "factor": "avg_house_price / unweighted_mean(main_residence_value_rank_basis | region, FRS-base owners, main_residence_value > 0)", "owner_rows": "main_residence_value > 0", - "skip_empty_or_nonpositive_regions": true + "skip_empty_or_nonpositive_regions": true, + "denominator": "main_residence_value_rank_basis" }, { "kind": "record_mass_conservation_receipt", @@ -3888,7 +4632,7 @@ "other_residential_property_value", "non_residential_property_value" ], - "notes": "Deterministically rescales owner rows so regional unweighted owner means match the public house-price reference. Northern Ireland has no reference row and is never scaled; empty and nonpositive regions are skipped. The unweighted mean follows the incumbent behavior. The factor is read off the main residence and has always scaled the property total; since microcosm#1063 it scales the total's other components in the frame as well (land, other houses, buildings), so an owner's property_wealth stays the sum of its components after the uprating. Mortgage debt is never uprated." + "notes": "Deterministically rescales owner rows by each region's factor: its public house-price reference over the unweighted mean of main_residence_value_rank_basis, the WAS chain's draw of the main residence, among its FRS-base owners. was_wealth keeps that draw beside the values it reads off each England and Wales owner's constituency, which it divides by the same factor, so the uprating returns those values to the constituency's knots (microcosm#931); Scotland's owners hold the draw itself, so the region's unweighted owner mean matches the reference as before. Northern Ireland has no reference row and is never scaled; empty and nonpositive regions are skipped. The unweighted mean follows the incumbent behavior. The factor has always scaled the property total; since microcosm#1063 it scales the total's other components in the frame as well (land, other houses, buildings), so an owner's property_wealth stays the sum of its components after the uprating. Mortgage debt is never uprated." }, { "stage": "high_value_home_support", @@ -3901,17 +4645,19 @@ "kind": "split_top_value_owner_households", "stratum_lower": 0.95, "copies": 6, - "stratum_rule": "within each FRS region, owner households (tenure owned outright or with a mortgage, main_residence_value > 0, positive weight) ordered by main_residence_value ascending then household_id ascending; a household whose weighted mid-rank (cumulative weight before it plus half its own, over the region's owner weight) is at or above stratum_lower is selected", - "copy_rule": "every selected household becomes copies rows at weight / copies", - "id_remapping": "id_multiplier_for_values; copy k takes every entity id + k x multiplier", + "stratum_rule": "within each FRS region, owner households (tenure owned outright or with a mortgage, main_residence_value > 0, positive weight) ordered by main_residence_value ascending, households tied on it in ascending order of a uniform keyed on their own household_id under seed and salt, then household_id ascending; a household whose weighted mid-rank (cumulative weight before it plus half its own, over the region's owner weight) is at or above stratum_lower is selected", + "copy_rule": "every selected household becomes copies rows at weight / copies, except where none of the 64 heaviest households left whole, tried heaviest first, tied weights in row order, can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight", + "id_remapping": "copy-id table field high_value_support; copy k takes every entity id + k x the field's radix (10**12)", "flag_column": "household_is_high_value_support_copy", "copies_column": "high_value_support_copies", "copy_index_column": "high_value_support_copy_index", - "values": "every column copied unchanged; no draw, no seed, no salt", + "values": "every column copied unchanged; the stratum's tie-break is the stage's only draw", "weight_kind_out": "importance", "conservation": "exact_total", "declared_factor": 1.0, - "reason": "High-value home support divides every owner household in the top value stratum of its region into light copies at equal weight; every household's mass and the total household mass are conserved." + "reason": "High-value home support divides every owner household in the top value stratum of its region into light copies at equal weight; every household's mass and the total household mass are conserved.", + "salt": "high_value_home_support:value_tie", + "seed": 0 } ], "outputs": [ @@ -3920,71 +4666,7 @@ "high_value_support_copy_index" ], "rewrites": [], - "notes": "Spine-only, deterministic, mass-conserving support split for the dense graph's post-geography home value map (microcosm#1240). The value node reads every owner clone's home value off its assigned constituency's Land Registry sale distribution at the household's weighted rank among its region's owners, so each seat's homes over 2 million rest on the few households whose rank is in the top of the region; this stage copies every owner household in the top five per cent of its region six ways at equal weight, each copy carrying its index in high_value_support_copy_index (0 on roots), the explicit lineage the geography identity keys on, and the value node gives each copy its own keyed rank inside the stratum. No value is assigned and no draw is made here." - }, - { - "stage": "regional_home_values", - "survey": "HM Land Registry Price Paid latest sales uprated with the UK House Price Index, owner-weighted by region (the dense value node's knots, region level)", - "source": "uk/high_value_home_value_knots.json, built by tools/build_uk_high_value_home_stock.py from PolicyEngine/chronicle#343's registrations", - "grain": "household", - "artifacts": [ - { - "role": "value_knots", - "kind": "public_aggregate_reference", - "resource": "high_value_home_value_knots.json", - "format": "json", - "runtime_sha256_required": true - } - ], - "operations": [ - { - "kind": "assign_regional_home_values", - "knots": "high_value_home_value_knots.json", - "knots_level": "region", - "region_map": { - "EAST_MIDLANDS": "E12000004", - "EAST_OF_ENGLAND": "E12000006", - "LONDON": "E12000007", - "NORTH_EAST": "E12000001", - "NORTH_WEST": "E12000002", - "SOUTH_EAST": "E12000008", - "SOUTH_WEST": "E12000009", - "WALES": "W92000004", - "WEST_MIDLANDS": "E12000005", - "YORKSHIRE": "E12000003" - }, - "rows": "owner households (tenure owned outright or with a mortgage) with main_residence_value > 0 in the mapped regions", - "rank": "weighted mid-rank of main_residence_value among the owner households of the FRS region at the spine's weights, ties broken by a keyed draw; high-value support copies take support_stratum_lower + (1 - support_stratum_lower) * draw", - "support_stratum_lower": 0.95, - "stream": [ - "sha256-u53-v1", - "uk-spine-regional-home-values-v1", - 0, - 0 - ], - "value": "np.interp(rank, grid, region knots)", - "property_wealth": "moves by the value's change, floored at 0", - "rank_basis_column": "main_residence_value_rank_basis" - }, - { - "kind": "record_mass_conservation_receipt", - "entity": "household", - "reason": "Regional home values assigned on the source spine: household weights pass through unchanged and total household mass is conserved.", - "declared_factor": 1.0, - "gate_coupling": "The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this stage-specific reason." - } - ], - "outputs": [ - "main_residence_value_rank_basis" - ], - "rewrites": [ - "main_residence_value", - "property_wealth" - ], - "nonnegative_outputs": [ - "main_residence_value_rank_basis" - ], - "notes": "The national line's home values (microcosm#1112, microcosm#1095). The national release solves the bound spine, which has no area, so main_residence_value stayed the WAS-imputed value after the regional uprating, not a Land Registry value; the dense line reads values off the assigned constituency's distribution only after areas exist (uk.full.property.value, microcosm#1240). This stage gives every England-and-Wales owner household with a value the one its region's uprated Land Registry distribution holds at the household's weighted mid-rank among the owner households of its FRS region, at the spine's weights, ranked as the dense node ranks (ties broken by a keyed draw; the high-value support copies spread over the top stratum by their own draws), and moves property_wealth by the change. Scotland, Northern Ireland, renters and owners without a value keep theirs, and the council-tax band is untouched. The replaced value is kept in main_residence_value_rank_basis, which the dense node ranks on, so the dense line's values, wealth and bands do not change; the release boundary drops it on both lines. The value reaches three national rows: obr.sdlt through the main-home purchase value, and ons.land.land_value and ons.land.household_land_value through property_wealth and the engine's land intensity; corporate land value does not move. The region knots run up to each region's largest sale and down to its smallest, as the dense line's constituency knots do, so the national line places more homes in the top surcharge bands than the derived stock facts; the tails are to be fixed on both lines together (#1112, María's ruling of 2026-10-17)." + "notes": "Spine-only, deterministic, mass-conserving support split (microcosm#1240). Within each FRS region, every owner household in the top five per cent of the region's owner weight by main_residence_value is copied six ways at equal weight, each copy carrying its index in high_value_support_copy_index (0 on roots) and its ids minted in the copy-id table's high_value_support field. The stage runs after the regional property uprating, so it selects on the values was_wealth reads off each England and Wales owner's constituency, which the uprating returns to their knots (microcosm#931), and every copy keeps its parent's value, area and geography key. The exact-total correction lands on one of the 64 heaviest households the stage leaves whole, and on a root or copy only where none of them can carry it (copy_rule). No value is assigned here; the one draw is the stratum's tie-break, a uniform keyed on each household's own household_id (seed, salt) that orders owners tied on value, so no copy or clone ordinal ranks highest on every tie." }, { "stage": "property_components", @@ -4086,14 +4768,15 @@ 100000 ], "band_basis": "PRIS 2024-25 Table 13 (all tax entities) scaled by Table 1's individuals' share of landlords", - "order": "profit descending, person_id ascending", + "order": "profit descending, landlords tied on profit in ascending order of a uniform keyed on person_id under walk_seed and tie_salt, then person_id ascending", "walk": "top band first, each band filled with whole persons to its share of the individuals-basis landlords times the spine's landlord weight (the spine's total before calibration differs from PRIS's, and filling the published counts would push every profit rank into a higher band), at one seeded systematic offset (the CGT Table 3 walk's rule); the lowest band takes every remaining landlord", "walk_seed": 0, "within_band": "linear in the weight-ranked position within the band, from its upper bound for the highest profit to its lower bound; never below the landlord's profit", "top_band": "max(100,000, k x profit) with k at least 1, solved by bisection so the band's weighted mean receipts equal the individuals' Table 2 receipts less the bounded bands' realised mean receipts at their individuals-basis counts, over the top band's count; where the floor at k = 1 already exceeds that mean the multiplier stays 1 and the overshoot per landlord is recorded", "maximum_top_band_multiplier": 5.0, "sublet_receipts": "the household reference person's sub-letting rent adds to receipts with no expenses; Rent a Room receipts, which the engine's receipts concept excludes, until PolicyEngine/policyengine-uk#2002 is released", - "residuals": "landlords without profit (FRS loss-makers, whose loss counts as zero, and tape landlords at zero) carry no receipts although PRIS counts them; property_finance_costs_brought_forward stays unset" + "residuals": "landlords without profit (FRS loss-makers, whose loss counts as zero, and tape landlords at zero) carry no receipts although PRIS counts them; property_finance_costs_brought_forward stays unset", + "tie_salt": "allocate_property_rental_income:tie" }, { "kind": "record_mass_conservation_receipt", @@ -6849,7 +7532,7 @@ "frs_benunit_capital", "would_claim_uc" ], - "notes": "Runs after the last universal_credit_reported writer and before CGT cloning. Redraws every SPI benefit unit's capital, since the channel's incomes are SPI draws and the copied FRS answer no longer belongs to the unit (uk-data#495, microcosm#1095), from weighted base-FRS units with available capital in cells of UC reporter status by financial investment income band (interest, dividends and other investment income; property income is excluded because TOTCAPB4 counts accounts and assets) by is_uc_couple (two retained FRS claimant/partner roles, independently of legal is_married) by dependent-children band. A cell with fewer than 20 donors widens in the declared coarsening order, and reporter status is never coarsened. It then applies the monotone reported-UC OR refresh. The recorded capitals add property (uk-data#495, microcosm#1095), since policyengine-uk 2.177.7 replaces every capital source with a recorded figure of 0 or more and takes it as countable, and TOTCAPB4 has no property. WAS records the property for the household, and each means test counts the claimant's and partner's own capital (UC Regs 2013 reg 18; SPCA 2002 s.5). The household reference person's benefit unit therefore holds all of the household's property, and no other unit records any (María's ruling of 2026-10-12), since that person is the one in whose name the home is owned or rented, the best single reading of the owner. The engine shares household property over every claimant or partner instead, which puts a parent's property on an adult child's claim; a recorded figure consumes its own unit's share, so the engine's shares reach only a unit on the unavailable sentinel, and since policyengine-uk 2.136.1 such a unit takes its share whatever a recording unit beside it holds (the receipt counts any unit left there). The property counts at 90% of its value, market value less 10% for the costs of sale, less the debt secured on it, class by class and floored at zero per class (UC Regs 2013 reg 49(1); SPC Regs 2002 reg 19), as policyengine-uk values Universal Credit property capital. The was_wealth stage draws the debt on other residential and on non-residential property (microcosm#1095), and owned land carries none, as the engine has no input for it (pe-uk#1969). Universal Credit counts other residential and non-residential property, and Pension Credit counts those and owned land. From policyengine-uk 2.173.3 Council Tax Reduction counts the same three at the same value, from the household's own sources, except that a working-age applicant with Universal Credit takes the Universal Credit assessment. A unit that reports Universal Credit keeps its receipt and records the carrier alone for Universal Credit, since DWP assessed its capital to pay it and the property is imputed without that receipt. frs_benunit_capital stays the financial carrier, and its unavailable sentinel -1 passes through to both. Both recorded figures project to later years by GDP per head, as financial capital does, while policyengine-uk 2.139.1 projects other residential property by house prices, so the other residential property in a recorded figure grows faster than the engine's own projection of it; by the OBR indices the engine carries, it grows 1.5% faster by 2025 and 2.0% by 2026 from a 2024 base." + "notes": "Runs after the last universal_credit_reported writer and before CGT cloning. Redraws every SPI benefit unit's capital, since the channel's incomes are SPI draws and the copied FRS answer no longer belongs to the unit (uk-data#495, microcosm#1095), from weighted base-FRS units with available capital in cells of UC reporter status by financial investment income band (interest, dividends and other investment income; property income is excluded because TOTCAPB4 counts accounts and assets) by is_uc_couple (two retained FRS claimant/partner roles, independently of legal is_married) by dependent-children band. A cell with fewer than 20 donor survey records (a unit's geographic clones count once, every other copy as a record of its own) widens in the declared coarsening order, and reporter status is never coarsened. It then applies the monotone reported-UC OR refresh. The recorded capitals add property (uk-data#495, microcosm#1095), since policyengine-uk 2.177.7 replaces every capital source with a recorded figure of 0 or more and takes it as countable, and TOTCAPB4 has no property. WAS records the property for the household, and each means test counts the claimant's and partner's own capital (UC Regs 2013 reg 18; SPCA 2002 s.5). The household reference person's benefit unit therefore holds all of the household's property, and no other unit records any (María's ruling of 2026-10-12), since that person is the one in whose name the home is owned or rented, the best single reading of the owner. The engine shares household property over every claimant or partner instead, which puts a parent's property on an adult child's claim; a recorded figure consumes its own unit's share, so the engine's shares reach only a unit on the unavailable sentinel, and since policyengine-uk 2.136.1 such a unit takes its share whatever a recording unit beside it holds (the receipt counts any unit left there). The property counts at 90% of its value, market value less 10% for the costs of sale, less the debt secured on it, class by class and floored at zero per class (UC Regs 2013 reg 49(1); SPC Regs 2002 reg 19), as policyengine-uk values Universal Credit property capital. The was_wealth stage draws the debt on other residential and on non-residential property (microcosm#1095), and owned land carries none, as the engine has no input for it (pe-uk#1969). Universal Credit counts other residential and non-residential property, and Pension Credit counts those and owned land. From policyengine-uk 2.173.3 Council Tax Reduction counts the same three at the same value, from the household's own sources, except that a working-age applicant with Universal Credit takes the Universal Credit assessment. A unit that reports Universal Credit keeps its receipt and records the carrier alone for Universal Credit, since DWP assessed its capital to pay it and the property is imputed without that receipt. frs_benunit_capital stays the financial carrier, and its unavailable sentinel -1 passes through to both. Both recorded figures project to later years by GDP per head, as financial capital does, while policyengine-uk 2.139.1 projects other residential property by house prices, so the other residential property in a recorded figure grows faster than the engine's own projection of it; by the OBR indices the engine carries, it grows 1.5% faster by 2025 and 2.0% by 2026 from a 2024 base." }, { "stage": "uc_limited_capability", @@ -7395,18 +8078,20 @@ "other_residential_property_value", "non_residential_property_value" ], - "selection": "within each income band walk households in investable wealth descending, household_id ascending, accumulating positive household weight until the cumulative weight reaches the band's support mass; whole households; an exhausted pool is recorded, not refused", + "selection": "within each income band walk households in investable wealth descending, households tied on it in ascending order of a uniform keyed on their own household_id under seed and salt, then household_id ascending, accumulating positive household weight until the cumulative weight reaches the band's support mass; whole households; an exhausted pool is recorded, not refused", "maximum_copy_weight": 60.0, - "copy_rule": "copies = ceil(household weight / maximum_copy_weight); every copy carries weight / copies", - "id_remapping": "id_multiplier_for_values; copy k takes every entity id + k x multiplier", + "copy_rule": "copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies, except where none of the 64 heaviest households left whole, tried heaviest first, tied weights in row order, can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight; of those 64, a selected household left at one copy can carry the correction only where its corrected weight keeps its one copy and the walk's selection, a check the fallback does not make", + "id_remapping": "copy-id table field cgt_support_split; copy k takes every entity id + k x the field's radix (10**13), k up to 9,999, and a household needing more copies is refused", "flag_column": "household_is_cgt_support_copy", "copies_column": "cgt_support_copies", "copy_index_column": "cgt_support_copy_index", - "values": "every column copied unchanged; no draw, no seed, no salt", + "values": "every column copied unchanged; the selection's tie-break is the stage's only draw", "weight_kind_out": "importance", "conservation": "exact_total", "declared_factor": 1.0, - "reason": "Capital-gains support split divides the wealthiest households of each HMRC Table 3 income band into light copies at equal weight; every household's mass and the total household mass are conserved." + "reason": "Capital-gains support split divides the wealthiest households of each HMRC Table 3 income band into light copies at equal weight; every household's mass and the total household mass are conserved.", + "salt": "cgt_support_split:wealth_tie", + "seed": 0 } ], "outputs": [ @@ -7415,7 +8100,7 @@ "cgt_support_copy_index" ], "rewrites": [], - "notes": "Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 before the clone (30 after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. No value is assigned and no draw is made; the clone stage gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact)." + "notes": "Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. The exact-total correction lands on one of the 64 heaviest households the split leaves whole, and on a root or copy only where none of them can carry it (copy_rule). The shared fallback does not make the one-copy check copy_rule names, so where none of the 64 can carry the correction, the fallback can put it on a one-copy household the check passed over, and the identity tool's selection recompute can then count that household as a family the stage never made, which the bracketed family test does not report, or miss one it made, as under the rule before. No value is assigned, and the one draw is the walk's tie-break, a uniform keyed on each household's own household_id (seed, salt) that orders households tied on wealth, so no copy or clone ordinal is taken first on every tie; the incidence clone stage (cgt_incidence_clone) gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact)." }, { "stage": "cgt_incidence_clone", @@ -7442,7 +8127,7 @@ "mass_split": 0.5, "weight_kind_out": "importance", "conservation": "exact_total", - "id_remapping": "id_multiplier_for_values", + "id_remapping": "copy-id table field cgt_incidence_clone; the twin takes every entity id + the field's radix (10**17)", "declared_factor": 1.0, "reason": "Capital-gains incidence clone splits every household's mass equally across original and clone records; total household mass is conserved." }, @@ -7610,7 +8295,7 @@ { "kind": "rank_preserving_allocation", "within": "income band x age group x region group cell, then the income band's pooled unassigned gainers", - "ordering": "a rank key descending, person_id ascending on ties: among gainers, the household investable-wealth rank times the wealth rank weight plus the prior-gain rank times its complement, both as average ranks over the gainers", + "ordering": "a rank key descending, then the rank-tie uniform ascending, then person_id ascending: among gainers, the household investable-wealth rank times the wealth rank weight plus the prior-gain rank times its complement, both as average ranks over the gainers; the rank-tie uniform is one identity-keyed uniform per person, keyed on person_id under seed_base and salted hmrc_cgt_gains_spine:rank_tie:, which every walk, plan and the remainder read", "wealth_rank_weight": 0.75, "investable_wealth_columns": [ "gross_financial_wealth", @@ -7637,7 +8322,7 @@ "mean_repair_reason": "Published counts round to the nearest thousand and amounts to the nearest million; two cells of the 2024-25 table (gains of GBP 1m to 2m at taxable incomes of GBP 37,700 to 49,999 and GBP 100,000 to 125,139) imply a mean below their own band, and repaired means clamp just inside the violated boundary.", "bottom_band_floor": "annual exempt amount plus one pound", "seed_base": 552, - "seed_mixing": "seed combined with the build period; each walk draws its rounding offset in cell order then pooled-fallback order, then each plan draws its quantile jitter, income bands ascending and gain bands highest first", + "seed_mixing": "seed combined with the build period, which salts every draw; each walk's rounding offset and each plan's quantile jitter is an identity-keyed uniform on its own label, with no stream order, so none depends on which other walks run or skip: a cell walk keys on cell::: and an income band's pooled walk on pool:, salted hmrc_cgt_gains_spine:offset:, and a plan on plan::, salted hmrc_cgt_gains_spine:jitter:, bands and age groups by their lower bounds and region groups by their labels", "deterministic": true, "cell_means": "ratios of the published Table 3 2024-25 cell gains to cell taxpayers, both rounded by the publisher (counts to the nearest thousand, amounts to the nearest million), so derived rather than published numbers; suppressed-count cells take the band-total mean; every mean is repaired into its band" }, @@ -7649,8 +8334,8 @@ "resource_vintage": "2017-18 nominal pounds as published (Advani and Summers 2020, CAGE WP 465, Table A1); no uprating is applied", "banding": "Advani-Summers total-income bands at the person's component-sum income without allowance subtraction, the banding the prior draw uses", "range": "quantiles strictly between the band's degree-1 quantile spline zero crossing and its crossing of the build period's annual exempt amount, so every amount lies in (0, AEA]", - "mapping": "remainder gainers take the midpoints of the weight-proportional quantile strata that partition the range, evaluated on the band's spline; zero-weight persons take a nominal stratum of the band's smallest positive weight and carry no mass; no seeded draw is consumed, so the walk and plan draw order is unchanged", - "ordering": "existing gains ascending, person_id descending on ties (the prior-gain ranking reversed; the allocation and the within-band draws rank on the wealth-blended key)", + "mapping": "remainder gainers take the midpoints of the weight-proportional quantile strata that partition the range, evaluated on the band's spline; zero-weight persons take a nominal stratum of the band's smallest positive weight and carry no mass; the mapping makes no draw of its own", + "ordering": "existing gains ascending, then the rank-tie uniform descending, then person_id descending (the prior-gain ranking reversed; the allocation and the within-band draws rank on the wealth-blended key)", "quantile_points": [ 0.05, 0.1, @@ -7720,8 +8405,8 @@ { "kind": "pair_clone_households_to_originals", "clone_flag_column": "household_is_capital_gains_clone", - "id_remapping": "id_multiplier_for_values", - "pairing": "clone household_id minus the clone stage's id multiplier, recomputed from the pre-clone rows (clone flag false; the support split's copies are pre-clone rows with their own clones)", + "id_remapping": "copy-id table field cgt_incidence_clone; the twin takes every entity id + the field's radix (10**17)", + "pairing": "clone household_id minus the cgt_incidence_clone field's declared radix (the support split's copies are pre-clone rows with their own clones)", "requirement": "bijection between clone and original households; an unpaired clone or original fails the build" }, { @@ -7832,7 +8517,7 @@ "realization": "probability carried as weight, no draw: a household carrying k liable gainers becomes 2^k arms, one per subset of residential gainers, at the household's weight times the product of p over the residential gainers and 1 - p over the others; the arm with no residential gainer keeps the household's ids; at design weights the residential count and gains equal the solved expectations sum(p w) and sum(p w g) in every gain band", "maximum_liable_gainers_per_household": 3, "gainer_order": "person_id ascending within the household", - "id_remapping": "id_multiplier_for_values; arm j takes every entity id + j x multiplier, bit m of j marking the household's m-th liable gainer residential", + "id_remapping": "copy-id table field cgt_residential_split, the top field; arm j takes every entity id + j x the field's radix (10**18), bit m of j marking the household's m-th liable gainer residential", "flag_column": "household_is_cgt_residential_clone", "index_column": "cgt_residential_clone_index", "probability_column": "cgt_residential_probability", @@ -7856,7 +8541,7 @@ "cgt_residential_probability", "capital_gains_residential_property" ], - "notes": "Structural stage after the incidence anchor and before the asset-type stage (microcosm#1063). The residential-property flag used to be realised by a weighted systematic walk, one whole flag per row, whose gains error is bounded by one row's stake: at the top of the gains distribution one stake is several times a band's expected residential gains, so the realised gains sat short of expectation by the lottery expectation of the giant stakes or over it by one of them. This stage solves the same logistic (intercept to the Table 8a 2024-25 taxpayers on the individuals basis, slope to the gains, with the stock shift) and carries the probability as weight instead: every household with liable gainers becomes 2^k arms at product weights, one per subset of residential gainers, the arm with no residential gainer keeping the household's ids and arm j taking every entity id plus j times the id multiplier, the lineage rule the executor, the geography identity kernel and the identity tool read for the support split and the clone. At design weights the residential count and gains equal the solved expectations in every gain band, with no draw, no offset and no seed; the calibration, which binds hmrc.cgt.residential_property_taxpayers and hmrc.cgt.residential_property_gains, decides between the arms afterwards as it does between the incidence clone's halves. Every household's mass and composition and the total household mass are conserved; the receipt is gated at stage time." + "notes": "Structural stage after the incidence anchor and before the asset-type stage (microcosm#1063). The residential-property flag used to be realised by a weighted systematic walk, one whole flag per row, whose gains error is bounded by one row's stake: at the top of the gains distribution one stake is several times a band's expected residential gains, so the realised gains sat short of expectation by the lottery expectation of the giant stakes or over it by one of them. This stage solves the same logistic (intercept to the Table 8a 2024-25 taxpayers on the individuals basis, slope to the gains, with the stock shift) and carries the probability as weight instead: every household with liable gainers becomes 2^k arms at product weights, one per subset of residential gainers, the arm with no residential gainer keeping the household's ids and arm j taking every entity id plus j times the radix of the copy-id table's top field (microcosm#931), the lineage rule the executor and the identity tool read for the support split and the clone. At design weights the residential count and gains equal the solved expectations in every gain band, with no draw, no offset and no seed; the calibration, which binds hmrc.cgt.residential_property_taxpayers and hmrc.cgt.residential_property_gains, decides between the arms afterwards as it does between the incidence clone's halves. Every household's mass and composition and the total household mass are conserved; the receipt is gated at stage time." }, { "stage": "hmrc_cgt_asset_type_spine", @@ -7920,7 +8605,7 @@ "qualifying_amount": "below the lifetime limit a claimant's whole net gain qualifies; in the open top band the qualifying gain is the lifetime limit", "top_band_count": "the published top-band qualifying gains divided by the lifetime limit; the published count is rounded to the nearest thousand", "basis_assumption": "Table 4.1 counts qualifying gains before losses and the annual exempt amount while the frame carries net gains after losses; Investors' Relief is merged with BADR, as in the engine's capital_gains_badr input; the whole-gain assumption reproduces Table 4.1's tax column to within half a percent", - "realization": "weighted systematic sampling within each band in ascending (gain, person_id) order with one seeded offset per band, drawn in band order whether or not the band carries a target", + "realization": "weighted systematic sampling within each band in ascending gain order, persons tied on gain in ascending order of a uniform keyed on person_id under seed and salted assign_badr_qualifying_gains:tie:, then person_id ascending, with one identity-keyed offset per band, keyed on its bounds (|, open for the top band) and salted assign_badr_qualifying_gains:offset:, with no stream order, so a band's offset depends neither on which bands carry a target nor on which other bands the facts declare", "seed": 555, "seed_mixing": "seed combined with the build period", "policy_parameters": [ @@ -7963,7 +8648,7 @@ "weight_fitting": "one fit per claimant status: multiplicative updates of the type weights until the household-weighted expected gains shares by type match that status's share targets; a fit refuses if it has not converged within the iteration limit, and the stage refuses up front when claimants hold more of the gains than the claimant categories' share", "weight_fitting_iterations": 200, "weight_fitting_tolerance": 1e-06, - "realization": "error diffusion per claimant status in descending (gain, person_id) order: each person adds weight x gain x type probability to a running balance per type and takes the type it may hold with the largest balance, which is drawn down by the whole weight x gain, so the realised gains by type track the fitted composition instead of following the heaviest rows; deterministic, no seed", + "realization": "error diffusion per claimant status in descending gain order, persons tied on gain in ascending order of a uniform keyed on person_id under the assign_badr_qualifying_gains seed and salted assign_main_asset_type:tie:, then person_id ascending: each person adds weight x gain x type probability to a running balance per type and takes the type it may hold with the largest balance, which is drawn down by the whole weight x gain, so the realised gains by type track the fitted composition instead of following the heaviest rows; that tie-break is its only draw", "output_column": "capital_gains_asset_type", "value_domain": [ "none", @@ -8096,7 +8781,7 @@ "candidates": "a bonus (frs_bonus_amount, or an SPI-channel adult's drawn bonus) > 0 and employment_income at least the anchor's annual upper earnings limit, after the salary conversion", "move": "the bonus, at most move_pay_share_cap of employment_income, to pension_contributions_via_bonus_sacrifice; employment_income unchanged, since a sacrificed bonus is never received", "move_pay_share_cap": 0.5, - "spi_bonus_draw": "an SPI-channel adult with pay draws the bonus share of pay (frs_bonus_amount over employment_income) of a base-FRS adult with pay in the same pay cell, by person weight, keyed on person_id; the cells are the base-FRS paid adults' weighted pay quantiles at spi_cell_quantiles, and a cell with fewer than spi_min_donor_sources donor source households joins the one below", + "spi_bonus_draw": "an SPI-channel adult with pay draws the bonus share of pay (frs_bonus_amount over employment_income) of a base-FRS adult with pay in the same pay cell and the same geographic clone (household_clone_index; a spine without it is one clone), by person weight, keyed on person_id; the cells are the base-FRS paid adults' weighted pay quantiles over every clone at spi_cell_quantiles, and a cell with fewer than spi_min_donor_sources donor source households joins the one below, and the lowest the one above", "spi_cell_quantiles": [ 0.1, 0.2, @@ -8138,7 +8823,7 @@ "employee_pension_contributions", "employment_income" ], - "notes": "The QRF trains only on the 2024-25 FRS asked subset with pay. Its sign gate is cross-fitted (gate_calibration: cross_fitted; microcosm#1359), as lcfs_consumption's are: it is calibrated on the donors its fold gates left out, so a sign is drawn at its donor rate on people the gate was not fitted to, with each household's members in one fold. Salary sacrifice is pay given up, so every amount, reported or predicted, is limited to the pay the person still receives, and a person without pay has none (uk-data#518, microcosm#1095); the limit bites above all on SPI-channel rows, which keep their FRS twin's answer beside pay redrawn from the SPI. The second arm creates support toward the reviewed 5.4m staging target by moving contributors' pension amounts, up to half their pay, into salary sacrifice and lowering their pay by the same amount, since the engine reads employment_income after sacrifice; what does not move stays an employee contribution, so the sacrifice never exceeds the pay that remains. Before microcosm#1069 c9 the pay stayed whole and the converted record lost its net-pay relief. The third arm adds bonus sacrifice beside pay toward the OBR's 2027-28 base above GBP 2,000 (microcosm#1340). A sacrificed bonus is never received, so the FRS records it in neither pay nor its bonus amounts, and pay stays as it is (María's ruling of 2026-10-24 on Vahid's review of microcosm#1383); a received FRS bonus marks a bonus-paying job and its size. An SPI-channel adult's FRS bonus is its twin's, beside the tape's pay, so it draws a bonus share of pay from a base-FRS adult in its pay cell instead." + "notes": "The QRF trains only on the 2024-25 FRS asked subset with pay. Its sign gate is cross-fitted (gate_calibration: cross_fitted; microcosm#1359), as lcfs_consumption's are: it is calibrated on the donors its fold gates left out, so a sign is drawn at its donor rate on people the gate was not fitted to, with each household's members in one fold. Salary sacrifice is pay given up, so every amount, reported or predicted, is limited to the pay the person still receives, and a person without pay has none (uk-data#518, microcosm#1095); the limit bites above all on SPI-channel rows, which keep their FRS twin's answer beside pay redrawn from the SPI. The second arm creates support toward the reviewed 5.4m staging target by moving contributors' pension amounts, up to half their pay, into salary sacrifice and lowering their pay by the same amount, since the engine reads employment_income after sacrifice; what does not move stays an employee contribution, so the sacrifice never exceeds the pay that remains. Before microcosm#1069 c9 the pay stayed whole and the converted record lost its net-pay relief. The third arm adds bonus sacrifice beside pay toward the OBR's 2027-28 base above GBP 2,000 (microcosm#1340). A sacrificed bonus is never received, so the FRS records it in neither pay nor its bonus amounts, and pay stays as it is (María's ruling of 2026-10-24 on Vahid's review of microcosm#1383); a received FRS bonus marks a bonus-paying job and its size. An SPI-channel adult's FRS bonus is its twin's, beside the tape's pay, so it draws a bonus share of pay from a base-FRS adult in its pay cell, in its own geographic clone, instead." }, { "stage": "carers_allowance_take_up", @@ -8363,12 +9048,12 @@ "maximum_extended_childcare_hours_usage", "max_free_entitlement_hours_used" ], - "notes": "Runs after the last copy stage and before private_school_attendance, whose net-income ranking counts the childcare entitlements and Tax-Free Childcare. frs_take_up draws the childcare flags and the extended-hours usage on FRS benefit units before any copy exists, so every SPI support copy, income-band donor, high-value home support copy, CGT support copy, CGT incidence clone and residential arm inherited its host's draws, and a cell with few hosts fixed the take-up of all their copies. This stage re-draws would_claim_tfc, would_claim_extended_childcare, would_claim_universal_childcare (nested under the extended flag) and would_claim_targeted_childcare at the contract rates, and maximum_extended_childcare_hours_usage from the contract's clipped normal, for the copies only, with frs_take_up's own helper, seed 0 and output-name salts, keyed on the copy's own benunit_id. A copy's structural id is its source id plus a multiple of an offset above every id in the frame, so it is unique, differs from every native id and gives the copy streams independent of its host's; a rebuild of the same frame draws the same values. Native benefit units keep their frs_take_up draws. The anchored Child Benefit, Pension Credit and Universal Credit flags and would_claim_uc_childcare are left as they are. It also re-draws the copies' children's max_free_entitlement_hours_used, the under-2 usage, against their own unit's re-drawn working-parent flag, keyed on their own person_id." + "notes": "Runs after the last copy stage and before private_school_attendance, whose net-income ranking counts the childcare entitlements and Tax-Free Childcare. frs_take_up draws the childcare flags and the extended-hours usage on FRS benefit units, each geographic clone's on its own ids (microcosm#931), before any other copy exists, so every SPI support copy, income-band donor, high-value home support copy, CGT support copy, CGT incidence clone and residential arm inherited its host's draws, and a cell with few hosts fixed the take-up of all their copies. This stage re-draws would_claim_tfc, would_claim_extended_childcare, would_claim_universal_childcare (nested under the extended flag) and would_claim_targeted_childcare at the contract rates, and maximum_extended_childcare_hours_usage from the contract's clipped normal, for the copies only, with frs_take_up's own helper, seed 0 and output-name salts, keyed on the copy's own benunit_id. A copy's structural id is its source id plus a multiple of an offset above every id in the frame, so it is unique, differs from every native id and gives the copy streams independent of its host's; a rebuild of the same frame draws the same values. Native benefit units, every geographic clone's included, keep their frs_take_up draws. The anchored Child Benefit, Pension Credit and Universal Credit flags and would_claim_uc_childcare are left as they are. It also re-draws the copies' children's max_free_entitlement_hours_used, the under-2 usage, against their own unit's re-drawn working-parent flag, keyed on their own person_id." }, { "stage": "private_school_attendance", "survey": "policyengine-uk private-school attendance imputation, stored at the spine's weights", - "source": "policyengine-uk attends_private_school (the gov.simulation.private_school_vat attendance rates and factor), evaluated by the pinned engine", + "source": "policyengine-uk attends_private_school (the gov.simulation.private_school_vat attendance rates and factor) over every person in the frame, from the pinned engine's values and parameters", "grain": "person", "artifacts": [ { @@ -8398,7 +9083,7 @@ "outputs": [ "attends_private_school" ], - "notes": "The last spine stage before the ISA subscriptions (uk-data#568, microcosm#1095). policyengine-uk imputes attends_private_school with a formula that ranks households by net income at their weights and compares a percentile rate, scaled to the national pupil count, with the stored attends_private_school_random_draw. Left to the formula, attendance is re-imputed in every simulation, so a reform to income tax, National Insurance or benefits switches pupils in or out, a population under one million people of weight imputes none (pe-uk#1905), and the calibration and the release read different flags once the weights change. This stage materializes the engine's attendance once, at the calibration year and the spine's design weights, and stores it, as the root stage stores months_since_last_birthday, so the solve and the release read the same flags; no stage after it copies or re-keys a person. Stored attendance no longer responds to income changes or to reforms of the attendance parameters, and a reform to the fee VAT does not move pupils either way in the pinned engine." + "notes": "The last spine stage before the ISA subscriptions (uk-data#568, microcosm#1095). policyengine-uk imputes attends_private_school with a formula that ranks households by net income at their weights and compares a percentile rate, scaled to the national pupil count, with the stored attends_private_school_random_draw. Left to the formula, attendance is re-imputed in every simulation, so a reform to income tax, National Insurance or benefits switches pupils in or out, a population under one million people of weight imputes none (pe-uk#1905), and the calibration and the release read different flags once the weights change. This stage computes the formula's attendance once, over the whole frame at the calibration year and the spine's design weights, and stores it, as the root stage stores months_since_last_birthday, so the solve and the release read the same flags; no stage after it copies or re-keys a person. The formula ranks the people a simulation holds, so a simulation over part of the frame would rank that part alone (microcosm#931). The stage therefore also restates the formula's last step over every person in the frame, from the engine's own values at the calibration year, each decided by a household's own records, and its own parameters: household market income plus benefits less the members' income tax and National Insurance, each household's weight once per member, microdf's weighted percentile rank (the rank the formula calls) over the people of positive weight, the attendance rates and population factor, nobody while those weights sum below one million, and the stored draw for each under-18. Where the engine computes the frame in one simulation (the national line, and a spine of one geographic clone), the stage stores the engine's flags once the restatement equals them bit for bit, so every such build re-checks it; where the households carry more than one clone, the engine computes the frame clone by clone and the stage stores the restatement over the whole pool. The receipt names the route and whether the engine's flags were checked. Stored attendance no longer responds to income changes or to reforms of the attendance parameters, and a reform to the fee VAT does not move pupils either way in the pinned engine." }, { "artifacts": [ @@ -8444,7 +9129,7 @@ "stocks_and_shares_isa_subscriptions", "lifetime_isa_payments" ], - "notes": "Imputes ISA subscriptions per adult by type, and the per-person ISA balances, on the final spine (microcosm#1415). Each household's was_wealth cash and stocks-and-shares ISA totals are split across its adults by the holding pattern of a WAS round-8 donor household with the same number of adults, so person balances sum exactly to the household totals, which are unchanged. Subscribers in 2023-24 are adults aged 18 or over in households holding the type, raked by class (cash only, stocks and shares only, both) to Table 9.7's income bands, Table 9.8's age and sex and Table 9.9's regions and selected by keyed walks, the both class first. Amounts follow Table 9.7's subscription bands within each income band and class, uniform inside a band and exactly GBP 20,000 at the top; the both class splits each person's amount by their balance shares, shifted so the class's stocks-and-shares total is Table 9.4's 2023-24 stocks-and-shares amount less the stocks-and-shares-only payments. People grow to 2024-25 by Table 9.4's account growth (an upper bound on people, as from 6 April 2024 a person may pay into several ISAs of one type), and amounts scale to Table 9.4's 2024-25 amounts per type, holding subscribers already at the GBP 20,000 overall limit and moving combined amounts between GBP 18,000 (Table 9.7's last band floor below the limit) and the limit on a line up to it, so a factor below one leaves no empty range under the limit (María's rulings of 29 October 2026): Table 9.7's cash bands sit above Table 9.4, and growing that excess by Table 9.4's ratio against the limit pushed the subscribers below it far up. Every Lifetime ISA holder aged 18 to 47 (the ages that can hold one in 2024-25) with room under the overall limit pays in, amounts in (0, GBP 4,000] drawn to Table 9.4's 2024-25 Lifetime ISA amount. No ISA fact is a calibration target: binding the cells the stage draws to would be circular, so the reviewed fence keeps them measured only.", + "notes": "Imputes ISA subscriptions per adult by type, and the per-person ISA balances, on the final spine (microcosm#1415). Each household's was_wealth cash and stocks-and-shares ISA totals are split across its adults by the holding pattern of a WAS round-8 donor household with the same number of adults, so person balances sum exactly to the household totals, which are unchanged. Subscribers in 2023-24 are adults aged 18 or over in households holding the type, raked by class (cash only, stocks and shares only, both) to Table 9.7's income bands, Table 9.8's age and sex and Table 9.9's regions and selected by keyed walks, the both class first. Amounts follow Table 9.7's subscription bands within each income band and class, uniform inside a band and exactly GBP 20,000 at the top; the both class splits each person's amount by their balance shares, shifted so the class's stocks-and-shares total is Table 9.4's 2023-24 stocks-and-shares amount less the stocks-and-shares-only payments. Amounts go in the order of a key that blends each subscriber's weighted rank on the class's balance (cash, stocks and shares, or the two together) with a keyed uniform at balance_rank_weight; tied balances share one weighted mid-rank, so the keyed uniform alone orders them. People grow to 2024-25 by Table 9.4's account growth (an upper bound on people, as from 6 April 2024 a person may pay into several ISAs of one type), and amounts scale to Table 9.4's 2024-25 amounts per type, holding subscribers already at the GBP 20,000 overall limit and moving combined amounts between GBP 18,000 (Table 9.7's last band floor below the limit) and the limit on a line up to it, so a factor below one leaves no empty range under the limit (María's rulings of 29 October 2026): Table 9.7's cash bands sit above Table 9.4, and growing that excess by Table 9.4's ratio against the limit pushed the subscribers below it far up. Every Lifetime ISA holder aged 18 to 47 (the ages that can hold one in 2024-25) with room under the overall limit pays in, amounts in (0, GBP 4,000] drawn to Table 9.4's 2024-25 Lifetime ISA amount and ordered by the same blend, on the Lifetime ISA balance over every holder aged 18 to 47, with tied balances again sharing one mid-rank; the Lifetime ISA operation declares no rank weight and reuses the amounts operation's balance_rank_weight. No ISA fact is a calibration target: binding the cells the stage draws to would be circular, so the reviewed fence keeps them measured only.", "operations": [ { "artifact_role": "isa_statistics_facts", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index 9e36d8ecba..9beb49b264 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -565,11 +565,13 @@ stages: - liable_for_share_of_household_rent - household_id - region + - imd_decile - tenure_type - accommodation_type - num_bedrooms - council_tax_reported - council_tax_band + - council_tax_band_observed - council_tax_rebate - council_tax_single_adult_raw - water_and_sewerage_charges @@ -622,7 +624,7 @@ stages: - frs_company_car_price_band - frs_company_car_fuel_code - frs_company_car_contribution - notes: Root E2 spine assembly. It carries direct raw mappings only; education-grant aggregate and council-tax reported fields remain raw carriers for E3. Benefit take-up, BRMA/LHA assignment, stochastic flags, and imputations are intentionally absent from this stage, except two identity-keyed draws. One is the UC start-up claim recency (uk-data#527). The other is each person's months_since_last_birthday, which policyengine-uk reads to place a date of birth (the two-child limit, the State Pension scheme, the Pension Credit qualifying age). Each single year of age and sex is spread evenly over the year at the FRS weights, in an order set by a draw on the person's stable id, as the engine spreads it when the column is unset, and the column is stored so every later engine read, the calibration's and the release's included, takes the same months whatever weights or identifiers the file carries (microcosm#1095). The UC start-up period needs the raw claim, interview and job dates only this stage reads, so a UC record without a linked claim date takes its claim recency from a benefit-unit-keyed draw at the linked share. The 2024-25 househol.tab is not sernum-ordered; the declared household sort guards stable identity. care_hours carries the FRS banded care-hours code as weekly hours (lower band edge), and frs_care_hours_in_household the largest of the four in-household care bands (HOURAB, HOURAH, HOURCB, HOURCH) the same way, a stage-internal carrier the Carer's Allowance draw reads and the release drops (microcosm#1418); would_claim_carers_allowance follows reported receipt, so the engine keeps Carer's Allowance on reported claimants while the hours qualify carers for the Universal Credit carer element (#882); after the SPI income chain, carers_allowance_take_up opens the flag to carers the earnings limit alone keeps from the allowance (microcosm#1319). SEINCAM2, the self-employment income the respondent gives, splits by its sign (uk-data#528), so profits go to self_employment_income and losses to trading_loss as a positive amount, the two inputs policyengine-uk reads from 2.131.0 (pe-uk#2085). The engine uprates trading_loss into each projected year as it does self_employment_income, so a loss persists like a profit; the engine documents that a one-off loss would be set to zero in later years, which a one-year survey cannot tell apart. Six of policyengine-uk's eleven owned-company inputs (pe-uk#1965) come from the FRS job tab (microcosm#1095). A director (DIRECTOR 1 or NATURE 1) who owns at least half of the company (DIRPERC 1 to 3, the ruling of 2026-10-16) stands as its sole owner or partner and works in it. The company is their main employment when the job is their first (JOBTYPE 1); it carries on a property business when the main job's SIC division is 68 and a trade otherwise, and its income share is the dividends the job records (DIVIDEND, after tax), the part of the company's profit the survey sees. A company held in another job carries on neither, since ADULT.SIC codes the main job only, which leaves the engine's company-owner rules off for it. The stocks and the intermediaries inputs keep the engine's defaults. frs_company_car_price_band, frs_company_car_fuel_code, frs_company_car_private_fuel and frs_company_car_contribution carry each person's company car from the FRS job tab (CARVAL, FUELTYP, EXPBEN03, and CARAMT where CARCON is 1, on their EXPBEN01 job with the lowest JOBTYPE), stage-internal carriers the company_cars stage reads and the release drops (PolicyEngine/uk-autumn-budget-dashboard-2026#316). frs_uc_lcw_status carries DWP's limited capability for work status on a linked Universal Credit claim (BENEFITS.UCLCW, coded 0 None, 1 LCFW Group and 2 LCFWRA Group in SN 9563's UKDA data dictionary; blank, as UCSTART is, where the claim has no linked record) on the UC row's person, NOT_LINKED for everyone else; the FRS keeps one UC row per claim, couples included, so the status is the claim's. It is a stage-internal carrier the uc_limited_capability stage reads and the release drops (microcosm#1095, Y1). frs_uc_carer_element carries DWP's carer element on a linked Universal Credit claim (BENEFITS.UCCAREEL, which SN 9563's UKDA data dictionary labels the weekly carer element amount and gives no value labels; the 2024-25 file codes it 0 or 1 on every linked claim, read as NONE or CARER on a claim with a linked record, UCSTART present, and NOT_LINKED on any other) on the UC row's person, NOT_LINKED for everyone else; frs_disability reads it only to check the carer link against DWP's record, and the release drops it (A3 (f) of the November event-data plan's PR3, microcosm#1095). frs_care_recipients carries each person's main care recipients from the FRS care tab (CARE.NEEDPER, WHOLOO01 to WHOLOO14 and HOUR01 to HOUR14, coded as HOURTOT is) as a bitmask, with bit n for household member n among the people given the person's largest weekly band and bit 15 for anyone outside the household among them; frs_disability reads it to write cares_for_severely_disabled_person, and the release drops it (A3 of the November event-data plan's PR3, microcosm#1095). + notes: Root E2 spine assembly. It carries direct raw mappings only; education-grant aggregate and council-tax reported fields remain raw carriers for E3. Benefit take-up, BRMA/LHA assignment, stochastic flags, and imputations are intentionally absent from this stage, except two identity-keyed draws. One is the UC start-up claim recency (uk-data#527). The other is each person's months_since_last_birthday, which policyengine-uk reads to place a date of birth (the two-child limit, the State Pension scheme, the Pension Credit qualifying age). Each single year of age and sex is spread evenly over the year at the FRS weights, in an order set by a draw on the person's stable id, as the engine spreads it when the column is unset, and the column is stored so every later engine read, the calibration's and the release's included, takes the same months whatever weights or identifiers the file carries (microcosm#1095). The UC start-up period needs the raw claim, interview and job dates only this stage reads, so a UC record without a linked claim date takes its claim recency from a benefit-unit-keyed draw at the linked share. The 2024-25 househol.tab is not sernum-ordered; the declared household sort guards stable identity. care_hours carries the FRS banded care-hours code as weekly hours (lower band edge), and frs_care_hours_in_household the largest of the four in-household care bands (HOURAB, HOURAH, HOURCB, HOURCH) the same way, a stage-internal carrier the Carer's Allowance draw reads and the release drops (microcosm#1418); would_claim_carers_allowance follows reported receipt, so the engine keeps Carer's Allowance on reported claimants while the hours qualify carers for the Universal Credit carer element (#882); after the SPI income chain, carers_allowance_take_up opens the flag to carers the earnings limit alone keeps from the allowance (microcosm#1319). SEINCAM2, the self-employment income the respondent gives, splits by its sign (uk-data#528), so profits go to self_employment_income and losses to trading_loss as a positive amount, the two inputs policyengine-uk reads from 2.131.0 (pe-uk#2085). The engine uprates trading_loss into each projected year as it does self_employment_income, so a loss persists like a profit; the engine documents that a one-off loss would be set to zero in later years, which a one-year survey cannot tell apart. Six of policyengine-uk's eleven owned-company inputs (pe-uk#1965) come from the FRS job tab (microcosm#1095). A director (DIRECTOR 1 or NATURE 1) who owns at least half of the company (DIRPERC 1 to 3, the ruling of 2026-10-16) stands as its sole owner or partner and works in it. The company is their main employment when the job is their first (JOBTYPE 1); it carries on a property business when the main job's SIC division is 68 and a trade otherwise, and its income share is the dividends the job records (DIVIDEND, after tax), the part of the company's profit the survey sees. A company held in another job carries on neither, since ADULT.SIC codes the main job only, which leaves the engine's company-owner rules off for it. The stocks and the intermediaries inputs keep the engine's defaults. frs_company_car_price_band, frs_company_car_fuel_code, frs_company_car_private_fuel and frs_company_car_contribution carry each person's company car from the FRS job tab (CARVAL, FUELTYP, EXPBEN03, and CARAMT where CARCON is 1, on their EXPBEN01 job with the lowest JOBTYPE), stage-internal carriers the company_cars stage reads and the release drops (PolicyEngine/uk-autumn-budget-dashboard-2026#316). frs_uc_lcw_status carries DWP's limited capability for work status on a linked Universal Credit claim (BENEFITS.UCLCW, coded 0 None, 1 LCFW Group and 2 LCFWRA Group in SN 9563's UKDA data dictionary; blank, as UCSTART is, where the claim has no linked record) on the UC row's person, NOT_LINKED for everyone else; the FRS keeps one UC row per claim, couples included, so the status is the claim's. It is a stage-internal carrier the uc_limited_capability stage reads and the release drops (microcosm#1095, Y1). frs_uc_carer_element carries DWP's carer element on a linked Universal Credit claim (BENEFITS.UCCAREEL, which SN 9563's UKDA data dictionary labels the weekly carer element amount and gives no value labels; the 2024-25 file codes it 0 or 1 on every linked claim, read as NONE or CARER on a claim with a linked record, UCSTART present, and NOT_LINKED on any other) on the UC row's person, NOT_LINKED for everyone else; frs_disability reads it only to check the carer link against DWP's record, and the release drops it (A3 (f) of the November event-data plan's PR3, microcosm#1095). frs_care_recipients carries each person's main care recipients from the FRS care tab (CARE.NEEDPER, WHOLOO01 to WHOLOO14 and HOUR01 to HOUR14, coded as HOURTOT is) as a bitmask, with bit n for household member n among the people given the person's largest weekly band and bit 15 for anyone outside the household among them; frs_disability reads it to write cares_for_severely_disabled_person, and the release drops it (A3 of the November event-data plan's PR3, microcosm#1095). imd_decile, each household's decile on its own nation's index of multiple deprivation (IMDE, IMDW, IMDS or IMDN), 1 to 10, and 0 where none is observed or the region is unknown, and council_tax_band_observed, False in Northern Ireland, for a CTBAND code outside 1 to 9, for a band the household's region does not use and for an unknown region (council_tax_band itself is unchanged), are stage-internal carriers the clone-first spine's area assignment conditions on and the release drops; they stay out of every receipt and of the spine sidecar's nonzero shares, and a household tab without one of the four IMD variables is refused (microcosm#931). - stage: age_tail survey: Family Resources Survey 2024-25 and ONS mid-year population estimates source: Office for National Statistics mid-year population estimates for the 80+ age bands by sex, via the chronicle ledger facts the calibration register binds (target family ons_population). @@ -890,11 +892,15 @@ stages: stored: the Scottish households' council_tax after impute_cell_means, at the spine design weights factor: level / stored, one factor on every Scottish household's council_tax; a frame with no Scottish bill to level keeps its bills as derived weights: spine_design_weights + - kind: derive + output: council_tax_bill_provenance + formula: 'first match: not_levied where raw gvtregno == 13 (Northern Ireland, which pays domestic rates), whatever branch its bill takes; else the branch derive_council_tax takes: missing where the bill is missing (raw ctannual < 0 or NaN, or ctannual == 0 with ctpaid == 1 and ctreb != 1); observed_gross where ctreb == 1 and ctrebamt > 0 (the reported reduction added back); unknown_reduction where ctreb == 1 (the larger of the netted ctannual and the cell mean); else observed_gross (the netted ctannual, kept at zero where ctannual == 0 and ctpaid != 1, as with ctpaid 2 and 3)' outputs: - council_tax + - council_tax_bill_provenance nonnegative_outputs: - council_tax - notes: 'Re-reads raw househol.tab because spine council_tax_reported clips missing values. FRS 2024-25 CTANNUAL is the bill after discounts and council tax reduction, and the engine reads council_tax before the reduction, so a household reporting a reduction gets its CTREBAMT added back and the cells pool non-recipients only (uk-data#496/#499, microcosm#1095). Scottish water and sewerage charges are netted before cell means at the shares DWP''s CTANNUAL carries (the gross charges in full for non-recipients, 65% for reduction recipients); no-donor cells impute zero. A household where someone pays council tax (CTPAID 1) with no reported reduction and a zero CTANNUAL owes a bill, so it takes its cell mean as a missing bill does (microcosm#1278). The dead ct_mean.replace(-1, ...) branch is intentionally dropped. Last, the Scottish bills are levelled with one factor to the Scottish Government''s own measure of the 2024-25 liability before council tax reduction (S2(a) of the November event-data plan, microcosm#1095; María''s signature of 2026-11-09): the sum over the 32 council areas of CTAXBASE 2025''s Band D equivalent dwellings before council tax reduction at September 2024, the middle of 2024-25, times CTAS 2026''s average Band D council tax for 2024-25, over the Scottish households'' bills at the spine design weights. Band D equivalents count dwellings after discounts and exemptions, as the bills do. The bound Scottish band stock (scotgov.council_tax_stock) already gives the household mix, so the level moves the bills'' amounts and no weight; obr.council_tax_scotland stays a held-out diagnostic. A frame with no Scottish bill to level keeps its bills as derived, and the stage receipt says so. The liability counts every chargeable dwelling, including about 21,600 second homes and 43,500 empty homes liable for council tax (empty more than 6 months) in September 2024, which no FRS household occupies (Scottish Government, Second Homes and Empty Properties in September 2025, from the councils'' CTAXBASE returns). So the level is the liability times Scotland''s main-residence share at September 2024: one less those dwellings over CTAXBASE 2024''s 2.60 million chargeable dwellings, about 97.5% (B11 of the November event-data plan''s PR3; Vahid''s second review of microcosm#1550; María''s signature of 2026-11-11). The share is taken in dwellings, since no published table splits Band D equivalents by class, and the stage receipt records the counts and the share. It nets out less than these dwellings'' whole liability: councils can charge second homes and long-term empty homes a premium of up to double the rate, and homes empty 6 months or less, also liable, are not counted.' + notes: 'Re-reads raw househol.tab because spine council_tax_reported clips missing values. FRS 2024-25 CTANNUAL is the bill after discounts and council tax reduction, and the engine reads council_tax before the reduction, so a household reporting a reduction gets its CTREBAMT added back and the cells pool non-recipients only (uk-data#496/#499, microcosm#1095). Scottish water and sewerage charges are netted before cell means at the shares DWP''s CTANNUAL carries (the gross charges in full for non-recipients, 65% for reduction recipients); no-donor cells impute zero. A household where someone pays council tax (CTPAID 1) with no reported reduction and a zero CTANNUAL owes a bill, so it takes its cell mean as a missing bill does (microcosm#1278). The dead ct_mean.replace(-1, ...) branch is intentionally dropped. It also derives council_tax_bill_provenance, the branch each bill takes (not_levied in Northern Ireland, whatever its branch); it is dropped at the release boundary, and council_tax_localise alone reads it. Last, the Scottish bills are levelled with one factor to the Scottish Government''s own measure of the 2024-25 liability before council tax reduction (S2(a) of the November event-data plan, microcosm#1095; María''s signature of 2026-11-09): the sum over the 32 council areas of CTAXBASE 2025''s Band D equivalent dwellings before council tax reduction at September 2024, the middle of 2024-25, times CTAS 2026''s average Band D council tax for 2024-25, over the Scottish households'' bills at the spine design weights. Band D equivalents count dwellings after discounts and exemptions, as the bills do. The bound Scottish band stock (scotgov.council_tax_stock) already gives the household mix, so the level moves the bills'' amounts and no weight; obr.council_tax_scotland stays a held-out diagnostic. A frame with no Scottish bill to level keeps its bills as derived, and the stage receipt says so. The liability counts every chargeable dwelling, including about 21,600 second homes and 43,500 empty homes liable for council tax (empty more than 6 months) in September 2024, which no FRS household occupies (Scottish Government, Second Homes and Empty Properties in September 2025, from the councils'' CTAXBASE returns). So the level is the liability times Scotland''s main-residence share at September 2024: one less those dwellings over CTAXBASE 2024''s 2.60 million chargeable dwellings, about 97.5% (B11 of the November event-data plan''s PR3; Vahid''s second review of microcosm#1550; María''s signature of 2026-11-11). The share is taken in dwellings, since no published table splits Band D equivalents by class, and the stage receipt records the counts and the share. It nets out less than these dwellings'' whole liability: councils can charge second homes and long-term empty homes a premium of up to double the rate, and homes empty 6 months or less, also liable, are not counted.' - stage: frs_disability survey: Family Resources Survey 2024-25 source: Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 tabs. plus policyengine-uk DWP parameters. @@ -1049,6 +1055,450 @@ stages: nonnegative_outputs: - disabled_students_allowance_eligible_expenses notes: Runs before BRMA and always runs; the DSA seed evaluates at the release calibration year (uk-data#478) and the modelling-start gate compares against that year, not the frame period. EDUCQUAL grant-split compatibility is re-cited to the FRS 2024-25 SN 9563 dictionary; codes 1-87 are empirically present and code 87 remains undocumented/defaulted. +- stage: frs_hmrc_spine_leaves + survey: Family Resources Survey 2024-25 + source: Department for Work and Pensions Family Resources Survey 2024-25 raw adult.tab and benefits.tab, caller-supplied local input + grain: person + artifacts: + - role: frs_table + table: adult + kind: licensed_microdata + format: tab + vintage: '2024_25' + locator: adult.tab + sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d + size_bytes: 34885825 + runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 + - role: frs_table + table: benefits + kind: licensed_microdata + format: tab + vintage: '2024_25' + locator: benefits.tab + sha256: f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3 + size_bytes: 2362329 + runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 + - role: frs_table + table: job + kind: licensed_microdata + format: tab + vintage: '2024_25' + locator: job.tab + sha256: eb7faf7ada3a3851cb2afb83e2983f8907ffeec897cfbe01e56cb0dfefa853e2 + size_bytes: 10518760 + runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 + - role: frs_table + table: penprov + kind: licensed_microdata + format: tab + vintage: '2024_25' + locator: penprov.tab + sha256: 9e53de0dc969baec000b3cd68387f0f2dfb3f678732e408de175e0a1d6e3fdc1 + size_bytes: 513614 + runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 + - role: employer_rate_bands + kind: public_aggregate_reference + resource: ashe_employer_pension_contribution_rates.json + format: json + runtime_sha256_required: true + operations: + - kind: retain_adjudicated_frs_hmrc_leaves + population: uk_frs_raw_spine + source_vintage: 2024-25 + mapped_build_period: 2024 + annualization: weekly raw FRS amounts * 52 + status: adjudicated_partial_replay + retained_full_constituents: + hmrc_spi_pay: + spi_concept: PAY + scope: full + raw_sources: + - ADULT.INEARNS + formula: max(0, ADULT.INEARNS) * 52 + hmrc_spi_unemployment_benefit_income: + spi_concept: UBISJA + scope: full + raw_sources: + - BENEFITS.BENEFIT=14:BENAMT + - BENEFITS.BENEFIT=19:BENAMT + formula: sum(BENAMT where BENEFIT in {14, 19}) * 52 + hmrc_spi_incapacity_benefit_income: + spi_concept: INCPBEN + scope: full + raw_sources: + - BENEFITS.BENEFIT=17:BENAMT + formula: sum(BENAMT where BENEFIT == 17) * 52 + observed_support: structural zero in the FRS 2023-24 and 2024-25; retained so future vintages flow + retained_named_subsets: + ossben_identifiable_subset: + spi_concept: OSSBEN + raw_sources: + - BENEFITS.BENEFIT=13:BENAMT + - BENEFITS.BENEFIT=16,VAR2 in {1,3}:BENAMT + formula: sum(BENAMT where BENEFIT == 13 or (BENEFIT == 16 and VAR2 in {1, 3})) * 52 + scope: identifiable_subset + srp_regular_code5: + spi_concept: SRP + raw_sources: + - BENEFITS.BENEFIT=5:BENAMT + formula: sum(BENAMT where BENEFIT == 5) * 52 + scope: regular_code5_subset + source_absent_full_constituents: + - EPB + - EXPS + - TAXTERM + - MOTHINC + - OTHERINC + full_concepts_forbidden_on_frs: + - hmrc_spi_employment_benefits + - hmrc_spi_employment_expenses + - hmrc_spi_taxable_termination_pay + - hmrc_spi_miscellaneous_employment_income + - hmrc_spi_other_income + - hmrc_spi_other_social_security_income + - hmrc_spi_state_pension_income + forbid_proxy_substitution: + - employment_income + - miscellaneous_income + fail_on_missing_retained_constituent: true + fail_on_full_concept_alias: true + - kind: draw_employer_pension_contributions_from_rate_bands + output: employer_pension_contributions + artifact_role: employer_rate_bands + resource: ashe_employer_pension_contribution_rates.json + scheme_source: PENPROV.STEMPPEN 1 to 4 (employer schemes; occupational before group personal, group stakeholder, other) + sector_source: JOB.JOBSECT (first job with a stated sector) + occupational_split: defined benefit or defined contribution by ASHE Table P2 job counts in the sector + rate_basis: full pay (employment_income) + top_band_ceiling: 0.3 + seed: 0 + salts: + - employer_pension_occupational_type + - employer_pension_contribution_rate + outputs: + - hmrc_spi_pay + - hmrc_spi_unemployment_benefit_income + - hmrc_spi_incapacity_benefit_income + - ossben_identifiable_subset + - srp_regular_code5 + - employer_pension_contributions + nonnegative_outputs: + - hmrc_spi_pay + - hmrc_spi_unemployment_benefit_income + - hmrc_spi_incapacity_benefit_income + - ossben_identifiable_subset + - srp_regular_code5 + - employer_pension_contributions + notes: 'Retains adjudicated raw-FRS HMRC leaves on the raw FRS spine, where person_id equals the raw sernum*1000+person identity, then draws employer pension contributions: each member of an employer scheme (PENPROV.STEMPPEN 1 to 4) with earnings takes an ASHE 2024 employer contribution rate for their scheme type and sector (Table P10 band shares, full pay, all industries; occupational schemes split into defined benefit and defined contribution by the sector''s Table P2 job counts), uniform within the band (top band 20% to 30%), times pay. It replaces the incumbent''s three times the employee contribution (microcosm#1069 c9, R6).' +- stage: frs_annual_income_spells + survey: Family Resources Survey 2024-25 + source: Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 adult, job and household tabs; the OBR average earnings index pinned in hmrc_uprating_engine_pins.json. + grain: person + artifacts: + - role: frs_table + table: adult + kind: licensed_microdata + format: tab + vintage: '2024_25' + locator: adult.tab + sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d + size_bytes: 34885825 + runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 + - role: frs_table + table: job + kind: licensed_microdata + format: tab + vintage: '2024_25' + locator: job.tab + sha256: eb7faf7ada3a3851cb2afb83e2983f8907ffeec897cfbe01e56cb0dfefa853e2 + size_bytes: 10518760 + runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 + - role: frs_table + table: househol + kind: licensed_microdata + format: tab + vintage: '2024_25' + locator: househol.tab + sha256: 2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5 + size_bytes: 12108606 + runtime_sha256_required: true + tax_year_start: 2024 + ukds_study_number: 9563 + doi: 10.5255/UKDA-SN-9563-1 + - role: earnings_index + kind: public_aggregate_reference + resource: hmrc_uprating_engine_pins.json + format: json + runtime_sha256_required: true + operations: + - kind: read_tables + format: tab + delimiter: "\t" + lowercase_columns: true + numeric_errors: coerce + runtime_sha256_required: true + - kind: derive_annual_income_spells + outputs: + - employment_income_annual_share + - employment_income_other_spells + - self_employment_income_annual_share + - self_employment_income_other_spells + - private_pension_income_annual_share + - contributory_benefits_annual_share + window_months: 12 + leaving_date_share: 0.75 + unknown_leaving_months: 6.0 + donor_recent_start_months: 24 + minimum_donors: 30 + donor_age_edges: + - 25 + - 35 + - 50 + - 65 + seed: 0 + salt: employment_income_other_spells + earnings_index: gov.economic_assumptions.indices.obr.average_earnings + earnings_index_from: '2023-01-01' + earnings_index_to: '2024-01-01' + maximum_uprating_months: 24 + outputs: + - employment_income_annual_share + - employment_income_other_spells + - self_employment_income_annual_share + - self_employment_income_other_spells + - private_pension_income_annual_share + - contributory_benefits_annual_share + nonnegative_outputs: + - employment_income_annual_share + - employment_income_other_spells + - self_employment_income_annual_share + - self_employment_income_other_spells + - private_pension_income_annual_share + - contributory_benefits_annual_share + rewrites: [] + notes: 'Describes each FRS adult''s 12 months before the interview beside the current income columns, which it never changes, so the HMRC income-band measures can read annual income while poverty, benefits and the published engine tax keep current income. HMRC counts income received over a tax year: a person who started, left or changed a job, or had a gap, sits in another HMRC band than current pay puts them in, or in none. Outputs: the share of the year paid at today''s employee pay and annual pay from other spells (chiefly a recent leaver''s last job), the same pair for self-employment, and the shares of the year a private pension started on retiring and contributory JSA, ESA and Incapacity Benefit started on leaving work were in payment; an annual amount is the current amount times its share plus the other spells. The State Pension needs none: policyengine-uk''s months_since_state_pension_age gives its share. Months come from CURRJOBM and JOB.WRKPREV for people in work (a person in work is never a leaver), the SAMESIT/SDEMP calendar only where it records a change, and LSTYR, LSTWRK1/2 or RETIRE1 for people out of work who worked in the last 12 months (NUMJOB2 and their last-job JOB rows); a leaving date 12 months or more before the interview gives none. A leaver''s pay is the last job''s usual gross pay (JOB.UGROSS, asked of people out of work who last worked in the previous 12 months, in respect of that job), scaled to the DV concept by the weighted job-level UGRSPAY/UGROSS ratio on current employees who report both and uprated by the pinned OBR average earnings index for the months since leaving; otherwise a donor''s weekly pay, drawn by GROSS4 weight from current employees in their first 24 months in the job within age band x sex x full- or part-time cells (sex dropped below 30 donors), on an identity-keyed uniform (seed 0, salt employment_income_other_spells). Self-employment keeps the whole year where SEJBLONG is one or more, (SEJBMTHS + 1)/12 under a year and at most the calendar''s self-employed months where self-employment is the activity today (the calendar records the main activity only, so a side business beside a job keeps its whole year); a ceased self-employed last job pays SEINCAMT (else PROFIT1, a loss as zero) for its SELWKS weeks; SEINCAM2 is already uprated to the interview by the FRS, so it is not uprated again. The receipt gives each source''s support as a band of distinct households, ratios only where three or more households carry them, and the exact zeros its stage gate reads.' +- stage: geographic_expand + survey: Family Resources Survey 2024-25 + source: Microcosm structural operator (microcosm#931); the FRS spine's own households, no publisher input + grain: household + artifacts: [] + operations: + - kind: expand_geographic_clones + entity: household + clone_index_columns: + person: person_clone_index + benunit: benunit_clone_index + household: household_clone_index + ordinal_bound: 100 + id_remapping: copy-id table field geographic_support; clone k takes every entity id + k x the field's radix (10**9) + order: clone_major + weight_split: equal + weight_kind_out: design + source_columns: + - source_household_id + - source_year + - source_household_key + conservation: household_total_to_rounding + declared_factor: 1.0 + reason: Geographic clones split each household's design weight equally among its clones, so household mass is conserved. + outputs: + - person_clone_index + - benunit_clone_index + - household_clone_index + - source_household_id + - source_year + - source_household_key + nonnegative_outputs: + - person_clone_index + - benunit_clone_index + - household_clone_index + - source_household_id + - source_year + rewrites: [] + notes: 'Gives every FRS household K geographic clones before any other copy step (microcosm#931), so that each clone can later take an area of its own while together they stand for the survey household once. Clone 0 is the frame''s own rows and ids; clone k repeats every person, benefit unit and household with each id minted in the copy-id table''s geographic_support field at ordinal k (id_remapping), and every table carries its rows'' clone, 0 to K - 1, in its clone index column. Each clone carries an equal share of its household''s design weight, the same float in every clone, and the frame''s mass log gains one record whose reason names no clone count. Every household carries its source_household_id (its clone-0 id), source_year (the frame''s year) and source_household_key, so the stages after it count true sources in survey households. K is a parameter of the stage''s graph node, never of this declaration, so the declaration and its contract are the same at every K. The stage runs after frs_hmrc_spine_leaves and frs_annual_income_spells, which join the raw FRS tabs on raw person ids, and before the area stages (atomic_area_assign, local_authority and rfdr), frs_take_up, frs_person_draws, frs_household_draws and area_brma, which run on every clone. Its evidence gives K, the copy-id field and its radix, the distinct source households rounded by the shared disclosure rules, and the weight split and kind; no row count, id or weight.' +- stage: geographic_identity + survey: Family Resources Survey 2024-25 + source: Microcosm structural operator (microcosm#931); the FRS vintage frs_release.json pins, no publisher input + grain: household + artifacts: + - role: frs_release + kind: public_aggregate_reference + resource: frs_release.json + format: json + runtime_sha256_required: true + operations: + - kind: key_households_by_clone_path + source: frs + vintage_resource: frs_release.json + outputs: + - geography_household_key + rewrites: [] + notes: 'Keys every household for the area assignment (microcosm#931): geography_household_key, from the identity source (source), the FRS vintage frs_release.json pins for the household''s source_year, its source_household_id and its clone path, read from the lineage columns geographic_expand writes, never from ids, weights or values: no branch for clone 0 and the geographic_support branch at ordinal k for clone k. So clone 0 keys as a one-clone spine keys the same base household, and growing K never changes an existing household''s key. A source year the release pins no vintage for, and keys that repeat, are refused by count. spi_copy_areas gives each SPI copy and income band donor a key of its own, over its own copy path, and the later copy steps copy a household''s key with the household. Its evidence gives the distinct source households rounded by the shared disclosure rules, that the keys are unique, the source, the vintages and the columns it reads; no key is printed.' +- stage: atomic_area_assign + survey: Family Resources Survey 2024-25 + source: 'Microcosm structural operator (microcosm#931): the shared atomic-area operator over the three conditioned atomic-area supports sources.yaml declares, each system''s areas with their IMD deciles and, in England, Wales and Scotland, their households by council-tax band' + grain: household + artifacts: [] + operations: + - kind: assign_atomic_area + identity: geography_household_key + sources: + uk_ew_output_area_2021: uk_ew_output_area_2021_conditioned_support + uk_scotland_output_area_2022: uk_scotland_output_area_2022_conditioned_support + uk_ni_data_zone_2021: uk_ni_data_zone_2021_conditioned_support + constraints: + - input: region + support: frs_region + required: true + - input: atomic_draw_imd_decile + support: imd_decile + required: false + weight_by: + input: atomic_draw_council_tax_band + default: households + bands: + uk_ew_output_area_2021: + - A + - B + - C + - D + - E + - F + - G + - H + - I + uk_scotland_output_area_2022: + - A + - B + - C + - D + - E + - F + - G + - H + uk_ni_data_zone_2021: [] + draw_inputs: + atomic_draw_imd_decile: + column: imd_decile + unobserved: 0 + atomic_draw_council_tax_band: + column: council_tax_band + observed: council_tax_band_observed + missing_on_copies_of: spi_support_channel + fallback_levels: + - name: L0 + constraints: + - region + - atomic_draw_imd_decile + weight_by: true + - name: L1 + constraints: + - region + - atomic_draw_imd_decile + weight_by: false + - name: L2 + constraints: + - region + weight_by: true + - name: L3 + constraints: + - region + weight_by: false + layers: + - oa_code + - lsoa_code + - msoa_code + - local_authority_code + - ward_code + - constituency_code + - region_code + - itl3_code + - itl2_code + - itl1_code + outputs: + - atomic_area_code + - atomic_area_system + - atomic_area_basis + stream: uk-spine-atomic-area-v1 + seed: 0 + outputs: + - atomic_area_code + - atomic_area_system + - atomic_area_basis + - oa_code + - lsoa_code + - msoa_code + - local_authority_code + - ward_code + - constituency_code + - region_code + - itl3_code + - itl2_code + - itl1_code + rewrites: [] + notes: 'Draws each geographic clone''s atomic area within its FRS region (microcosm#931) with the shared atomic-area operator, on the three conditioned supports the operation names: a 2021 output area in England and Wales, a 2022 output area in Scotland and a 2021 data zone in Northern Ireland. The draw is conditioned on the household''s IMD decile, its imd_decile as "1" to "10" and missing where it is 0, and, where council_tax_band_observed is true, on its council-tax band, by which England, Wales and Scotland weight each area''s households; Northern Ireland''s supports carry no band, so it draws by households. A household whose cell holds no area falls back a level at a time, L0 (decile, by band), L1 (decile, by households), L2 (by band) and L3 (by households), each within the region, as the shared fallback-level chooser picks it, and a level''s dropped input is missing, so one definition serves every level. The inputs are a narrow, transient frame keyed on geography_household_key, so each clone''s draw depends on its key, its own inputs, the supports and the stream alone, and growing K never moves an existing clone''s area. A copy of the SPI support channel would draw at L3, its decile and band missing by rule; none exists yet here, and spi_copy_areas draws the copies'' areas. The stage writes the three assignment columns and the ten ladder columns in the graph''s string dtype, and refuses when the shared operator''s mapping-integrity validation fails. Its evidence gives K, the seed, the stream, the definition''s and supports'' digests, the validation outcome and the distinct source households by system, by level and by fallback reason, rounded by the shared disclosure rules; no key, area code or count of a cell.' +- stage: local_authority + survey: Family Resources Survey 2024-25 + source: The committed ONS April 2023 local authority district names roster (local_authority_names.json) and policyengine-uk's local_authority members + grain: household + artifacts: + - role: local_authority_names + kind: public_aggregate_reference + resource: local_authority_names.json + format: json + runtime_sha256_required: true + operations: + - kind: resolve_local_authority + code_column: local_authority_code + output: local_authority + names_resource: local_authority_names.json + vintage: 2023_april_lad + outputs: + - local_authority + rewrites: [] + notes: 'Resolves each household''s engine local_authority from the local_authority_code its area draw gives it (microcosm#931), through the committed April 2023 local authority names roster (local_authority_names.json), fail-closed: a code off the roster refuses the stage, and the engine input is never defaulted (microcosm#953). The column is written in the graph''s string dtype. Its evidence gives K, the roster''s vintage and digest and the distinct source households, rounded by the shared disclosure rules.' +- stage: rfdr + survey: Family Resources Survey 2024-25 + source: HMRC Notice 2001's Rural Fuel Duty Relief areas, as the committed RFDR area layer (rfdr_area_layer.json) places them in 2021 England and Wales and 2022 Scotland output areas + grain: household + artifacts: + - role: rfdr_area_layer + kind: public_aggregate_reference + resource: rfdr_area_layer.json + format: json + runtime_sha256_required: true + operations: + - kind: assign_rfdr_flags + layer_resource: rfdr_area_layer.json + method: uk-rfdr-area-layer-v1 + area_system: atomic_area_system + area_code: atomic_area_code + roster_sources: + uk_ew_output_area_2021: uk_ew_output_area_2021_conditioned_support + uk_scotland_output_area_2022: uk_scotland_output_area_2022_conditioned_support + require_flagged: false + outputs: + - in_rural_fuel_duty_relief_area + rewrites: [] + notes: Sets each household's in_rural_fuel_duty_relief_area from its drawn output area through the committed Rural Fuel Duty Relief area layer (rfdr_area_layer.json), as a boolean (microcosm#931). Every output area the layer flags must be an area of its system's conditioned support (roster_sources), so the layer and the supports share one code vintage; a system none of whose households is flagged is reported, as a boolean, and not refused (require_flagged), since a spine, a sampled one above all, can miss every flagged area by chance. Northern Ireland has no area in the scheme, so its households are never flagged. Its evidence gives K, the layer's and the supports' digests and the method, the roster check, those booleans and, among clone-0 rows, the distinct flagged source households by system, rounded by the shared disclosure rules. - stage: frs_take_up survey: Family Resources Survey 2024-25 source: Family Resources Survey 2024-25 reported receipt anchors plus sourced UK take-up contract rates. @@ -1159,7 +1609,7 @@ stages: nonnegative_outputs: - maximum_extended_childcare_hours_usage - max_free_entitlement_hours_used - notes: Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100; salts are output variable names. Reported positive receipt anchors are transient and consumed only. The Universal Credit draw's population is the benefit units with a UC claimant or partner (is_uc_claimant) aged 18 or over and under Pension Credit qualifying age (66 for everyone who can reach it in this release), the age and role conditions of policyengine-uk 2.177.7's is_uc_eligible; the engine lockstep test pins it and names the engine cases not mirrored (16- and 17-year-old claimants under the UC Regs 2013 reg 8 exceptions, mixed-age couples on the Pension Credit route, from 2.128.0 pension-age families moved onto Universal Credit when their legacy benefit closes, SI 2014/1230 reg 60A, and from 2.173.2 a unit whose working-age claimants all receive education with no exception, since frs_education sets is_in_full_time_education from the FRS). So the contract rate is a share of those units, an anchor outside them stays true, and any other unit is never drawn; a unit with no claimant or partner of that age keeps its reported Housing Benefit, which a claim it cannot make no longer zeroes (#882, microcosm#867, microcosm#1095). The universal childcare flag is nested under the working-parent one. A unit that claims the extended hours claims the universal hours, and any other unit claims them at the contract's universal_childcare rate, the rate conditional on not claiming the extended hours; the extended draw is unchanged. childcare_copy_take_up later re-draws the childcare flags and hours of the support and CGT copies on their own benefit units. A child of DfE age 0 or 1 (its age on 31 December of the fit's model year, the age DfE counts by) in a unit that claims the working-parent hours uses them at its age's contract rate, extended_childcare_age_0_usage or extended_childcare_age_1_usage (DfE registers far fewer of the youngest eligible children than of those aged 2 to 4, and fewer at 9 to 11 months than at age 1); a child that does not gets max_free_entitlement_hours_used 0, so the engine funds it no hours, and every other person keeps the engine's default of 30. + notes: Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100; salts are output variable names. Reported positive receipt anchors are transient and consumed only. The three anchored draws, would_claim_child_benefit, would_claim_pc and would_claim_uc, count their contract targets within each geographic clone (household_clone_index, microcosm#931), so each clone fills its own residual to int(rate * n_units) over its own units, clone 0 takes a one-clone spine's flags at any K, and a spine without the clone index counts over every unit. The Universal Credit draw's population is the benefit units with a UC claimant or partner (is_uc_claimant) aged 18 or over and under Pension Credit qualifying age (66 for everyone who can reach it in this release), the age and role conditions of policyengine-uk 2.177.7's is_uc_eligible; the engine lockstep test pins it and names the engine cases not mirrored (16- and 17-year-old claimants under the UC Regs 2013 reg 8 exceptions, mixed-age couples on the Pension Credit route, from 2.128.0 pension-age families moved onto Universal Credit when their legacy benefit closes, SI 2014/1230 reg 60A, and from 2.173.2 a unit whose working-age claimants all receive education with no exception, since frs_education sets is_in_full_time_education from the FRS). So the contract rate is a share of those units, an anchor outside them stays true, and any other unit is never drawn; a unit with no claimant or partner of that age keeps its reported Housing Benefit, which a claim it cannot make no longer zeroes (#882, microcosm#867, microcosm#1095). The universal childcare flag is nested under the working-parent one. A unit that claims the extended hours claims the universal hours, and any other unit claims them at the contract's universal_childcare rate, the rate conditional on not claiming the extended hours; the extended draw is unchanged. childcare_copy_take_up later re-draws the childcare flags and hours of the support and CGT copies on their own benefit units. A child of DfE age 0 or 1 (its age on 31 December of the fit's model year, the age DfE counts by) in a unit that claims the working-parent hours uses them at its age's contract rate, extended_childcare_age_0_usage or extended_childcare_age_1_usage (DfE registers far fewer of the youngest eligible children than of those aged 2 to 4, and fewer at 9 to 11 months than at age 1); a child that does not gets max_free_entitlement_hours_used 0, so the engine funds it no hours, and every other person keeps the engine's default of 30. - stage: frs_person_draws survey: Family Resources Survey 2024-25 source: Sourced UK take-up contract rates and identity-keyed deterministic draws. @@ -1198,7 +1648,7 @@ stages: rate_key: apprentice_stock_19_to_24 assignment_unit: person eligibility: 'employees (employment_status FT_EMPLOYED or PT_EMPLOYED, from frs_employment) aged 21 to 24 who are not in full-time education (is_in_full_time_education, from frs_education; the FRS codes no apprentice as in full-time education): SSCBA 1992 s.9A already zero-rates every earner under 21 and s.9B ends the apprentice zero rate at 25, so no other age changes employer NICs, and leaving them False keeps is_in_non_advanced_education, minimum_wage, Care to Learn and the 16 to 19 Bursary Fund unchanged; the flagged keep their age-related minimum_wage too, because policyengine-uk 2.162.8 (policyengine-uk#2683) gives the apprentice rate only under 19 or with is_in_first_year_of_apprenticeship, which the build leaves at its default (False), and the stage refuses an engine without that split or the s.9B threshold' - rate_derivation: 'the contract''s apprentice_stock_19_to_24 is DfE''s England apprentices in learning at the start of the academic year (participation less starts) per head aged 19 to 24, a point-in-time count taken as uniform across those ages and UK-wide; the target is r times the weighted population aged 21 to 24 at the frame''s household weights, and eligible employees are flagged in the order of their identity-keyed uniform until their weight is nearest it (uc_take_up''s rule), or all of them when they weigh no more than it; the receipt records the target, the flagged weight, the support bands and the relief the flags imply at the installed engine''s Class 1 schedule, and uk_stage_frs_person_draws_apprentices gates them, the relief at or below HMRC''s NIC - NS1' + rate_derivation: 'the contract''s apprentice_stock_19_to_24 is DfE''s England apprentices in learning at the start of the academic year (participation less starts) per head aged 19 to 24, a point-in-time count taken as uniform across those ages and UK-wide; the target is r times the weighted population aged 21 to 24 at the frame''s household weights, counted within each geographic clone (household_clone_index, microcosm#931): each clone''s eligible employees are flagged in the order of their identity-keyed uniform until their weight is nearest its own target (uc_take_up''s rule), or all of them when they weigh no more than it; the receipt records the target, the flagged weight, the support bands and the relief the flags imply at the installed engine''s Class 1 schedule, and uk_stage_frs_person_draws_apprentices gates them, the relief at or below HMRC''s NIC - NS1' seed: 0 outputs: - would_claim_marriage_allowance @@ -1242,299 +1692,82 @@ stages: - main_residential_property_purchased_is_first_home - property_purchased notes: Identity-keyed seed 0 streams replace the incumbent sequential take-up seed 100. TV evasion remains an independent draw, matching the incumbent reference share. -- stage: frs_brma - survey: Family Resources Survey 2024-25 - source: Census private-rented households by region, BRMA and bedrooms (ONS and NISRA 2021, NRS 2022) and policyengine-uk LHA_category predictor. - grain: household - artifacts: - - role: count_resource - resource: brma_private_rented_households.json - kind: public_aggregated_counts - format: json - operations: - - kind: materialize_rules_engine_predictors - predictors: - - LHA_category - consumed_only: true - - kind: sample_categorical_from_count_table - output: brma - count_resource: brma_private_rented_households.json - cell_columns: - - region - - LHA_category - assignment_grain: benunit - collapse_to: household - collapse_method: uniform_member_pick - seed: 0 - outputs: - - brma - notes: Identity-keyed seed 0 streams replace the incumbent BRMA seed 0 sequential generator. Household collapse uses salt brma:household_pick. -- stage: frs_hmrc_spine_leaves - survey: Family Resources Survey 2024-25 - source: Department for Work and Pensions Family Resources Survey 2024-25 raw adult.tab and benefits.tab, caller-supplied local input - grain: person - artifacts: - - role: frs_table - table: adult - kind: licensed_microdata - format: tab - vintage: '2024_25' - locator: adult.tab - sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d - size_bytes: 34885825 - runtime_sha256_required: true - tax_year_start: 2024 - ukds_study_number: 9563 - doi: 10.5255/UKDA-SN-9563-1 - - role: frs_table - table: benefits - kind: licensed_microdata - format: tab - vintage: '2024_25' - locator: benefits.tab - sha256: f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3 - size_bytes: 2362329 - runtime_sha256_required: true - tax_year_start: 2024 - ukds_study_number: 9563 - doi: 10.5255/UKDA-SN-9563-1 - - role: frs_table - table: job - kind: licensed_microdata - format: tab - vintage: '2024_25' - locator: job.tab - sha256: eb7faf7ada3a3851cb2afb83e2983f8907ffeec897cfbe01e56cb0dfefa853e2 - size_bytes: 10518760 - runtime_sha256_required: true - tax_year_start: 2024 - ukds_study_number: 9563 - doi: 10.5255/UKDA-SN-9563-1 - - role: frs_table - table: penprov - kind: licensed_microdata - format: tab - vintage: '2024_25' - locator: penprov.tab - sha256: 9e53de0dc969baec000b3cd68387f0f2dfb3f678732e408de175e0a1d6e3fdc1 - size_bytes: 513614 - runtime_sha256_required: true - tax_year_start: 2024 - ukds_study_number: 9563 - doi: 10.5255/UKDA-SN-9563-1 - - role: employer_rate_bands - kind: public_aggregate_reference - resource: ashe_employer_pension_contribution_rates.json - format: json - runtime_sha256_required: true - operations: - - kind: retain_adjudicated_frs_hmrc_leaves - population: uk_frs_raw_spine - source_vintage: 2024-25 - mapped_build_period: 2024 - annualization: weekly raw FRS amounts * 52 - status: adjudicated_partial_replay - retained_full_constituents: - hmrc_spi_pay: - spi_concept: PAY - scope: full - raw_sources: - - ADULT.INEARNS - formula: max(0, ADULT.INEARNS) * 52 - hmrc_spi_unemployment_benefit_income: - spi_concept: UBISJA - scope: full - raw_sources: - - BENEFITS.BENEFIT=14:BENAMT - - BENEFITS.BENEFIT=19:BENAMT - formula: sum(BENAMT where BENEFIT in {14, 19}) * 52 - hmrc_spi_incapacity_benefit_income: - spi_concept: INCPBEN - scope: full - raw_sources: - - BENEFITS.BENEFIT=17:BENAMT - formula: sum(BENAMT where BENEFIT == 17) * 52 - observed_support: structural zero in the FRS 2023-24 and 2024-25; retained so future vintages flow - retained_named_subsets: - ossben_identifiable_subset: - spi_concept: OSSBEN - raw_sources: - - BENEFITS.BENEFIT=13:BENAMT - - BENEFITS.BENEFIT=16,VAR2 in {1,3}:BENAMT - formula: sum(BENAMT where BENEFIT == 13 or (BENEFIT == 16 and VAR2 in {1, 3})) * 52 - scope: identifiable_subset - srp_regular_code5: - spi_concept: SRP - raw_sources: - - BENEFITS.BENEFIT=5:BENAMT - formula: sum(BENAMT where BENEFIT == 5) * 52 - scope: regular_code5_subset - source_absent_full_constituents: - - EPB - - EXPS - - TAXTERM - - MOTHINC - - OTHERINC - full_concepts_forbidden_on_frs: - - hmrc_spi_employment_benefits - - hmrc_spi_employment_expenses - - hmrc_spi_taxable_termination_pay - - hmrc_spi_miscellaneous_employment_income - - hmrc_spi_other_income - - hmrc_spi_other_social_security_income - - hmrc_spi_state_pension_income - forbid_proxy_substitution: - - employment_income - - miscellaneous_income - fail_on_missing_retained_constituent: true - fail_on_full_concept_alias: true - - kind: draw_employer_pension_contributions_from_rate_bands - output: employer_pension_contributions - artifact_role: employer_rate_bands - resource: ashe_employer_pension_contribution_rates.json - scheme_source: PENPROV.STEMPPEN 1 to 4 (employer schemes; occupational before group personal, group stakeholder, other) - sector_source: JOB.JOBSECT (first job with a stated sector) - occupational_split: defined benefit or defined contribution by ASHE Table P2 job counts in the sector - rate_basis: full pay (employment_income) - top_band_ceiling: 0.3 - seed: 0 - salts: - - employer_pension_occupational_type - - employer_pension_contribution_rate - outputs: - - hmrc_spi_pay - - hmrc_spi_unemployment_benefit_income - - hmrc_spi_incapacity_benefit_income - - ossben_identifiable_subset - - srp_regular_code5 - - employer_pension_contributions - nonnegative_outputs: - - hmrc_spi_pay - - hmrc_spi_unemployment_benefit_income - - hmrc_spi_incapacity_benefit_income - - ossben_identifiable_subset - - srp_regular_code5 - - employer_pension_contributions - notes: 'Retains adjudicated raw-FRS HMRC leaves on the raw FRS spine, where person_id equals the raw sernum*1000+person identity, then draws employer pension contributions: each member of an employer scheme (PENPROV.STEMPPEN 1 to 4) with earnings takes an ASHE 2024 employer contribution rate for their scheme type and sector (Table P10 band shares, full pay, all industries; occupational schemes split into defined benefit and defined contribution by the sector''s Table P2 job counts), uniform within the band (top band 20% to 30%), times pay. It replaces the incumbent''s three times the employee contribution (microcosm#1069 c9, R6).' -- stage: frs_annual_income_spells +- stage: area_brma survey: Family Resources Survey 2024-25 - source: Department for Work and Pensions Family Resources Survey 2024-25, UK Data Service SN 9563, DOI 10.5255/UKDA-SN-9563-1; local licensed 2024_25 adult, job and household tabs; the OBR average earnings index pinned in hmrc_uprating_engine_pins.json. - grain: person - artifacts: - - role: frs_table - table: adult - kind: licensed_microdata - format: tab - vintage: '2024_25' - locator: adult.tab - sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d - size_bytes: 34885825 - runtime_sha256_required: true - tax_year_start: 2024 - ukds_study_number: 9563 - doi: 10.5255/UKDA-SN-9563-1 - - role: frs_table - table: job - kind: licensed_microdata - format: tab - vintage: '2024_25' - locator: job.tab - sha256: eb7faf7ada3a3851cb2afb83e2983f8907ffeec897cfbe01e56cb0dfefa853e2 - size_bytes: 10518760 - runtime_sha256_required: true - tax_year_start: 2024 - ukds_study_number: 9563 - doi: 10.5255/UKDA-SN-9563-1 - - role: frs_table - table: househol - kind: licensed_microdata - format: tab - vintage: '2024_25' - locator: househol.tab - sha256: 2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5 - size_bytes: 12108606 + source: 'Great Britain: the committed area-to-BRMA layer (brma_area_layer.json). Northern Ireland: census private-rented households by region, BRMA and bedrooms (ONS and NISRA 2021, NRS 2022) and the policyengine-uk LHA_category predictor.' + grain: household + artifacts: + - role: brma_area_layer + kind: public_aggregate_reference + resource: brma_area_layer.json + format: json runtime_sha256_required: true - tax_year_start: 2024 - ukds_study_number: 9563 - doi: 10.5255/UKDA-SN-9563-1 - - role: earnings_index + - role: count_resource kind: public_aggregate_reference - resource: hmrc_uprating_engine_pins.json + resource: brma_private_rented_households.json format: json runtime_sha256_required: true operations: - - kind: read_tables - format: tab - delimiter: "\t" - lowercase_columns: true - numeric_errors: coerce - runtime_sha256_required: true - - kind: derive_annual_income_spells - outputs: - - employment_income_annual_share - - employment_income_other_spells - - self_employment_income_annual_share - - self_employment_income_other_spells - - private_pension_income_annual_share - - contributory_benefits_annual_share - window_months: 12 - leaving_date_share: 0.75 - unknown_leaving_months: 6.0 - donor_recent_start_months: 24 - minimum_donors: 30 - donor_age_edges: - - 25 - - 35 - - 50 - - 65 + - kind: materialize_rules_engine_predictors + predictors: + - LHA_category + consumed_only: true + - kind: assign_area_brma + output: brma + area_system: atomic_area_system + layer_resource: brma_area_layer.json + method: uk-brma-area-layer-v1 + area_codes: + uk_ew_output_area_2021: lsoa_code + uk_scotland_output_area_2022: oa_code + count_resource: brma_private_rented_households.json + count_table_systems: + - uk_ni_data_zone_2021 + cell_columns: + - region + - LHA_category + assignment_grain: benunit + collapse_to: household + collapse_method: uniform_member_pick seed: 0 - salt: employment_income_other_spells - earnings_index: gov.economic_assumptions.indices.obr.average_earnings - earnings_index_from: '2023-01-01' - earnings_index_to: '2024-01-01' - maximum_uprating_months: 24 + salts: + benunit: brma + household_pick: brma:household_pick outputs: - - employment_income_annual_share - - employment_income_other_spells - - self_employment_income_annual_share - - self_employment_income_other_spells - - private_pension_income_annual_share - - contributory_benefits_annual_share - nonnegative_outputs: - - employment_income_annual_share - - employment_income_other_spells - - self_employment_income_annual_share - - self_employment_income_other_spells - - private_pension_income_annual_share - - contributory_benefits_annual_share + - brma rewrites: [] - notes: 'Describes each FRS adult''s 12 months before the interview beside the current income columns, which it never changes, so the HMRC income-band measures can read annual income while poverty, benefits and the published engine tax keep current income. HMRC counts income received over a tax year: a person who started, left or changed a job, or had a gap, sits in another HMRC band than current pay puts them in, or in none. Outputs: the share of the year paid at today''s employee pay and annual pay from other spells (chiefly a recent leaver''s last job), the same pair for self-employment, and the shares of the year a private pension started on retiring and contributory JSA, ESA and Incapacity Benefit started on leaving work were in payment; an annual amount is the current amount times its share plus the other spells. The State Pension needs none: policyengine-uk''s months_since_state_pension_age gives its share. Months come from CURRJOBM and JOB.WRKPREV for people in work (a person in work is never a leaver), the SAMESIT/SDEMP calendar only where it records a change, and LSTYR, LSTWRK1/2 or RETIRE1 for people out of work who worked in the last 12 months (NUMJOB2 and their last-job JOB rows); a leaving date 12 months or more before the interview gives none. A leaver''s pay is the last job''s usual gross pay (JOB.UGROSS, asked of people out of work who last worked in the previous 12 months, in respect of that job), scaled to the DV concept by the weighted job-level UGRSPAY/UGROSS ratio on current employees who report both and uprated by the pinned OBR average earnings index for the months since leaving; otherwise a donor''s weekly pay, drawn by GROSS4 weight from current employees in their first 24 months in the job within age band x sex x full- or part-time cells (sex dropped below 30 donors), on an identity-keyed uniform (seed 0, salt employment_income_other_spells). Self-employment keeps the whole year where SEJBLONG is one or more, (SEJBMTHS + 1)/12 under a year and at most the calendar''s self-employed months where self-employment is the activity today (the calendar records the main activity only, so a side business beside a job keeps its whole year); a ceased self-employed last job pays SEINCAMT (else PROFIT1, a loss as zero) for its SELWKS weeks; SEINCAM2 is already uprated to the interview by the FRS, so it is not uprated again. The receipt gives each source''s support as a band of distinct households, ratios only where three or more households carry them, and the exact zeros its stage gate reads.' + notes: 'Writes each household''s brma, in the place frs_brma held, after the draws (microcosm#931): LHA_category reads frs_take_up''s would_claim_uc. Great Britain households read it off their drawn LSOA (England and Wales) or output area (Scotland) through the committed area-to-BRMA layer (brma_area_layer.json), fail-closed on an area the layer does not hold or places in another FRS region. Northern Ireland has no open BRMA geography: one engine call over the whole frame materialises LHA_category, and its benefit units draw from the census count table by region and LHA category (count_table_systems) with frs_brma''s identity-keyed seed 0 streams, salt brma for the benefit units and brma:household_pick for the household collapse, so each takes the BRMA frs_brma drew for it. brma stays an object column. Its evidence gives K, the layer''s and the count table''s digests, the seed and salts, and the distinct source households each Great Britain grain reads and those Northern Ireland draws, rounded by the shared disclosure rules.' - stage: spi_support_channel survey: Family Resources Survey 2024-25 - source: Synthetic SPI support channel sampled uniformly without replacement from the raw FRS spine before cloning. + source: Synthetic SPI support channel drawn from every geographic clone of the raw FRS spine's households as a keyed Bernoulli share, each household on its own identity-keyed uniform. grain: household artifacts: [] operations: - kind: stack_zero_weight_donors count: 10000 seed: 42 + salt: spi_support_channel:inclusion flag_column: household_is_spi_synthetic channels: base: frs synthetic: spi - draw: uniform_without_replacement + draw: 'keyed_bernoulli_share: stable_identity_uniforms(household_id, seed, salt) < min(1, count x sample fraction / distinct source households)' + id_remapping: copy-id table field spi_support_channel; the SPI copy takes every entity id + 1 x the field's radix (10**11) - kind: gate_zero_weight_strata gate: uk_zero_weight_strata_gate declarations: - - name: e7_spi_synthetic_preclone + - name: e7_spi_synthetic_share selector: household_is_spi_synthetic: true - maximum_zero_weight_rows: 10000 - reason: E7 stacks exactly 10,000 zero-weight SPI-synthetic households on the raw pre-clone UK FRS spine. + maximum_zero_weight_rows_rule: + rule: binomial_share_band + sigma: 4 + reason: E7 stacks a keyed Bernoulli share of the raw UK FRS spine's households as zero-weight SPI-synthetic households, so their count is binomial; it may exceed the share of the household rows entering the stage by at most the repo's four-sigma binomial band, floored at one row. - kind: allocate_zero_weight_prior_mass share: 0.5 strata: - region + geographic_clone_stratum: household_clone_index pension_age_share: 0.2 pension_age_stratum: any_member_at_or_over_state_pension_age weight_kind_out: importance @@ -1550,9 +1783,6 @@ stages: - household_support_channel - household_support_clone_index - household_source_id - - source_household_id - - source_year - - source_household_key nonnegative_outputs: - person_support_clone_index - person_source_id @@ -1560,9 +1790,7 @@ stages: - benunit_source_id - household_support_clone_index - household_source_id - - source_household_id - - source_year - notes: "Stacks the pre-clone SPI support channel, gates the 10,000 zero-weight synthetic households under a stage-local declaration, and allocates prior mass with exact national conservation within each region, split by whether the household has a member at or over State Pension age (the engine's at the build instant): the SPI channel takes 50 percent of the stratum mass of households without one and pension_age_share of those with one. The channel's households are drawn unweighted with one weight per stratum and every adult in them carries a taxpayer's income, so a region-only allocation gave the channel more pension-age mass than the FRS design gives pension-age households (microcosm#1069)." + notes: "Stacks the SPI support channel as a keyed Bernoulli share (microcosm#931): a household is copied when its uniform, keyed on its own household id under the stage's seed and salt, falls below min(1, 10,000 times the sample fraction over the distinct source households entering the stage). Each household's uniform depends on its own id and the stage's seed and salt alone, and whether it is copied also on that probability. The denominator counts a source once however many geographic clones carry it, so every clone draws at the same probability on its own ids, and each clone's copy count is binomial with mean min(10,000 times the sample fraction, its source households). A stage-local declaration gates the zero-weight synthetic households at the share of the household rows entering the stage plus the repo's four-sigma binomial band. The stage then allocates prior mass with exact national conservation within each region, and within each geographic clone where the frame carries household_clone_index, split by whether the household has a member at or over State Pension age (the engine's at the build instant): the SPI channel takes 50 percent of the stratum mass of households without one and pension_age_share of those with one. A stratum with base mass and no SPI copy refuses the stage at every clone count: a geographic clone's stratum that the share left without a copy is never given its base mass back, which would leave that clone short of its SPI share, so it refuses as a one-clone spine's stratum would (microcosm#931). The channel's households are drawn unweighted with one weight per stratum and every adult in them carries a taxpayer's income, so a region-only allocation gave the channel more pension-age mass than the FRS design gives pension-age households (microcosm#1069)." - stage: spi_income_band_donors survey: Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Income Tax liabilities statistics July 2026 Table 2.5 source: Reserved SPI support households for the HMRC Table 2.5 total-income bands from GBP 200,000, stacked at the published band taxpayer count over the donors per band (PolicyEngine/chronicle#280 lane). @@ -1601,27 +1829,28 @@ stages: minimum_donors_per_band: 120 maximum_donor_weight: 250.0 count_rule: max(minimum_donors_per_band, ceil(published band taxpayers / maximum_donor_weight)) - weight_rule: published band taxpayers / band donor count, equal within a band + weight_rule: published band taxpayers / (geographic clone count x one clone's band donor count), equal within a band expected_band_count: 4 candidate_population: raw FRS base channel claimants and partners (is_uc_claimant) aged 16 and over candidate_order: person_id ascending breaks equal draw keys - draw: 'weighted_without_replacement_identity_keyed: Efraimidis-Spirakis keys log(u) / (household weight x propensity) on stable_identity_uniforms(person_id, seed, salt:band); the largest keys are seated, one carrier per household, the highest band first' + draw: 'weighted_without_replacement_identity_keyed within each geographic clone (a spine without household_clone_index is one clone): Efraimidis-Spirakis keys log(u) / (household weight x propensity) on stable_identity_uniforms(person_id, seed, salt:band); the largest keys are seated, one carrier per household, the highest band first' propensity: SPI 2022-23 FACT-weighted band share by region, sex and age band tape_preparation: 'as hmrc_spi_income_spine prepares the tape: its stage-1 seed and draw_spi_donor_ages_by_population' tape_preparation_seed: 42 seed: 3 salt: spi_income_band_donor_draw clone_index: 2 + id_remapping: copy-id table field spi_support_channel; a donor takes every entity id + 2 x the field's radix (10**11), and its source is read from its source-id columns, never from the id flag_column: household_is_spi_income_band_donor band_column: spi_income_band_donor_lower_bound carrier_column: person_is_spi_income_band_carrier - funding: 'stratum reallocation: every incumbent household of the stratum a donor''s source household belongs to is scaled by 1 - donor mass / stratum mass' + funding: 'stratum reallocation per geographic clone: every incumbent household of the stratum a donor''s source household belongs to, in the donor''s clone, is scaled by 1 - donor mass / stratum mass' funding_strata: - region maximum_reallocation_share: 0.5 maximum_donor_household_share: 0.15 - seating_scale: min(1, maximum_donor_household_share x eligible base households / expected donor count); band counts floored, at least one per band - mass_scale: min(1, min over funding strata of maximum_reallocation_share x stratum mass / seated donor mass); every band's weight scaled alike + seating_scale: min(1, maximum_donor_household_share x eligible base source households / expected donor count of one geographic clone); band counts floored, at least one per band + mass_scale: min(1, min over the funding strata of every geographic clone of maximum_reallocation_share x stratum mass / seated donor mass); one scale for every band and clone never_zero_weight: true weight_kind_out: importance conservation: exact_total @@ -1644,7 +1873,120 @@ stages: - household_source_id nonnegative_outputs: - spi_income_band_donor_lower_bound - notes: Reserves SPI-channel households for every Table 2.5 band from GBP 200,000 so the top of the income distribution has support before calibration. A band seats enough donors that none starts above the maximum weight and never fewer than the minimum, at equal weights of the build tax year's published taxpayers over the band's count. Carriers are FRS adults seated by identity-keyed draws on household weight times the tape's own band propensity given region, sex and age band; their households are copied whole at clone index 2 with the synthetic flag. The mass the donors carry is taken in proportion from the households of each donor's region, so the national total and every region's mass are unchanged (microcosm#1063). The income stage gives every carrier a band-conditional draw from the tape (resample_band_donor_leaves). + notes: Reserves SPI-channel households for every Table 2.5 band from GBP 200,000 so the top of the income distribution has support before calibration. A band seats enough donors that none starts above the maximum weight and never fewer than the minimum, at equal weights of the build tax year's published taxpayers over the band's count. Carriers are FRS adults seated by identity-keyed draws on household weight times the tape's own band propensity given region, sex and age band; their households are copied whole at clone index 2 with the synthetic flag. The mass the donors carry is taken in proportion from the households of each donor's region, so the national total and every region's mass are unchanged (microcosm#1063). On a clone-first spine each geographic clone seats its own donors by the same walk, and every clone takes the same seats, sized against the eligible households counted in distinct source households. Each donor weighs its band's weight over the clone count, and each clone's regions fund their own donors at one mass scale for every clone; a spine without household_clone_index is one clone (microcosm#931). The income stage gives every carrier a band-conditional draw from the tape (resample_band_donor_leaves). +- stage: spi_copy_areas + survey: Family Resources Survey 2024-25 + source: 'Microcosm structural operator (microcosm#931): atomic_area_assign''s draw and the local_authority, area_brma and rfdr lookups, re-run on the rows the SPI support channel copied' + grain: household + artifacts: + - role: frs_release + kind: public_aggregate_reference + resource: frs_release.json + format: json + runtime_sha256_required: true + - role: local_authority_names + kind: public_aggregate_reference + resource: local_authority_names.json + format: json + runtime_sha256_required: true + - role: brma_area_layer + kind: public_aggregate_reference + resource: brma_area_layer.json + format: json + runtime_sha256_required: true + - role: count_resource + kind: public_aggregate_reference + resource: brma_private_rented_households.json + format: json + runtime_sha256_required: true + - role: rfdr_area_layer + kind: public_aggregate_reference + resource: rfdr_area_layer.json + format: json + runtime_sha256_required: true + operations: + - kind: materialize_rules_engine_predictors + predictors: + - LHA_category + consumed_only: true + - kind: assign_spi_copy_areas + copies: + branch: spi_support_channel + paths_require: + - spi_support_channel + - geographic_support + kinds: + '1': spi_support_copy + '2': income_band_donor + key: + column: geography_household_key + source: frs + vintage_resource: frs_release.json + area_draw: + definition_of: atomic_area_assign + level: L3 + stream: uk-spine-atomic-area-v1 + seed: 0 + local_authority: + as_stage: local_authority + brma: + as_stage: area_brma + engine_households: northern_ireland_copies + engine_simulations: one + count_resource: brma_private_rented_households.json + count_table_systems: + - uk_ni_data_zone_2021 + cell_columns: + - region + - LHA_category + assignment_grain: benunit + collapse_to: household + collapse_method: uniform_member_pick + seed: 0 + salts: + benunit: brma + household_pick: brma:household_pick + rfdr: + as_stage: rfdr + require_flagged: false + rewrites: + - geography_household_key + - atomic_area_code + - atomic_area_system + - atomic_area_basis + - oa_code + - lsoa_code + - msoa_code + - local_authority_code + - ward_code + - constituency_code + - region_code + - itl3_code + - itl2_code + - itl1_code + - local_authority + - brma + - in_rural_fuel_duty_relief_area + outputs: [] + rewrites: + - geography_household_key + - atomic_area_code + - atomic_area_system + - atomic_area_basis + - oa_code + - lsoa_code + - msoa_code + - local_authority_code + - ward_code + - constituency_code + - region_code + - itl3_code + - itl2_code + - itl1_code + - local_authority + - brma + - in_rural_fuel_duty_relief_area + notes: Gives every row the SPI support channel copied, an SPI copy (spi_support_channel) or an income band donor (spi_income_band_donors), a key and a place of its own, where each copy had its parent's (microcosm#931). The key is geography_household_key over the copy's own path, read on the support channel's and the geographic clones' branches as geographic_identity reads a path, with the FRS vintage frs_release.json pins for its source_year; the area is drawn at L3, its region by households, with atomic_area_assign's definition, stream and seed, since a copy's decile and band inputs are missing by rule; then its local authority (as local_authority), its BRMA (as area_brma) and its fuel-relief flag (as rfdr). A Great Britain copy reads its BRMA off its area through the layer. Northern Ireland's copies draw theirs from the census count table on their own benefit units with area_brma's seed and salts, after one engine call over their households alone materialises their LHA_category, one simulation over every clone's copies (engine_simulations), so each copy takes a draw of its own (María's ruling of 2026-11-11). Every other row is left as it was, and each rewrite keeps its column's dtype; the copies later stages make inherit their parent's key and area. Its evidence gives K, the seed, the stream, the digests, the validation and roster outcomes and, among clone-0 rows, the distinct source households copied, by kind and system, and those whose copies are flagged, by Great Britain system, rounded by the shared disclosure rules; where one system of a kind rests on one or two source households, every system of that kind is withheld, zeros included, since at one clone the support channel's and the band donors' own copy counts would give it back by difference; no key or area code. - stage: hmrc_spi_income_spine survey: Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Personal Incomes Tables 3.6/3.7 2023-24 source: https://assets.publishing.service.gov.uk/media/69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods @@ -2791,10 +3133,32 @@ stages: notes: 'Runs after spi_employment_shell, which settles every row''s SOC 2020 major group and SIC 2007 division, and before any later stage copies households (microcosm#1490). The FRS codes the major group only; the APS codes the minor group, whose first two digits are the sub-major group. The donors are the APS adults (16 and over, not under-16 status) with a minor group: the main job''s for people in employment (ILODEFR 1), the last job''s otherwise, as the FRS records the main or last job. An employee''s pay (GRSSWK, gross weekly pay in the main job) is asked in the income waves only, so employees without one leave the donors and those with one carry the income weight PIWTA22; everyone else carries the person weight PWTA22. Each major group gets its own model over its sub-major groups: microcosm.fit''s CategoricalClassifier, the equal mixture of a regularised gradient-boosted classifier and a multinomial logit on spline features, on age (capped at 80, the FRS top code), sex, region, employment-status group (employee, self-employed, retired, other), annual pay and SIC 2007 section, the predictors both surveys record alike. Pay is an employee''s on both sides: the APS asks GRSSWK of employees only, so pay is weekly pay times 52 for an employee and zero for everyone else (the few on government employment and training programmes who report one included), and on the frame it is employment income for an employee, the tape''s PAY for an SPI income recipient, and zero outside the employee status group; the logit reads it through arcsinh; a section the group''s donors never hold sets no indicator, and the receipt counts the households that held one. That blend and predictor set held out best on the APS (five stratified folds by person; adding full or part time, usual hours and qualification gained less than 0.02 nats), and the same model trained on the APS scored the FRS''s own major groups within 0.01 nats of a model trained on the FRS. A person with no major group keeps NONE. Draws read uniforms keyed on person_source_id, so a copy draws its FRS twin''s group, except for the SPI income recipients, whose incomes and occupation the SPI stages drew and who key on their own person_id. Codes verified 2026-11-04 against the deposited SN 9408 data dictionary (apsp_a24m25_eul_pwta22_ukda_data_dictionary.rtf): SC20MMN and SC20LMN are minor groups 111 to 926 with -9 and -8 missing; ILODEFR 1 is in employment; INECAC05 1 is employee, 2 self-employed, 16 and 24 retired, 26 under 16, and every other code is grouped as other; SEX 1 male, 2 female; GOVTOF 1 to 13 are the government office regions with Merseyside (3) part of the North West, and agree with GOR9DCENSUS2021 on every record; INDS07M and INDS07L 1 to 21 are SIC 2007 sections A to U; GRSSWK, PWTA22 and PIWTA22 use -9 and -8 for missing. The receipt records the donors by major group and the held-out evidence per major group and pooled: log loss of the model, each learner and the training folds'' class shares; each group''s calibration slope; and sub-major share gaps within pay bands, regions, age bands and status groups; then the FRS rows'' expected sub-major shares against the donors'' within each major group, with counts banded by the shared disclosure rules.' - stage: spi_housing_shell survey: Family Resources Survey 2024-25 - source: Family Resources Survey 2024-25 base households' housing (tenure, dwelling type, bedrooms, council tax, housing costs and the housing-linked benefit reports), imputed onto the SPI support households from their own region, composition and post-SPI incomes. + source: Family Resources Survey 2024-25 base households' housing (tenure, dwelling type, bedrooms, council tax, housing costs and the housing-linked benefit reports), imputed onto the SPI support households from their own region, composition and post-SPI incomes; for the council tax of Great Britain's SPI support households, each local authority's Band D council tax for 2024-25, keyed by April 2023 local authority code, England's from MHCLG's Council Tax levels set by local authorities in England 2024 to 2025 (Table 8) and Wales's from StatsWales (council_tax_band_d_england_wales_2024_25.json), and Scotland's from the Scottish Government's CTAS 2026 Council Tax Assumptions, vendored from the pinned Chronicle feed (scotgov_band_d_council_tax_2024_25.json); the statutory band proportions and single-adult discount (Local Government Finance Act 1992, as amended); and the conditioned atomic-area supports sources.yaml declares, on which each such household's reference Band D is taken over its region's areas, weighted by households. grain: household - artifacts: [] + artifacts: + - role: council_tax_band_d + kind: public_aggregate_reference + resource: council_tax_band_d_england_wales_2024_25.json + format: json + runtime_sha256_required: true + - role: scotland_band_d_facts + kind: public_aggregate_reference + resource: scotgov_band_d_council_tax_2024_25.json + format: json + runtime_sha256_required: true operations: + - kind: verify_vendored_fact_resource + artifact_role: scotland_band_d_facts + resource: scotgov_band_d_council_tax_2024_25.json + feed_pin: chronicle_feed.json + record_sets: + - scotgov.ctas2026.band_d.fy2024 + source_vintage: 2024-25 (ScotGov CTAS 2026) + mapped_build_period: 2024 + period_mapping: fiscal_year_2024_25_is_the_build_period + require_before_source_read: true + runtime_sha256_required: true + fail_on_mismatch: true - kind: impute_spi_housing_shell training_rows: base_frs_channel_positive_weight rows: spi_channel @@ -2849,6 +3213,9 @@ stages: n_estimators: 100 seed: 0 gate_calibration: cross_fitted + - kind: derive + output: council_tax + formula: 'After the band draw and the chained amounts, on the Great Britain rows the SPI support channel copied (SPI copies and income band donors): the 2024-25 statute (council_tax_band_d.uk_statutory_council_tax) at the copy''s reference Band D, the exact expected 2024-25 Band D of the cell spi_copy_areas drew its area from (its region, by households, as council_tax_localise takes it), for its nation and drawn band, less the single-adult discount where council_tax_single_adult_raw is 1. Northern Ireland''s rows keep the chained draw. The bill reads neither the drawn area nor its local authority, so council_tax_localise localises it once.' outputs: [] rewrites: - tenure_type @@ -2868,10 +3235,164 @@ stages: - council_tax_rebate - housing_benefit_reported - council_tax_benefit_reported - notes: 'Runs after the SPI income chain and before the income-conditioned donor stages, the household counterpart of the SPI stage-2 FRS-only fill. The support copies and income band donors are whole FRS households, so without this stage each kept its FRS parent''s housing, drawn for the parent''s income. Trained on the FRS base households (positive household weight) and drawn for every SPI-channel household from engine-free predictors (region, household type, adults, children, single-adult status, reference-person age and household income components). Tenure, dwelling type, bedrooms and council-tax band are drawn in that order by weighted multiclass classifiers with an inverse-CDF draw; the housing amounts and the head benefit unit''s Housing Benefit and the reference person''s council tax benefit by one chained regime-gated QRF conditioned on the drawn categories, clipped to the training range. The QRF''s sign gates are cross-fitted (gate_calibration: cross_fitted; microcosm#1359), as lcfs_consumption''s are: each is calibrated on the donors its fold gates left out, so a sign is drawn at its donor rate on households the gates were not fitted to. Every draw reads uniforms keyed on household_id. An amount is zero wherever the training households have no positive value for its tenure or region; mortgage payments need a mortgaged tenure (true of every FRS household); Housing Benefit needs a rented tenure (the engine pays it to renting reporters only); council tax benefit is capped at council tax and the household council_tax_rebate equals it, both on the reference person as the FRS holds them. Region, single-adult status, BRMA (region and the recipient''s own LHA category), the flat-rate household draws and composition are kept; FRS base rows are byte-unchanged.' + notes: 'Runs after the SPI income chain and before the income-conditioned donor stages, the household counterpart of the SPI stage-2 FRS-only fill. The support copies and income band donors are whole FRS households, so without this stage each kept its FRS parent''s housing, drawn for the parent''s income. Trained on the FRS base households (positive household weight) and drawn for every SPI-channel household from engine-free predictors (region, household type, adults, children, single-adult status, reference-person age and household income components). Tenure, dwelling type, bedrooms and council-tax band are drawn in that order by weighted multiclass classifiers with an inverse-CDF draw; the housing amounts and the head benefit unit''s Housing Benefit and the reference person''s council tax benefit by one chained regime-gated QRF conditioned on the drawn categories, clipped to the training range. The QRF''s sign gates are cross-fitted (gate_calibration: cross_fitted; microcosm#1359), as lcfs_consumption''s are: each is calibrated on the donors its fold gates left out, so a sign is drawn at its donor rate on households the gates were not fitted to. Every draw reads uniforms keyed on household_id. An amount is zero wherever the training households have no positive value for its tenure or region; mortgage payments need a mortgaged tenure (true of every FRS household); Housing Benefit needs a rented tenure (the engine pays it to renting reporters only); council tax benefit is capped at council tax and the household council_tax_rebate equals it, both on the reference person as the FRS holds them. Region, single-adult status, BRMA (region and the recipient''s own LHA category), the flat-rate household draws and composition are kept; FRS base rows are byte-unchanged. Council tax stays a target of the chained QRF on every row, so every band and every other chained amount is the draw''s. After the learned zeros and the tenure rules, a Great Britain row''s drawn council tax is replaced by the 2024-25 statute at its reference Band D (the stage''s derive of council_tax): the expected 2024-25 Band D of the cell spi_copy_areas draws the row''s area from, its region by households, recomputed on the conditioned atomic-area supports as council_tax_localise recomputes it, for the row''s nation and drawn band, less the single-adult discount where council_tax_single_adult_raw is 1. A Northern Ireland row keeps the chained draw (María''s ruling of 2026-11-14). The bill reads neither the drawn area nor its local authority, so council_tax_localise localises it once. The council tax benefit is capped at the bill written. Loading the committed Band D holds Scotland''s vendored resource to the pinned Chronicle feed, and before it reads a row the stage checks that its Scottish declaration maps the facts to the frame''s period, as council_tax_localise does; it refuses unless the bill''s checks hold on what it writes, and its receipt records the bill''s method, inputs and checks under council_tax, with no number.' +- stage: private_rent + survey: Family Resources Survey 2024-25 + source: The BRMAs' weekly lists of rents (brma_rent_quantiles.json) and the census private-rented households by region, BRMA and bedrooms (brma_private_rented_households.json). + grain: household + artifacts: + - role: rent_quantiles + kind: public_aggregate_reference + resource: brma_rent_quantiles.json + format: json + runtime_sha256_required: true + - role: count_resource + kind: public_aggregate_reference + resource: brma_private_rented_households.json + format: json + runtime_sha256_required: true + operations: + - kind: localise_private_rents + tenure_type: RENT_PRIVATELY + excluded_regions: + - NORTHERN_IRELAND + identity: geography_household_key + process: rent_rank + stream: uk-spine-private-rent-v1 + seed: 0 + categories: + - B + - C + - D + - E + reference_channel: frs + quantiles_resource: brma_rent_quantiles.json + quantiles_method: uk-brma-rent-quantiles-v1 + counts_resource: brma_private_rented_households.json + weeks_in_year: 52 + scale_band: + - 0.25 + - 4.0 + sampled_region_renters: 10 + method: uk-private-rent-localisation-v1 + - kind: record_mass_conservation_receipt + entity: household + reason: 'Private rents read off the BRMA lists on the source spine: household weights pass through unchanged and total household mass is conserved.' + declared_factor: 1.0 + gate_coupling: The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this stage-specific reason. + outputs: [] + rewrites: + - rent + notes: 'Reads each Great Britain private renter''s rent (tenure_type RENT_PRIVATELY, a positive rent, outside Northern Ireland) off the list of rents of the BRMA its drawn area sits in (brma_rent_quantiles.json), as the dense line''s private-rent node did (microcosm#1262, microcosm#931). The renter keeps its rank among the FRS renters of its FRS region and bedroom category, the rank reads the list of its BRMA and category, and each region is scaled so that, on the census private-renter mix of its BRMAs (brma_private_rented_households.json), its mean rent is the FRS level. Each row ranks as a household of its own, on its geography_household_key, and its tie-break uniform is keyed on that key on the stage''s own stream at its declared seed, so two FRS renters at one rent, or two clones of one renter, take ranks of their own, and clone 0''s tie-breaks are the same at every K. A region''s FRS renters are counted in true source households, so a renter''s clones and copies count once: a sampled spine ranks and scales a region with fewer FRS renters than sampled_region_renters against Great Britain''s, and a full-scale spine ranks every region on its own and refuses one with none. Only rent is rewritten: reported Housing Benefit, Universal Credit and mortgage payments stay as reported, the weights pass through, and the stage records its household mass receipt. Its evidence gives K, the seed, the stream, the resources'' digests, the method, the strictness, the numbers of (region, bedroom category) cells ranked against their region or against Great Britain and, among clone-0 rows, the distinct source households re-drawn, rounded by the shared disclosure rules.' +- stage: council_tax_localise + survey: Family Resources Survey 2024-25 + source: Each Great Britain local authority's Band D council tax for 2024-25, keyed by April 2023 local authority code, England's from MHCLG's Council Tax levels set by local authorities in England 2024 to 2025 (Table 8) and Wales's from StatsWales (council_tax_band_d_england_wales_2024_25.json), and Scotland's from the Scottish Government's CTAS 2026 Council Tax Assumptions, vendored from the pinned Chronicle feed (scotgov_band_d_council_tax_2024_25.json); the statutory band proportions and single-adult discount (Local Government Finance Act 1992, as amended); and the conditioned atomic-area supports sources.yaml declares, on which each household's draw cell is recomputed. + grain: household + artifacts: + - role: council_tax_band_d + kind: public_aggregate_reference + resource: council_tax_band_d_england_wales_2024_25.json + format: json + runtime_sha256_required: true + - role: scotland_band_d_facts + kind: public_aggregate_reference + resource: scotgov_band_d_council_tax_2024_25.json + format: json + runtime_sha256_required: true + operations: + - kind: verify_vendored_fact_resource + artifact_role: scotland_band_d_facts + resource: scotgov_band_d_council_tax_2024_25.json + feed_pin: chronicle_feed.json + record_sets: + - scotgov.ctas2026.band_d.fy2024 + source_vintage: 2024-25 (ScotGov CTAS 2026) + mapped_build_period: 2024 + period_mapping: fiscal_year_2024_25_is_the_build_period + require_before_source_read: true + runtime_sha256_required: true + fail_on_mismatch: true + - kind: derive + output: council_tax_frs_bill + formula: 'max(council_tax - council_tax_rebate, 0) on every row, from the bill and rebate before this stage rewrites them: the paid bill was_wealth''s council-tax predictor reads, so the area does not enter that predictor (ruling 2)' + - kind: localise_council_tax + method: uk-council-tax-localise-v1 + band_d_resources: + england_wales: council_tax_band_d_england_wales_2024_25.json + scotland: scotgov_band_d_council_tax_2024_25.json + band_d_year: 2024-25 + authority_column: local_authority_code + reference: expected_band_d_of_draw_cell + draw_stage: atomic_area_assign + copy_draw_stage: spi_copy_areas + support_sources: + uk_ew_output_area_2021: uk_ew_output_area_2021_conditioned_support + uk_scotland_output_area_2022: uk_scotland_output_area_2022_conditioned_support + uk_ni_data_zone_2021: uk_ni_data_zone_2021_conditioned_support + provenance_column: council_tax_bill_provenance + ratio_classes: + - observed_gross + formula_classes: + - missing + - unknown_reduction + formula_needs: council_tax_band_observed + band_ratios: + ENGLAND: + A: 6/9 + B: 7/9 + C: 8/9 + D: 9/9 + E: 11/9 + F: 13/9 + G: 15/9 + H: 18/9 + WALES: + A: 6/9 + B: 7/9 + C: 8/9 + D: 9/9 + E: 11/9 + F: 13/9 + G: 15/9 + H: 18/9 + I: 21/9 + SCOTLAND: + A: 240/360 + B: 280/360 + C: 320/360 + D: 360/360 + E: 473/360 + F: 585/360 + G: 705/360 + H: 882/360 + single_adult_column: council_tax_single_adult_raw + single_adult_discount: 0.25 + scaled_columns: + household: + - council_tax_rebate + person: + - council_tax_benefit_reported + cap: reduction_and_rebate_at_bill + excluded_regions: + - NORTHERN_IRELAND + support_channel_rows: ratio_at_copy_draw_level + - kind: record_mass_conservation_receipt + entity: household + reason: 'Council tax localised by each household''s local authority on the source spine: household weights pass through unchanged and total household mass is conserved.' + declared_factor: 1.0 + gate_coupling: The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this stage-specific reason. + outputs: + - council_tax_frs_bill + nonnegative_outputs: + - council_tax_frs_bill + rewrites: + - council_tax + - council_tax_rebate + - council_tax_benefit_reported + notes: Prices each Great Britain household's council tax at the local authority the spine drew for it, after private_rent and just before the WAS chain (María's rulings of 2026-11-06, 2026-11-13 and 2026-11-14, microcosm#931). Before it rewrites anything, it keeps the paid bill, max(council_tax - council_tax_rebate, 0), as council_tax_frs_bill on every row; was_wealth's council-tax predictor, which was_lisa builds through was_wealth, reads that column, so the area does not enter it, and the release boundary drops it. A household's ratio is its authority's (local_authority_code) 2024-25 Band D over the expected Band D of its own draw cell, recomputed exactly from the inputs atomic_area_assign drew it on, each of the cell's areas weighted as the draw weights it; the rows the SPI support channel copied, SPI copies and income band donors, take the cell spi_copy_areas drew them from, their region by households. A row of the FRS channel whose bill frs_council_tax observed gross (council_tax_bill_provenance observed_gross) takes the ratio; a missing bill, or one netted of a reduction of unknown size (missing, unknown_reduction), takes the 2024-25 statute where council_tax_band_observed holds (the authority's Band D times its band's statutory proportion, less the single-adult discount where council_tax_single_adult_raw is 1) and the ratio on frs_council_tax's bill where it does not. The rebate and the one person's reported council tax reduction are scaled by the same ratio and capped at the new bill. Northern Ireland, which pays domestic rates, keeps every row as it is, with no cap. A Great Britain row whose drawn area is not an area of its recomputed cell is refused. The WAS chain's HBAI net income reads the localised bill and reduction. The weights pass through, and the stage records its household mass receipt. Its evidence gives K, the Band D resources' and the supports' digests, the method and its checks as booleans, with the distinct source households on each path (the FRS channel's over every clone, the support channel's among clone-0 rows) and, among clone-0 rows, those whose reductions were capped and Northern Ireland's whose reductions exceed their bill, counted through receipt_counts. - stage: was_wealth survey: Wealth and Assets Survey round 8 - source: Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab. + source: Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab. England and Wales owners' main residences are read off HM Land Registry Price Paid latest sales uprated with the UK House Price Index, by constituency (uk/high_value_home_value_knots.json, built by tools/build_uk_high_value_home_stock.py from PolicyEngine/chronicle#343's registrations), and the public regional reference (uk/regional_land_values.json) (microcosm#931). grain: household base_candidate: filename: populace_uk_2023.h5 @@ -2889,6 +3410,16 @@ stages: access: private_local_input locator: caller-supplied local input runtime_sha256_required: true + - role: value_knots + kind: public_aggregate_reference + resource: high_value_home_value_knots.json + format: json + runtime_sha256_required: true + - role: regional_reference + kind: public_aggregate_reference + resource: regional_land_values.json + format: json + runtime_sha256_required: true operations: - kind: derive scope: strict WAS donor cleaning with lower-case exact column matching and no fuzzy r/w fallback @@ -2914,6 +3445,7 @@ stages: num_children: NumChildR8 council_tax: DVCTaxAmtAnnualR8 rental_income: DVNetRentAmtAnnualR8_aggr + hbai_household_net_income: DVTotinc_bhcR8 derived: private_pension_wealth: totalpenr8_aggr - dvvaldbt_scaper8_aggr corporate_wealth_excl_isa: DVFESHARESR8_aggr + DVFShUKVR8_aggr + DVFCollVR8_aggr @@ -2933,7 +3465,7 @@ stages: region: GORR8 via incumbent REGIONS map - kind: materialize_rules_engine_predictors predictors: - - household_net_income + - hbai_household_net_income_before_reset_to_zero - private_pension_income - employment_income - self_employment_income @@ -2944,7 +3476,7 @@ stages: seed: 0 n_estimators: 100 predictors: - - household_net_income + - hbai_household_net_income - num_adults - num_children - private_pension_income @@ -3036,6 +3568,36 @@ stages: undrawn_fraction: 1.0 secured_debt_within: other_mortgage gate_calibration: cross_fitted + - kind: localise_main_residence_value + value_column: main_residence_value + after_segment: 2 + rows: owner households (tenure owned outright or with a mortgage) with a positive drawn main_residence_value in the FRS regions of England and Wales + rank: weighted mid-rank of the drawn main_residence_value among the owner households of the FRS region with a positive draw, both support channels, at the stage's household weights, ties broken by a draw keyed on the household id + stream: + - sha256-u53-v1 + - uk-spine-area-home-values-v1 + - 0 + - 0 + knots: high_value_home_value_knots.json + knots_level: constituency + constituency_column: constituency_code + regional_reference: regional_land_values.json + value: np.interp(rank, grid, the assigned constituency's knots) divided by the FRS region's factor, a home at the knots' cap taking the largest quotient the factor multiplies back to the cap or below + factor: avg_house_price / unweighted_mean(drawn main_residence_value | region, FRS-base owners, >0), the regional property uprating's factor on the draw + region_map: + EAST_MIDLANDS: E12000004 + EAST_OF_ENGLAND: E12000006 + LONDON: E12000007 + NORTH_EAST: E12000001 + NORTH_WEST: E12000002 + SOUTH_EAST: E12000008 + SOUTH_WEST: E12000009 + WALES: W92000004 + WEST_MIDLANDS: E12000005 + YORKSHIRE: E12000003 + rank_basis_column: main_residence_value_rank_basis + kept: Scotland, Northern Ireland, renters and owners without a positive draw keep the draw + method: uk-spine-area-home-values-v1 - kind: fold_into output: corporate_wealth inputs: @@ -3045,6 +3607,9 @@ stages: - corporate_wealth_excl_isa - kind: support_clip range: donor_realized + exempt: + - main_residence_value + - property_wealth - kind: allocate_within_group_waterfall source: household.student_loan_balance target: person.student_loan_balance @@ -3069,6 +3634,7 @@ stages: - gross_financial_wealth - net_financial_wealth - main_residence_value + - main_residence_value_rank_basis - other_residential_property_value - non_residential_property_value - savings @@ -3089,6 +3655,7 @@ stages: - private_pension_wealth - gross_financial_wealth - main_residence_value + - main_residence_value_rank_basis - other_residential_property_value - non_residential_property_value - savings @@ -3102,7 +3669,7 @@ stages: - unit_and_investment_trusts - other_residential_property_secured_debt - non_residential_property_secured_debt - notes: 'Ports incumbent WAS round-8 wealth imputation with signed E5 differences: exact lower-case column matching replaces the fuzzy r/w fallback; cash ISA uses DVCISAVR8_aggr and stocks-and-shares ISA uses DVIISAVR8_aggr; private pension wealth (WAS total private pension wealth less the current defined-benefit component; DB-type components at the SCAPE rate, money-purchase components at reported fund values) is drawn as its own output private_pension_wealth and no longer folded into corporate_wealth, which keeps only the share-like holdings - pension rights are disregarded capital in every means test (UC Regs 2013 Sch 10 para 10 and the HB/JSA/ESA/IS/SPC parallels; uk-data#452) - and the downstream chain segment conditions on both components; each chain segment starts its own fit and draw RNG streams from a child seed derived deterministically from the declared seed (start_chain respawns from the model seed on every call, so one shared model coupled the k-th target of every segment); corporate_wealth folds stocks-and-shares ISA after drawing corporate_wealth_excl_isa; recipient Northern Ireland regions are mapped to Wales for prediction only; student_loan_balance is allocated by household id rather than the incumbent positional off-by-one. Its QRF sign gates are cross-fitted (gate_calibration: cross_fitted; microcosm#1359), as lcfs_consumption''s are: each is calibrated on the donors its fold gates left out, so a sign is drawn at its donor rate on households the gates were not fitted to. Tenure enters every chain segment as one four-way category built the same way on both sides (microcosm#1063): the donor''s from Ten1R8 and DVPriRntR8 (owned outright; buying with a mortgage or part rent, part mortgage; rented or rent-free from a private landlord; rented from a council or housing association), the recipient''s from the frame''s tenure_type (council and housing association together), so the stage reads no engine tenure formula. Two columns are structural in tenure and are fitted and drawn inside their stratum only, zero by rule elsewhere: main_residence_value on the owner tenures (every WAS owner has a positive value and no renter has one) and main_residence_mortgage (TotMortR8) on the mortgaged tenure. mortgage_debt is HMortGR8, all of the household''s mortgages, and is derived as the main-residence mortgage plus the mortgages on other property (other_mortgage, HMortGR8 less TotMortR8 on the donor: buy-to-let, second homes and the rest), which are held on every tenure and drawn without a stratum, so an outright owner''s or a renter''s buy-to-let keeps its mortgage beside the property it is secured on (OthMortR8_sum is read for the receipt only); the donor''s conditioning copy carries the same rule, so consumer_debt is fitted on values recipients can hold. Totals are not drawn: property_wealth is the sum of land, the main residence, other houses, buildings and a drawn remainder (buy-to-let, overseas and other property), gross_financial_wealth is savings, the cash ISA, the share-like holdings and a drawn remainder, and net_financial_wealth is gross less consumer debt and the student loan balance - identities the pinned tab satisfies on every row and the cleaning step refuses a donor that breaks. A sum of donor-valued components that leaves the donor''s range for the total gives up the excess from the remainder first, and the receipt counts the rows. Eleven chain segments, each with its own child seed: land; the main residence; the other property components; pension and share-like wealth; the financial components, vehicles and the student loan balance; the main-residence mortgage; the other mortgages; consumer debt; then three appended after them, so the first eight draw as they did before them (microcosm#1095, the WAS property draw), each fitted on the donors and drawn for the recipients holding the column holder_strata names. The first is the directly held fraction of the share-like holdings outside ISAs, from which the stage writes directly_held_shares (WAS UK shares plus employee shares and options) and unit_and_investment_trusts (WAS collective investments, which the survey asks about in one question; uk-data#501), the inputs policyengine-uk reads inside corporate_wealth (pe-uk#1974): the split is written on the final corporate_wealth less the stocks-and-shares ISA, so the two and the ISA sum to corporate_wealth and the engine''s share_like_capital stays corporate_wealth; a household with such holdings but no drawn fraction (a clip that lowered the ISA alone) holds them directly (undrawn_fraction), and the receipt counts it. The other two are the debt secured on other residential property (WAS DVHseDebtR8_sum + DVBLtDebtR8_sum) and on non-residential property (DVBldDebtR8_sum), which policyengine-uk deducts after the sale costs in valuing Universal Credit property capital (UC Regs 2013 reg 49(1)(b)): each is zero where the final value of its asset is zero, and the two are parts of the mortgages on other property, so where their sum passes other_mortgage both are scaled down to it (secured_debt_within); the receipt counts the rows each rule touched, and the wealth_coherence gate holds the split and the debts as identities. HFINWNTR8_exSLC_Sum is net financial wealth excluding student loans, so gross less it is the non-student-loan financial liabilities; UKDS SN 7215 documentation is the codebook. Engine predictors materialize at their native entity and person/benunit values are summed to household, reproducing the incumbent map_to=household semantics; region is one-hot encoded jointly across donor and recipient (the incumbent''s dummy encoding), with unmapped donor GOR codes becoming all-zero dummy rows. The WAS and FRS predictor definitions are not fully like-for-like and are ported as-is; raw WAS missing values are blanket-filled with zero; UKDS negative sentinel codes (-9/-8/-7/-6) are recoded to zero for the nonnegative-domain columns the licensed audit found carrying them (vcarnr8: 2 rows; HBedRmR8: 95.8 percent - the bedrooms question is effectively unasked in the WAS household file, predictor-quality revisit registered on microcosm#145) - a signed difference vs the incumbent, which trains on raw sentinels; DVPriRntR8''s -9 is structural not-applicable so the private-renter reading is unchanged; genuinely negative domains are never recoded. The recipient council_tax predictor is the bill the household pays, council_tax less council_tax_rebate, the quantity the WAS donor records, since the spine''s council_tax is the liability before council tax reduction (uk-data#496/#499, microcosm#1095). The donor council_tax is DVCTaxAmtAnnualR8, the annual amount paid (CTAmtR8 is one instalment), its children are NumChildR8 (NumCh18R8 overlaps NumAdultR8), and the recipient adults and children are the household''s FRS claimants and partners and its other members (is_uc_claimant), which together make the household size on both sides; rental_income is the donor''s net rent DVNetRentAmtAnnualR8_aggr against the household''s property_income (uk-data#486/#495, microcosm#1095).' + notes: 'Ports incumbent WAS round-8 wealth imputation with signed E5 differences: exact lower-case column matching replaces the fuzzy r/w fallback; cash ISA uses DVCISAVR8_aggr and stocks-and-shares ISA uses DVIISAVR8_aggr; private pension wealth (WAS total private pension wealth less the current defined-benefit component; DB-type components at the SCAPE rate, money-purchase components at reported fund values) is drawn as its own output private_pension_wealth and no longer folded into corporate_wealth, which keeps only the share-like holdings - pension rights are disregarded capital in every means test (UC Regs 2013 Sch 10 para 10 and the HB/JSA/ESA/IS/SPC parallels; uk-data#452) - and the downstream chain segment conditions on both components; each chain segment starts its own fit and draw RNG streams from a child seed derived deterministically from the declared seed (start_chain respawns from the model seed on every call, so one shared model coupled the k-th target of every segment); corporate_wealth folds stocks-and-shares ISA after drawing corporate_wealth_excl_isa; recipient Northern Ireland regions are mapped to Wales for prediction only; student_loan_balance is allocated by household id rather than the incumbent positional off-by-one. Its QRF sign gates are cross-fitted (gate_calibration: cross_fitted; microcosm#1359), as lcfs_consumption''s are: each is calibrated on the donors its fold gates left out, so a sign is drawn at its donor rate on households the gates were not fitted to. Tenure enters every chain segment as one four-way category built the same way on both sides (microcosm#1063): the donor''s from Ten1R8 and DVPriRntR8 (owned outright; buying with a mortgage or part rent, part mortgage; rented or rent-free from a private landlord; rented from a council or housing association), the recipient''s from the frame''s tenure_type (council and housing association together), so the stage reads no engine tenure formula. Two columns are structural in tenure and are fitted and drawn inside their stratum only, zero by rule elsewhere: main_residence_value on the owner tenures (every WAS owner has a positive value and no renter has one) and main_residence_mortgage (TotMortR8) on the mortgaged tenure. mortgage_debt is HMortGR8, all of the household''s mortgages, and is derived as the main-residence mortgage plus the mortgages on other property (other_mortgage, HMortGR8 less TotMortR8 on the donor: buy-to-let, second homes and the rest), which are held on every tenure and drawn without a stratum, so an outright owner''s or a renter''s buy-to-let keeps its mortgage beside the property it is secured on (OthMortR8_sum is read for the receipt only); the donor''s conditioning copy carries the same rule, so consumer_debt is fitted on values recipients can hold. Totals are not drawn: property_wealth is the sum of land, the main residence, other houses, buildings and a drawn remainder (buy-to-let, overseas and other property), gross_financial_wealth is savings, the cash ISA, the share-like holdings and a drawn remainder, and net_financial_wealth is gross less consumer debt and the student loan balance - identities the pinned tab satisfies on every row and the cleaning step refuses a donor that breaks. A sum of donor-valued components that leaves the donor''s range for the total gives up the excess from the remainder first, and the receipt counts the rows. Eleven chain segments, each with its own child seed: land; the main residence; the other property components; pension and share-like wealth; the financial components, vehicles and the student loan balance; the main-residence mortgage; the other mortgages; consumer debt; then three appended after them, so the first eight draw as they did before them (microcosm#1095, the WAS property draw), each fitted on the donors and drawn for the recipients holding the column holder_strata names. The first is the directly held fraction of the share-like holdings outside ISAs, from which the stage writes directly_held_shares (WAS UK shares plus employee shares and options) and unit_and_investment_trusts (WAS collective investments, which the survey asks about in one question; uk-data#501), the inputs policyengine-uk reads inside corporate_wealth (pe-uk#1974): the split is written on the final corporate_wealth less the stocks-and-shares ISA, so the two and the ISA sum to corporate_wealth and the engine''s share_like_capital stays corporate_wealth; a household with such holdings but no drawn fraction (a clip that lowered the ISA alone) holds them directly (undrawn_fraction), and the receipt counts it. The other two are the debt secured on other residential property (WAS DVHseDebtR8_sum + DVBLtDebtR8_sum) and on non-residential property (DVBldDebtR8_sum), which policyengine-uk deducts after the sale costs in valuing Universal Credit property capital (UC Regs 2013 reg 49(1)(b)): each is zero where the final value of its asset is zero, and the two are parts of the mortgages on other property, so where their sum passes other_mortgage both are scaled down to it (secured_debt_within); the receipt counts the rows each rule touched, and the wealth_coherence gate holds the split and the debts as identities. HFINWNTR8_exSLC_Sum is net financial wealth excluding student loans, so gross less it is the non-student-loan financial liabilities; UKDS SN 7215 documentation is the codebook. Engine predictors materialize at their native entity and person/benunit values are summed to household, reproducing the incumbent map_to=household semantics; region is one-hot encoded jointly across donor and recipient (the incumbent''s dummy encoding), with unmapped donor GOR codes becoming all-zero dummy rows. The household income predictor, hbai_household_net_income, matches the donor''s DVTotinc_bhcR8, net regular household income before housing costs less council tax, to the engine''s closest concept, policyengine-uk''s HBAI net income before housing costs and before the reset to zero (hbai_household_net_income_before_reset_to_zero, which the LCFS, ETB and NTS stages also read; microcosm#1164). The engine''s household_net_income it replaces also takes off stamp duty, the TV licence and the business rates and corporate stamp duty that shareholding spreads over the households the engine simulates, and adds the imputed value of NHS, education and transport spending (microcosm#931). The WAS and FRS predictor definitions, that income among them, are not fully like-for-like and are ported as-is: the recipient''s income also counts HBAI''s in-kind items (free school meals, school milk and fruit, the free TV licence''s value, Healthy Start vouchers and the Tax-Free Childcare top-up) and the Warm Home Discount, which WAS''s regular income is not documented to include, and how WAS treats pension contributions, maintenance paid, student loan repayments and negative incomes is not verified; raw WAS missing values are blanket-filled with zero; UKDS negative sentinel codes (-9/-8/-7/-6) are recoded to zero for the nonnegative-domain columns the licensed audit found carrying them (vcarnr8: 2 rows; HBedRmR8: 95.8 percent - the bedrooms question is effectively unasked in the WAS household file, predictor-quality revisit registered on microcosm#145) - a signed difference vs the incumbent, which trains on raw sentinels; DVPriRntR8''s -9 is structural not-applicable so the private-renter reading is unchanged; genuinely negative domains are never recoded. The recipient council_tax predictor is the bill the household pays, council_tax less council_tax_rebate, the quantity the WAS donor records, since the spine''s council_tax is the liability before council tax reduction (uk-data#496/#499, microcosm#1095). The donor council_tax is DVCTaxAmtAnnualR8, the annual amount paid (CTAmtR8 is one instalment), its children are NumChildR8 (NumCh18R8 overlaps NumAdultR8), and the recipient adults and children are the household''s FRS claimants and partners and its other members (is_uc_claimant), which together make the household size on both sides; rental_income is the donor''s net rent DVNetRentAmtAnnualR8_aggr against the household''s property_income (uk-data#486/#495, microcosm#1095).' - stage: was_lisa survey: Wealth and Assets Survey round 8 source: Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain). @@ -3183,7 +3750,7 @@ stages: sentinel_recode_columns: - employment_income household_predictors: - - household_net_income + - hbai_household_net_income - num_adults - num_children - gross_financial_wealth @@ -3196,9 +3763,7 @@ stages: basis: LISAs have been opened only by people aged 18 to 39 since 6 April 2017, so no holder was older than 44 by the end of round-8 fieldwork (March 2022); a holder recorded in an age band starting at 45 or above is a misreported ISA type and is recoded to a non-holder. About GBP 30,500 of contributions, government bonus and the one-off 2017-18 Help to Buy ISA transfer was attainable before returns, so a holder above GBP 40,000 stays a holder but leaves the balance fit. - kind: materialize_rules_engine_predictors predictors: - - household_net_income - - num_adults - - num_children + - hbai_household_net_income_before_reset_to_zero - private_pension_income - employment_income - self_employment_income @@ -3216,7 +3781,7 @@ stages: age_group_lower_bounds: [0, 25, 35, 45, 55] log1p_predictors: - employment_income - - household_net_income + - hbai_household_net_income - gross_financial_wealth - cash_isa - stocks_and_shares_isa @@ -3250,7 +3815,7 @@ stages: - age_band - is_female - employment_income - - household_net_income + - hbai_household_net_income - gross_financial_wealth - savings - cash_isa @@ -3280,7 +3845,7 @@ stages: nonnegative_outputs: - lifetime_isa_balance - household_lifetime_isa_balance - notes: 'Imputes Lifetime ISA holdings per adult from the WAS round-8 person tab (microcosm#1003). The donor is every non-dependent adult of the person tab joined many-to-one on CASER8 to the cleaned household tab (the was_wealth donor, same pins); the released value DVFLISAvR8 (the ONS-completed FLISAVR8_i) must sum to the household aggregate DVFLISAVR8_aggr on every household or the stage refuses the tab, and a missing or negative value is refused rather than read as a zero balance. Response classes (observed, banded with the ONS value inside the reported band, ONS-imputed value or holding, rule-impossible) are receipted and never used as predictors. The declared credibility rule recodes holders recorded in an age band starting at 45 or above to non-holders and keeps holders above the balance ceiling as holders outside the balance fit (basis on the clean operation). Ownership is a weighted logistic regression with a ridge penalty on the slopes and an unpenalised intercept, so the donor''s weighted ownership share is reproduced in-sample, on the age group, the log of person earnings, household net income, gross financial wealth, cash ISA, stocks-and-shares ISA and savings, private renting, sex and the number of children; the house regime gate (gradient boosting) is miscalibrated for an outcome held by under one per cent of adults with about 120 donor holders. Recipients aged 18 or over hold when a uniform keyed on person_id falls below their probability; no upper age limit is imposed. The balance is the regime-gated QRF fitted on the credible holders and drawn at the owner''s keyed quantile (predict_positive_from_uniforms), so it is positive exactly when the person holds. Household predictors reuse the was_wealth definitions; tenure is the private-renter flag on both sides (DVPriRntR8 == 1; tenure_type RENT_PRIVATELY), not the engine is_renting. WAS counts LISAs inside gross financial wealth, so a household total above its gross_financial_wealth draw is scaled down pro rata (receipted) and ownership is re-derived from the capped balance; was_wealth columns are never rewritten. Balances are April 2020 to March 2022 pounds from a Great Britain donor and are not uprated; Northern Ireland recipients are predicted from the GB model. WAS records holdings, not annual contributions, and the stage derives no contribution or bonus. HMRC''s 2024-25 Lifetime ISA accounts subscribed to (Table 9.4) anchor the holders (microcosm#1415): the ownership logit shifts up on ages 18 to 47, the ages that can hold and pay into one in 2024-25 (opened at 18 to 39 since April 2017), until the holders at those ages who survive the financial-wealth cap equal HMRC''s payers, the fewest holders consistent with them; the same keyed uniforms decide ownership, so the anchor only adds holders, and a target it cannot reach is named, not failed. Where the cap binds, an added holder scales the household''s balances down pro rata, existing holders'' included. Three later stages read the whole frame (uc_reporter_redraw, uc_take_up, pension_credit_take_up), and the Lifetime ISA counts as Universal Credit and Pension Credit capital, so the added holders can move their draws; the pair build counts the benefit units whose entitlement moves (María''s ruling of 29 October 2026).' + notes: 'Imputes Lifetime ISA holdings per adult from the WAS round-8 person tab (microcosm#1003). The donor is every non-dependent adult of the person tab joined many-to-one on CASER8 to the cleaned household tab (the was_wealth donor, same pins); the released value DVFLISAvR8 (the ONS-completed FLISAVR8_i) must sum to the household aggregate DVFLISAVR8_aggr on every household or the stage refuses the tab, and a missing or negative value is refused rather than read as a zero balance. Response classes (observed, banded with the ONS value inside the reported band, ONS-imputed value or holding, rule-impossible) are receipted and never used as predictors. The declared credibility rule recodes holders recorded in an age band starting at 45 or above to non-holders and keeps holders above the balance ceiling as holders outside the balance fit (basis on the clean operation). Ownership is a weighted logistic regression with a ridge penalty on the slopes and an unpenalised intercept, so the donor''s weighted ownership share is reproduced in-sample, on the age group, the log of person earnings, household net income (the was_wealth predictor: DVTotinc_bhcR8 on the donor and, on the recipient, the engine''s closest concept, policyengine-uk''s HBAI net income before the reset to zero, not fully like-for-like; microcosm#931), gross financial wealth, cash ISA, stocks-and-shares ISA and savings, private renting, sex and the number of children; the house regime gate (gradient boosting) is miscalibrated for an outcome held by under one per cent of adults with about 120 donor holders. Recipients aged 18 or over hold when a uniform keyed on person_id falls below their probability; no upper age limit is imposed. The balance is the regime-gated QRF fitted on the credible holders and drawn at the owner''s keyed quantile (predict_positive_from_uniforms), so it is positive exactly when the person holds. Household predictors reuse the was_wealth definitions; tenure is the private-renter flag on both sides (DVPriRntR8 == 1; tenure_type RENT_PRIVATELY), not the engine is_renting. WAS counts LISAs inside gross financial wealth, so a household total above its gross_financial_wealth draw is scaled down pro rata (receipted) and ownership is re-derived from the capped balance; was_wealth columns are never rewritten. Balances are April 2020 to March 2022 pounds from a Great Britain donor and are not uprated; Northern Ireland recipients are predicted from the GB model. WAS records holdings, not annual contributions, and the stage derives no contribution or bonus. HMRC''s 2024-25 Lifetime ISA accounts subscribed to (Table 9.4) anchor the holders (microcosm#1415): the ownership logit shifts up on ages 18 to 47, the ages that can hold and pay into one in 2024-25 (opened at 18 to 39 since April 2017), until the holders at those ages who survive the financial-wealth cap equal HMRC''s payers, the fewest holders consistent with them; the same keyed uniforms decide ownership, so the anchor only adds holders, and a target it cannot reach is named, not failed. Where the cap binds, an added holder scales the household''s balances down pro rata, existing holders'' included. Three later stages read the whole frame (uc_reporter_redraw, uc_take_up, pension_credit_take_up), and the Lifetime ISA counts as Universal Credit and Pension Credit capital, so the added holders can move their draws; the pair build counts the benefit units whose entitlement moves (María''s ruling of 29 October 2026).' - stage: regional_property_uprating survey: Public regional property reference source: MHCLG dwellings and the UK House Price Index regional average price averaged over the twelve months of calendar 2024 from the July 2026 full file, the frame's valuation basis, so the engine's annual uprating from 2024 projects the value once (microcosm#1240 fix 5). @@ -3298,7 +3863,8 @@ stages: operations: - kind: uprate_to_regional_reference resource: regional_land_values.json - factor: avg_house_price / unweighted_mean(main_residence_value | region, >0) + factor: avg_house_price / unweighted_mean(main_residence_value_rank_basis | region, FRS-base owners, main_residence_value > 0) + denominator: main_residence_value_rank_basis owner_rows: main_residence_value > 0 skip_empty_or_nonpositive_regions: true - kind: record_mass_conservation_receipt @@ -3313,7 +3879,7 @@ stages: - owned_land - other_residential_property_value - non_residential_property_value - notes: Deterministically rescales owner rows so regional unweighted owner means match the public house-price reference. Northern Ireland has no reference row and is never scaled; empty and nonpositive regions are skipped. The unweighted mean follows the incumbent behavior. The factor is read off the main residence and has always scaled the property total; since microcosm#1063 it scales the total's other components in the frame as well (land, other houses, buildings), so an owner's property_wealth stays the sum of its components after the uprating. Mortgage debt is never uprated. + notes: 'Deterministically rescales owner rows by each region''s factor: its public house-price reference over the unweighted mean of main_residence_value_rank_basis, the WAS chain''s draw of the main residence, among its FRS-base owners. was_wealth keeps that draw beside the values it reads off each England and Wales owner''s constituency, which it divides by the same factor, so the uprating returns those values to the constituency''s knots (microcosm#931); Scotland''s owners hold the draw itself, so the region''s unweighted owner mean matches the reference as before. Northern Ireland has no reference row and is never scaled; empty and nonpositive regions are skipped. The unweighted mean follows the incumbent behavior. The factor has always scaled the property total; since microcosm#1063 it scales the total''s other components in the frame as well (land, other houses, buildings), so an owner''s property_wealth stays the sum of its components after the uprating. Mortgage debt is never uprated.' - stage: high_value_home_support survey: Family Resources Survey 2024-25 and SPI synthetic support, after the WAS wealth draw and the regional property uprating source: Microcosm structural operator (microcosm#1240 fix 1); no publisher input @@ -3323,13 +3889,15 @@ stages: - kind: split_top_value_owner_households stratum_lower: 0.95 copies: 6 - stratum_rule: within each FRS region, owner households (tenure owned outright or with a mortgage, main_residence_value > 0, positive weight) ordered by main_residence_value ascending then household_id ascending; a household whose weighted mid-rank (cumulative weight before it plus half its own, over the region's owner weight) is at or above stratum_lower is selected - copy_rule: every selected household becomes copies rows at weight / copies - id_remapping: id_multiplier_for_values; copy k takes every entity id + k x multiplier + stratum_rule: within each FRS region, owner households (tenure owned outright or with a mortgage, main_residence_value > 0, positive weight) ordered by main_residence_value ascending, households tied on it in ascending order of a uniform keyed on their own household_id under seed and salt, then household_id ascending; a household whose weighted mid-rank (cumulative weight before it plus half its own, over the region's owner weight) is at or above stratum_lower is selected + seed: 0 + salt: high_value_home_support:value_tie + copy_rule: every selected household becomes copies rows at weight / copies, except where none of the 64 heaviest households left whole, tried heaviest first, tied weights in row order, can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight + id_remapping: copy-id table field high_value_support; copy k takes every entity id + k x the field's radix (10**12) flag_column: household_is_high_value_support_copy copies_column: high_value_support_copies copy_index_column: high_value_support_copy_index - values: every column copied unchanged; no draw, no seed, no salt + values: every column copied unchanged; the stratum's tie-break is the stage's only draw weight_kind_out: importance conservation: exact_total declared_factor: 1.0 @@ -3339,56 +3907,7 @@ stages: - high_value_support_copies - high_value_support_copy_index rewrites: [] - notes: 'Spine-only, deterministic, mass-conserving support split for the dense graph''s post-geography home value map (microcosm#1240). The value node reads every owner clone''s home value off its assigned constituency''s Land Registry sale distribution at the household''s weighted rank among its region''s owners, so each seat''s homes over 2 million rest on the few households whose rank is in the top of the region; this stage copies every owner household in the top five per cent of its region six ways at equal weight, each copy carrying its index in high_value_support_copy_index (0 on roots), the explicit lineage the geography identity keys on, and the value node gives each copy its own keyed rank inside the stratum. No value is assigned and no draw is made here.' -- stage: regional_home_values - survey: HM Land Registry Price Paid latest sales uprated with the UK House Price Index, owner-weighted by region (the dense value node's knots, region level) - source: uk/high_value_home_value_knots.json, built by tools/build_uk_high_value_home_stock.py from PolicyEngine/chronicle#343's registrations - grain: household - artifacts: - - role: value_knots - kind: public_aggregate_reference - resource: high_value_home_value_knots.json - format: json - runtime_sha256_required: true - operations: - - kind: assign_regional_home_values - knots: high_value_home_value_knots.json - knots_level: region - region_map: - EAST_MIDLANDS: E12000004 - EAST_OF_ENGLAND: E12000006 - LONDON: E12000007 - NORTH_EAST: E12000001 - NORTH_WEST: E12000002 - SOUTH_EAST: E12000008 - SOUTH_WEST: E12000009 - WALES: W92000004 - WEST_MIDLANDS: E12000005 - YORKSHIRE: E12000003 - rows: owner households (tenure owned outright or with a mortgage) with main_residence_value > 0 in the mapped regions - rank: weighted mid-rank of main_residence_value among the owner households of the FRS region at the spine's weights, ties broken by a keyed draw; high-value support copies take support_stratum_lower + (1 - support_stratum_lower) * draw - support_stratum_lower: 0.95 - stream: - - sha256-u53-v1 - - uk-spine-regional-home-values-v1 - - 0 - - 0 - value: np.interp(rank, grid, region knots) - property_wealth: moves by the value's change, floored at 0 - rank_basis_column: main_residence_value_rank_basis - - kind: record_mass_conservation_receipt - entity: household - reason: 'Regional home values assigned on the source spine: household weights pass through unchanged and total household mass is conserved.' - declared_factor: 1.0 - gate_coupling: The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this stage-specific reason. - outputs: - - main_residence_value_rank_basis - rewrites: - - main_residence_value - - property_wealth - nonnegative_outputs: - - main_residence_value_rank_basis - notes: 'The national line''s home values (microcosm#1112, microcosm#1095). The national release solves the bound spine, which has no area, so main_residence_value stayed the WAS-imputed value after the regional uprating, not a Land Registry value; the dense line reads values off the assigned constituency''s distribution only after areas exist (uk.full.property.value, microcosm#1240). This stage gives every England-and-Wales owner household with a value the one its region''s uprated Land Registry distribution holds at the household''s weighted mid-rank among the owner households of its FRS region, at the spine''s weights, ranked as the dense node ranks (ties broken by a keyed draw; the high-value support copies spread over the top stratum by their own draws), and moves property_wealth by the change. Scotland, Northern Ireland, renters and owners without a value keep theirs, and the council-tax band is untouched. The replaced value is kept in main_residence_value_rank_basis, which the dense node ranks on, so the dense line''s values, wealth and bands do not change; the release boundary drops it on both lines. The value reaches three national rows: obr.sdlt through the main-home purchase value, and ons.land.land_value and ons.land.household_land_value through property_wealth and the engine''s land intensity; corporate land value does not move. The region knots run up to each region''s largest sale and down to its smallest, as the dense line''s constituency knots do, so the national line places more homes in the top surcharge bands than the derived stock facts; the tails are to be fixed on both lines together (#1112, María''s ruling of 2026-10-17).' + notes: 'Spine-only, deterministic, mass-conserving support split (microcosm#1240). Within each FRS region, every owner household in the top five per cent of the region''s owner weight by main_residence_value is copied six ways at equal weight, each copy carrying its index in high_value_support_copy_index (0 on roots) and its ids minted in the copy-id table''s high_value_support field. The stage runs after the regional property uprating, so it selects on the values was_wealth reads off each England and Wales owner''s constituency, which the uprating returns to their knots (microcosm#931), and every copy keeps its parent''s value, area and geography key. The exact-total correction lands on one of the 64 heaviest households the stage leaves whole, and on a root or copy only where none of them can carry it (copy_rule). No value is assigned here; the one draw is the stratum''s tie-break, a uniform keyed on each household''s own household_id (seed, salt) that orders owners tied on value, so no copy or clone ordinal ranks highest on every tie.' - stage: property_components survey: Survey of Personal Incomes Public Use Tape 2022-23 and HMRC Property Rental Income Statistics 2026 (tax year 2024-25) source: https://www.gov.uk/government/statistics/property-rental-income-statistics @@ -3463,9 +3982,10 @@ stages: population: persons with profit from property other than sub-letting above zero, on both channels, after the finance-cost add-back band_lower_bounds: [0, 10000, 20000, 30000, 40000, 50000, 60000, 70000, 80000, 90000, 100000] band_basis: PRIS 2024-25 Table 13 (all tax entities) scaled by Table 1's individuals' share of landlords - order: profit descending, person_id ascending + order: profit descending, landlords tied on profit in ascending order of a uniform keyed on person_id under walk_seed and tie_salt, then person_id ascending walk: top band first, each band filled with whole persons to its share of the individuals-basis landlords times the spine's landlord weight (the spine's total before calibration differs from PRIS's, and filling the published counts would push every profit rank into a higher band), at one seeded systematic offset (the CGT Table 3 walk's rule); the lowest band takes every remaining landlord walk_seed: 0 + tie_salt: allocate_property_rental_income:tie within_band: linear in the weight-ranked position within the band, from its upper bound for the highest profit to its lower bound; never below the landlord's profit top_band: max(100,000, k x profit) with k at least 1, solved by bisection so the band's weighted mean receipts equal the individuals' Table 2 receipts less the bounded bands' realised mean receipts at their individuals-basis counts, over the top band's count; where the floor at k = 1 already exceeds that mean the multiplier stays 1 and the overshoot per landlord is recorded maximum_top_band_multiplier: 5.0 @@ -5589,7 +6109,7 @@ stages: rewrites: - frs_benunit_capital - would_claim_uc - notes: Runs after the last universal_credit_reported writer and before CGT cloning. Redraws every SPI benefit unit's capital, since the channel's incomes are SPI draws and the copied FRS answer no longer belongs to the unit (uk-data#495, microcosm#1095), from weighted base-FRS units with available capital in cells of UC reporter status by financial investment income band (interest, dividends and other investment income; property income is excluded because TOTCAPB4 counts accounts and assets) by is_uc_couple (two retained FRS claimant/partner roles, independently of legal is_married) by dependent-children band. A cell with fewer than 20 donors widens in the declared coarsening order, and reporter status is never coarsened. It then applies the monotone reported-UC OR refresh. The recorded capitals add property (uk-data#495, microcosm#1095), since policyengine-uk 2.177.7 replaces every capital source with a recorded figure of 0 or more and takes it as countable, and TOTCAPB4 has no property. WAS records the property for the household, and each means test counts the claimant's and partner's own capital (UC Regs 2013 reg 18; SPCA 2002 s.5). The household reference person's benefit unit therefore holds all of the household's property, and no other unit records any (María's ruling of 2026-10-12), since that person is the one in whose name the home is owned or rented, the best single reading of the owner. The engine shares household property over every claimant or partner instead, which puts a parent's property on an adult child's claim; a recorded figure consumes its own unit's share, so the engine's shares reach only a unit on the unavailable sentinel, and since policyengine-uk 2.136.1 such a unit takes its share whatever a recording unit beside it holds (the receipt counts any unit left there). The property counts at 90% of its value, market value less 10% for the costs of sale, less the debt secured on it, class by class and floored at zero per class (UC Regs 2013 reg 49(1); SPC Regs 2002 reg 19), as policyengine-uk values Universal Credit property capital. The was_wealth stage draws the debt on other residential and on non-residential property (microcosm#1095), and owned land carries none, as the engine has no input for it (pe-uk#1969). Universal Credit counts other residential and non-residential property, and Pension Credit counts those and owned land. From policyengine-uk 2.173.3 Council Tax Reduction counts the same three at the same value, from the household's own sources, except that a working-age applicant with Universal Credit takes the Universal Credit assessment. A unit that reports Universal Credit keeps its receipt and records the carrier alone for Universal Credit, since DWP assessed its capital to pay it and the property is imputed without that receipt. frs_benunit_capital stays the financial carrier, and its unavailable sentinel -1 passes through to both. Both recorded figures project to later years by GDP per head, as financial capital does, while policyengine-uk 2.139.1 projects other residential property by house prices, so the other residential property in a recorded figure grows faster than the engine's own projection of it; by the OBR indices the engine carries, it grows 1.5% faster by 2025 and 2.0% by 2026 from a 2024 base. + notes: Runs after the last universal_credit_reported writer and before CGT cloning. Redraws every SPI benefit unit's capital, since the channel's incomes are SPI draws and the copied FRS answer no longer belongs to the unit (uk-data#495, microcosm#1095), from weighted base-FRS units with available capital in cells of UC reporter status by financial investment income band (interest, dividends and other investment income; property income is excluded because TOTCAPB4 counts accounts and assets) by is_uc_couple (two retained FRS claimant/partner roles, independently of legal is_married) by dependent-children band. A cell with fewer than 20 donor survey records (a unit's geographic clones count once, every other copy as a record of its own) widens in the declared coarsening order, and reporter status is never coarsened. It then applies the monotone reported-UC OR refresh. The recorded capitals add property (uk-data#495, microcosm#1095), since policyengine-uk 2.177.7 replaces every capital source with a recorded figure of 0 or more and takes it as countable, and TOTCAPB4 has no property. WAS records the property for the household, and each means test counts the claimant's and partner's own capital (UC Regs 2013 reg 18; SPCA 2002 s.5). The household reference person's benefit unit therefore holds all of the household's property, and no other unit records any (María's ruling of 2026-10-12), since that person is the one in whose name the home is owned or rented, the best single reading of the owner. The engine shares household property over every claimant or partner instead, which puts a parent's property on an adult child's claim; a recorded figure consumes its own unit's share, so the engine's shares reach only a unit on the unavailable sentinel, and since policyengine-uk 2.136.1 such a unit takes its share whatever a recording unit beside it holds (the receipt counts any unit left there). The property counts at 90% of its value, market value less 10% for the costs of sale, less the debt secured on it, class by class and floored at zero per class (UC Regs 2013 reg 49(1); SPC Regs 2002 reg 19), as policyengine-uk values Universal Credit property capital. The was_wealth stage draws the debt on other residential and on non-residential property (microcosm#1095), and owned land carries none, as the engine has no input for it (pe-uk#1969). Universal Credit counts other residential and non-residential property, and Pension Credit counts those and owned land. From policyengine-uk 2.173.3 Council Tax Reduction counts the same three at the same value, from the household's own sources, except that a working-age applicant with Universal Credit takes the Universal Credit assessment. A unit that reports Universal Credit keeps its receipt and records the carrier alone for Universal Credit, since DWP assessed its capital to pay it and the property is imputed without that receipt. frs_benunit_capital stays the financial carrier, and its unavailable sentinel -1 passes through to both. Both recorded figures project to later years by GDP per head, as financial capital does, while policyengine-uk 2.139.1 projects other residential property by house prices, so the other residential property in a recorded figure grows faster than the engine's own projection of it; by the OBR indices the engine carries, it grows 1.5% faster by 2025 and 2.0% by 2026 from a 2024 base. - stage: uc_limited_capability survey: Family Resources Survey 2024-25 (BENEFITS.UCLCW, DWP's limited capability status on a linked Universal Credit claim) and policyengine-uk's limited capability proxy, evaluated by the pinned engine source: frs_uc_lcw_status (frs_spine) on the UC row's person, and the locked engine's uc_limited_capability_for_WRA, uc_limited_capability_for_work, is_uc_assessed_claimant and is_carer_for_benefits on the frame as it stands @@ -6008,14 +6528,16 @@ stages: - corporate_wealth - other_residential_property_value - non_residential_property_value - selection: within each income band walk households in investable wealth descending, household_id ascending, accumulating positive household weight until the cumulative weight reaches the band's support mass; whole households; an exhausted pool is recorded, not refused + selection: within each income band walk households in investable wealth descending, households tied on it in ascending order of a uniform keyed on their own household_id under seed and salt, then household_id ascending, accumulating positive household weight until the cumulative weight reaches the band's support mass; whole households; an exhausted pool is recorded, not refused + seed: 0 + salt: cgt_support_split:wealth_tie maximum_copy_weight: 60.0 - copy_rule: copies = ceil(household weight / maximum_copy_weight); every copy carries weight / copies - id_remapping: id_multiplier_for_values; copy k takes every entity id + k x multiplier + copy_rule: copies = ceil(geographic clone count x household weight / maximum_copy_weight), the quotient snapped to the nearest integer within clone-count ulps when the clone count exceeds one; every copy carries weight / copies, except where none of the 64 heaviest households left whole, tried heaviest first, tied weights in row order, can carry the exact-total correction, when the shared fallback can put it on a root or copy, off weight / copies by about one ulp of the total weight; of those 64, a selected household left at one copy can carry the correction only where its corrected weight keeps its one copy and the walk's selection, a check the fallback does not make + id_remapping: copy-id table field cgt_support_split; copy k takes every entity id + k x the field's radix (10**13), k up to 9,999, and a household needing more copies is refused flag_column: household_is_cgt_support_copy copies_column: cgt_support_copies copy_index_column: cgt_support_copy_index - values: every column copied unchanged; no draw, no seed, no salt + values: every column copied unchanged; the selection's tie-break is the stage's only draw weight_kind_out: importance conservation: exact_total declared_factor: 1.0 @@ -6025,7 +6547,7 @@ stages: - cgt_support_copies - cgt_support_copy_index rewrites: [] - notes: 'Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 before the clone (30 after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. No value is assigned and no draw is made; the clone stage gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact).' + notes: 'Spine-only, deterministic, mass-conserving support split before the incidence clone (microcosm#1045). The Table 3 redraw owns every gain amount and places rows by wealth rank, so the top gain bands need light rows where the walk already places heavy ones: within each Table 3 income column the wealthiest households, up to twice the headroom times the published count of gainers at or above GBP 250,000 in that column, are split into copies of at most 60 divided by the geographic clone count before the incidence clone (half that after), each copy the same household at an equal share of its weight and carrying its index in cgt_support_copy_index (0 on roots), the explicit lineage the geography identity keys on. The exact-total correction lands on one of the 64 heaviest households the split leaves whole, and on a root or copy only where none of them can carry it (copy_rule). The shared fallback does not make the one-copy check copy_rule names, so where none of the 64 can carry the correction, the fallback can put it on a one-copy household the check passed over, and the identity tool''s selection recompute can then count that household as a family the stage never made, which the bracketed family test does not report, or miss one it made, as under the rule before. No value is assigned, and the one draw is the walk''s tie-break, a uniform keyed on each household''s own household_id (seed, salt) that orders households tied on wealth, so no copy or clone ordinal is taken first on every tie; the incidence clone stage (cgt_incidence_clone) gives every copy its own prior. This replaces the HMRC Table 2.1a band-donor stack, whose 30 heavy rows per band the redraw demoted below the exempt amount (the projection-entrants fence artefact).' - stage: cgt_incidence_clone survey: Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence source: Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465 @@ -6046,7 +6568,7 @@ stages: mass_split: 0.5 weight_kind_out: importance conservation: exact_total - id_remapping: id_multiplier_for_values + id_remapping: copy-id table field cgt_incidence_clone; the twin takes every entity id + the field's radix (10**17) declared_factor: 1.0 reason: Capital-gains incidence clone splits every household's mass equally across original and clone records; total household mass is conserved. - kind: draw_capital_gains_prior_from_banded_quantiles @@ -6184,7 +6706,7 @@ stages: reporting_grain: Table 6 age bands and the twelve region-tier areas - kind: rank_preserving_allocation within: income band x age group x region group cell, then the income band's pooled unassigned gainers - ordering: 'a rank key descending, person_id ascending on ties: among gainers, the household investable-wealth rank times the wealth rank weight plus the prior-gain rank times its complement, both as average ranks over the gainers' + ordering: 'a rank key descending, then the rank-tie uniform ascending, then person_id ascending: among gainers, the household investable-wealth rank times the wealth rank weight plus the prior-gain rank times its complement, both as average ranks over the gainers; the rank-tie uniform is one identity-keyed uniform per person, keyed on person_id under seed_base and salted hmrc_cgt_gains_spine:rank_tie:, which every walk, plan and the remainder read' wealth_rank_weight: 0.75 investable_wealth_columns: - gross_financial_wealth @@ -6208,7 +6730,7 @@ stages: mean_repair_reason: Published counts round to the nearest thousand and amounts to the nearest million; two cells of the 2024-25 table (gains of GBP 1m to 2m at taxable incomes of GBP 37,700 to 49,999 and GBP 100,000 to 125,139) imply a mean below their own band, and repaired means clamp just inside the violated boundary. bottom_band_floor: annual exempt amount plus one pound seed_base: 552 - seed_mixing: seed combined with the build period; each walk draws its rounding offset in cell order then pooled-fallback order, then each plan draws its quantile jitter, income bands ascending and gain bands highest first + seed_mixing: 'seed combined with the build period, which salts every draw; each walk''s rounding offset and each plan''s quantile jitter is an identity-keyed uniform on its own label, with no stream order, so none depends on which other walks run or skip: a cell walk keys on cell::: and an income band''s pooled walk on pool:, salted hmrc_cgt_gains_spine:offset:, and a plan on plan::, salted hmrc_cgt_gains_spine:jitter:, bands and age groups by their lower bounds and region groups by their labels' deterministic: true cell_means: ratios of the published Table 3 2024-25 cell gains to cell taxpayers, both rounded by the publisher (counts to the nearest thousand, amounts to the nearest million), so derived rather than published numbers; suppressed-count cells take the band-total mean; every mean is repaired into its band - kind: sub_aea_remainder @@ -6218,8 +6740,8 @@ stages: resource_vintage: 2017-18 nominal pounds as published (Advani and Summers 2020, CAGE WP 465, Table A1); no uprating is applied banding: Advani-Summers total-income bands at the person's component-sum income without allowance subtraction, the banding the prior draw uses range: quantiles strictly between the band's degree-1 quantile spline zero crossing and its crossing of the build period's annual exempt amount, so every amount lies in (0, AEA] - mapping: remainder gainers take the midpoints of the weight-proportional quantile strata that partition the range, evaluated on the band's spline; zero-weight persons take a nominal stratum of the band's smallest positive weight and carry no mass; no seeded draw is consumed, so the walk and plan draw order is unchanged - ordering: existing gains ascending, person_id descending on ties (the prior-gain ranking reversed; the allocation and the within-band draws rank on the wealth-blended key) + mapping: remainder gainers take the midpoints of the weight-proportional quantile strata that partition the range, evaluated on the band's spline; zero-weight persons take a nominal stratum of the band's smallest positive weight and carry no mass; the mapping makes no draw of its own + ordering: existing gains ascending, then the rank-tie uniform descending, then person_id descending (the prior-gain ranking reversed; the allocation and the within-band draws rank on the wealth-blended key) quantile_points: - 0.05 - 0.1 @@ -6271,8 +6793,8 @@ stages: operations: - kind: pair_clone_households_to_originals clone_flag_column: household_is_capital_gains_clone - id_remapping: id_multiplier_for_values - pairing: clone household_id minus the clone stage's id multiplier, recomputed from the pre-clone rows (clone flag false; the support split's copies are pre-clone rows with their own clones) + id_remapping: copy-id table field cgt_incidence_clone; the twin takes every entity id + the field's radix (10**17) + pairing: clone household_id minus the cgt_incidence_clone field's declared radix (the support split's copies are pre-clone rows with their own clones) requirement: bijection between clone and original households; an unpaired clone or original fails the build - kind: derive_reporter_composition_from_banded_quantiles resource: advani_summers_capital_gains_distribution.json @@ -6363,7 +6885,7 @@ stages: realization: 'probability carried as weight, no draw: a household carrying k liable gainers becomes 2^k arms, one per subset of residential gainers, at the household''s weight times the product of p over the residential gainers and 1 - p over the others; the arm with no residential gainer keeps the household''s ids; at design weights the residential count and gains equal the solved expectations sum(p w) and sum(p w g) in every gain band' maximum_liable_gainers_per_household: 3 gainer_order: person_id ascending within the household - id_remapping: id_multiplier_for_values; arm j takes every entity id + j x multiplier, bit m of j marking the household's m-th liable gainer residential + id_remapping: copy-id table field cgt_residential_split, the top field; arm j takes every entity id + j x the field's radix (10**18), bit m of j marking the household's m-th liable gainer residential flag_column: household_is_cgt_residential_clone index_column: cgt_residential_clone_index probability_column: cgt_residential_probability @@ -6383,7 +6905,7 @@ stages: nonnegative_outputs: - cgt_residential_probability - capital_gains_residential_property - notes: 'Structural stage after the incidence anchor and before the asset-type stage (microcosm#1063). The residential-property flag used to be realised by a weighted systematic walk, one whole flag per row, whose gains error is bounded by one row''s stake: at the top of the gains distribution one stake is several times a band''s expected residential gains, so the realised gains sat short of expectation by the lottery expectation of the giant stakes or over it by one of them. This stage solves the same logistic (intercept to the Table 8a 2024-25 taxpayers on the individuals basis, slope to the gains, with the stock shift) and carries the probability as weight instead: every household with liable gainers becomes 2^k arms at product weights, one per subset of residential gainers, the arm with no residential gainer keeping the household''s ids and arm j taking every entity id plus j times the id multiplier, the lineage rule the executor, the geography identity kernel and the identity tool read for the support split and the clone. At design weights the residential count and gains equal the solved expectations in every gain band, with no draw, no offset and no seed; the calibration, which binds hmrc.cgt.residential_property_taxpayers and hmrc.cgt.residential_property_gains, decides between the arms afterwards as it does between the incidence clone''s halves. Every household''s mass and composition and the total household mass are conserved; the receipt is gated at stage time.' + notes: 'Structural stage after the incidence anchor and before the asset-type stage (microcosm#1063). The residential-property flag used to be realised by a weighted systematic walk, one whole flag per row, whose gains error is bounded by one row''s stake: at the top of the gains distribution one stake is several times a band''s expected residential gains, so the realised gains sat short of expectation by the lottery expectation of the giant stakes or over it by one of them. This stage solves the same logistic (intercept to the Table 8a 2024-25 taxpayers on the individuals basis, slope to the gains, with the stock shift) and carries the probability as weight instead: every household with liable gainers becomes 2^k arms at product weights, one per subset of residential gainers, the arm with no residential gainer keeping the household''s ids and arm j taking every entity id plus j times the radix of the copy-id table''s top field (microcosm#931), the lineage rule the executor and the identity tool read for the support split and the clone. At design weights the residential count and gains equal the solved expectations in every gain band, with no draw, no offset and no seed; the calibration, which binds hmrc.cgt.residential_property_taxpayers and hmrc.cgt.residential_property_gains, decides between the arms afterwards as it does between the incidence clone''s halves. Every household''s mass and composition and the total household mass are conserved; the receipt is gated at stage time.' - stage: hmrc_cgt_asset_type_spine survey: HMRC Capital Gains Tax statistics 2026 release, Table 8 (UK residential property disposals, 2024-25, administrative), Table 4.1 (individuals claiming Business Asset Disposal Relief or Investors' Relief by band of qualifying gain, 2024-25, administrative) and Table 7 (disposals, proceeds and gains by asset type, 2023-24, sample-based), vendored from the pinned Chronicle feed source: https://www.gov.uk/government/statistics/capital-gains-tax-statistics @@ -6435,7 +6957,7 @@ stages: qualifying_amount: below the lifetime limit a claimant's whole net gain qualifies; in the open top band the qualifying gain is the lifetime limit top_band_count: the published top-band qualifying gains divided by the lifetime limit; the published count is rounded to the nearest thousand basis_assumption: Table 4.1 counts qualifying gains before losses and the annual exempt amount while the frame carries net gains after losses; Investors' Relief is merged with BADR, as in the engine's capital_gains_badr input; the whole-gain assumption reproduces Table 4.1's tax column to within half a percent - realization: weighted systematic sampling within each band in ascending (gain, person_id) order with one seeded offset per band, drawn in band order whether or not the band carries a target + realization: weighted systematic sampling within each band in ascending gain order, persons tied on gain in ascending order of a uniform keyed on person_id under seed and salted assign_badr_qualifying_gains:tie:, then person_id ascending, with one identity-keyed offset per band, keyed on its bounds (|, open for the top band) and salted assign_badr_qualifying_gains:offset:, with no stream order, so a band's offset depends neither on which bands carry a target nor on which other bands the facts declare seed: 555 seed_mixing: seed combined with the build period policy_parameters: @@ -6472,7 +6994,7 @@ stages: weight_fitting: 'one fit per claimant status: multiplicative updates of the type weights until the household-weighted expected gains shares by type match that status''s share targets; a fit refuses if it has not converged within the iteration limit, and the stage refuses up front when claimants hold more of the gains than the claimant categories'' share' weight_fitting_iterations: 200 weight_fitting_tolerance: 1.0e-06 - realization: 'error diffusion per claimant status in descending (gain, person_id) order: each person adds weight x gain x type probability to a running balance per type and takes the type it may hold with the largest balance, which is drawn down by the whole weight x gain, so the realised gains by type track the fitted composition instead of following the heaviest rows; deterministic, no seed' + realization: 'error diffusion per claimant status in descending gain order, persons tied on gain in ascending order of a uniform keyed on person_id under the assign_badr_qualifying_gains seed and salted assign_main_asset_type:tie:, then person_id ascending: each person adds weight x gain x type probability to a running balance per type and takes the type it may hold with the largest balance, which is drawn down by the whole weight x gain, so the realised gains by type track the fitted composition instead of following the heaviest rows; that tie-break is its only draw' output_column: capital_gains_asset_type value_domain: - none @@ -6577,7 +7099,7 @@ stages: candidates: a bonus (frs_bonus_amount, or an SPI-channel adult's drawn bonus) > 0 and employment_income at least the anchor's annual upper earnings limit, after the salary conversion move: the bonus, at most move_pay_share_cap of employment_income, to pension_contributions_via_bonus_sacrifice; employment_income unchanged, since a sacrificed bonus is never received move_pay_share_cap: 0.5 - spi_bonus_draw: an SPI-channel adult with pay draws the bonus share of pay (frs_bonus_amount over employment_income) of a base-FRS adult with pay in the same pay cell, by person weight, keyed on person_id; the cells are the base-FRS paid adults' weighted pay quantiles at spi_cell_quantiles, and a cell with fewer than spi_min_donor_sources donor source households joins the one below + spi_bonus_draw: an SPI-channel adult with pay draws the bonus share of pay (frs_bonus_amount over employment_income) of a base-FRS adult with pay in the same pay cell and the same geographic clone (household_clone_index; a spine without it is one clone), by person weight, keyed on person_id; the cells are the base-FRS paid adults' weighted pay quantiles over every clone at spi_cell_quantiles, and a cell with fewer than spi_min_donor_sources donor source households joins the one below, and the lowest the one above spi_cell_quantiles: - 0.1 - 0.2 @@ -6613,7 +7135,7 @@ stages: - pension_contributions_via_salary_sacrifice - employee_pension_contributions - employment_income - notes: 'The QRF trains only on the 2024-25 FRS asked subset with pay. Its sign gate is cross-fitted (gate_calibration: cross_fitted; microcosm#1359), as lcfs_consumption''s are: it is calibrated on the donors its fold gates left out, so a sign is drawn at its donor rate on people the gate was not fitted to, with each household''s members in one fold. Salary sacrifice is pay given up, so every amount, reported or predicted, is limited to the pay the person still receives, and a person without pay has none (uk-data#518, microcosm#1095); the limit bites above all on SPI-channel rows, which keep their FRS twin''s answer beside pay redrawn from the SPI. The second arm creates support toward the reviewed 5.4m staging target by moving contributors'' pension amounts, up to half their pay, into salary sacrifice and lowering their pay by the same amount, since the engine reads employment_income after sacrifice; what does not move stays an employee contribution, so the sacrifice never exceeds the pay that remains. Before microcosm#1069 c9 the pay stayed whole and the converted record lost its net-pay relief. The third arm adds bonus sacrifice beside pay toward the OBR''s 2027-28 base above GBP 2,000 (microcosm#1340). A sacrificed bonus is never received, so the FRS records it in neither pay nor its bonus amounts, and pay stays as it is (María''s ruling of 2026-10-24 on Vahid''s review of microcosm#1383); a received FRS bonus marks a bonus-paying job and its size. An SPI-channel adult''s FRS bonus is its twin''s, beside the tape''s pay, so it draws a bonus share of pay from a base-FRS adult in its pay cell instead.' + notes: 'The QRF trains only on the 2024-25 FRS asked subset with pay. Its sign gate is cross-fitted (gate_calibration: cross_fitted; microcosm#1359), as lcfs_consumption''s are: it is calibrated on the donors its fold gates left out, so a sign is drawn at its donor rate on people the gate was not fitted to, with each household''s members in one fold. Salary sacrifice is pay given up, so every amount, reported or predicted, is limited to the pay the person still receives, and a person without pay has none (uk-data#518, microcosm#1095); the limit bites above all on SPI-channel rows, which keep their FRS twin''s answer beside pay redrawn from the SPI. The second arm creates support toward the reviewed 5.4m staging target by moving contributors'' pension amounts, up to half their pay, into salary sacrifice and lowering their pay by the same amount, since the engine reads employment_income after sacrifice; what does not move stays an employee contribution, so the sacrifice never exceeds the pay that remains. Before microcosm#1069 c9 the pay stayed whole and the converted record lost its net-pay relief. The third arm adds bonus sacrifice beside pay toward the OBR''s 2027-28 base above GBP 2,000 (microcosm#1340). A sacrificed bonus is never received, so the FRS records it in neither pay nor its bonus amounts, and pay stays as it is (María''s ruling of 2026-10-24 on Vahid''s review of microcosm#1383); a received FRS bonus marks a bonus-paying job and its size. An SPI-channel adult''s FRS bonus is its twin''s, beside the tape''s pay, so it draws a bonus share of pay from a base-FRS adult in its pay cell, in its own geographic clone, instead.' - stage: carers_allowance_take_up survey: Family Resources Survey 2024-25 source: 'Independent Review of Carer''s Allowance Overpayments (July 2025), chapter 7, and DWP benefit statistics, through the UK take-up contract''s carers_allowance_newly_entitled rate, applied to PolicyEngine-UK''s Carer''s Allowance earnings test after the SPI income chain.' @@ -6793,10 +7315,10 @@ stages: - would_claim_targeted_childcare - maximum_extended_childcare_hours_usage - max_free_entitlement_hours_used - notes: Runs after the last copy stage and before private_school_attendance, whose net-income ranking counts the childcare entitlements and Tax-Free Childcare. frs_take_up draws the childcare flags and the extended-hours usage on FRS benefit units before any copy exists, so every SPI support copy, income-band donor, high-value home support copy, CGT support copy, CGT incidence clone and residential arm inherited its host's draws, and a cell with few hosts fixed the take-up of all their copies. This stage re-draws would_claim_tfc, would_claim_extended_childcare, would_claim_universal_childcare (nested under the extended flag) and would_claim_targeted_childcare at the contract rates, and maximum_extended_childcare_hours_usage from the contract's clipped normal, for the copies only, with frs_take_up's own helper, seed 0 and output-name salts, keyed on the copy's own benunit_id. A copy's structural id is its source id plus a multiple of an offset above every id in the frame, so it is unique, differs from every native id and gives the copy streams independent of its host's; a rebuild of the same frame draws the same values. Native benefit units keep their frs_take_up draws. The anchored Child Benefit, Pension Credit and Universal Credit flags and would_claim_uc_childcare are left as they are. It also re-draws the copies' children's max_free_entitlement_hours_used, the under-2 usage, against their own unit's re-drawn working-parent flag, keyed on their own person_id. + notes: Runs after the last copy stage and before private_school_attendance, whose net-income ranking counts the childcare entitlements and Tax-Free Childcare. frs_take_up draws the childcare flags and the extended-hours usage on FRS benefit units, each geographic clone's on its own ids (microcosm#931), before any other copy exists, so every SPI support copy, income-band donor, high-value home support copy, CGT support copy, CGT incidence clone and residential arm inherited its host's draws, and a cell with few hosts fixed the take-up of all their copies. This stage re-draws would_claim_tfc, would_claim_extended_childcare, would_claim_universal_childcare (nested under the extended flag) and would_claim_targeted_childcare at the contract rates, and maximum_extended_childcare_hours_usage from the contract's clipped normal, for the copies only, with frs_take_up's own helper, seed 0 and output-name salts, keyed on the copy's own benunit_id. A copy's structural id is its source id plus a multiple of an offset above every id in the frame, so it is unique, differs from every native id and gives the copy streams independent of its host's; a rebuild of the same frame draws the same values. Native benefit units, every geographic clone's included, keep their frs_take_up draws. The anchored Child Benefit, Pension Credit and Universal Credit flags and would_claim_uc_childcare are left as they are. It also re-draws the copies' children's max_free_entitlement_hours_used, the under-2 usage, against their own unit's re-drawn working-parent flag, keyed on their own person_id. - stage: private_school_attendance survey: policyengine-uk private-school attendance imputation, stored at the spine's weights - source: policyengine-uk attends_private_school (the gov.simulation.private_school_vat attendance rates and factor), evaluated by the pinned engine + source: policyengine-uk attends_private_school (the gov.simulation.private_school_vat attendance rates and factor) over every person in the frame, from the pinned engine's values and parameters grain: person artifacts: - role: frs_release @@ -6817,7 +7339,7 @@ stages: gate_coupling: The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this stage-specific reason. outputs: - attends_private_school - notes: 'The last spine stage before the ISA subscriptions (uk-data#568, microcosm#1095). policyengine-uk imputes attends_private_school with a formula that ranks households by net income at their weights and compares a percentile rate, scaled to the national pupil count, with the stored attends_private_school_random_draw. Left to the formula, attendance is re-imputed in every simulation, so a reform to income tax, National Insurance or benefits switches pupils in or out, a population under one million people of weight imputes none (pe-uk#1905), and the calibration and the release read different flags once the weights change. This stage materializes the engine''s attendance once, at the calibration year and the spine''s design weights, and stores it, as the root stage stores months_since_last_birthday, so the solve and the release read the same flags; no stage after it copies or re-keys a person. Stored attendance no longer responds to income changes or to reforms of the attendance parameters, and a reform to the fee VAT does not move pupils either way in the pinned engine.' + notes: 'The last spine stage before the ISA subscriptions (uk-data#568, microcosm#1095). policyengine-uk imputes attends_private_school with a formula that ranks households by net income at their weights and compares a percentile rate, scaled to the national pupil count, with the stored attends_private_school_random_draw. Left to the formula, attendance is re-imputed in every simulation, so a reform to income tax, National Insurance or benefits switches pupils in or out, a population under one million people of weight imputes none (pe-uk#1905), and the calibration and the release read different flags once the weights change. This stage computes the formula''s attendance once, over the whole frame at the calibration year and the spine''s design weights, and stores it, as the root stage stores months_since_last_birthday, so the solve and the release read the same flags; no stage after it copies or re-keys a person. The formula ranks the people a simulation holds, so a simulation over part of the frame would rank that part alone (microcosm#931). The stage therefore also restates the formula''s last step over every person in the frame, from the engine''s own values at the calibration year, each decided by a household''s own records, and its own parameters: household market income plus benefits less the members'' income tax and National Insurance, each household''s weight once per member, microdf''s weighted percentile rank (the rank the formula calls) over the people of positive weight, the attendance rates and population factor, nobody while those weights sum below one million, and the stored draw for each under-18. Where the engine computes the frame in one simulation (the national line, and a spine of one geographic clone), the stage stores the engine''s flags once the restatement equals them bit for bit, so every such build re-checks it; where the households carry more than one clone, the engine computes the frame clone by clone and the stage stores the restatement over the whole pool. The receipt names the route and whether the engine''s flags were checked. Stored attendance no longer responds to income changes or to reforms of the attendance parameters, and a reform to the fee VAT does not move pupils either way in the pinned engine.' - stage: isa_subscriptions survey: HMRC Annual savings statistics (September 2026 release); Wealth and Assets Survey round 8 source: HMRC Individual Savings Account statistics Tables 9.4, 9.6 and 9.7 to 9.9, the cash ISA limit reduction impact note and Lifetime ISA Table 1a, via the vendored facts; Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed round-8 person and household tabs (interviews April 2020 to March 2022, Great Britain). @@ -6952,7 +7474,7 @@ stages: - cash_isa_subscriptions - stocks_and_shares_isa_subscriptions - lifetime_isa_payments - notes: 'Imputes ISA subscriptions per adult by type, and the per-person ISA balances, on the final spine (microcosm#1415). Each household''s was_wealth cash and stocks-and-shares ISA totals are split across its adults by the holding pattern of a WAS round-8 donor household with the same number of adults, so person balances sum exactly to the household totals, which are unchanged. Subscribers in 2023-24 are adults aged 18 or over in households holding the type, raked by class (cash only, stocks and shares only, both) to Table 9.7''s income bands, Table 9.8''s age and sex and Table 9.9''s regions and selected by keyed walks, the both class first. Amounts follow Table 9.7''s subscription bands within each income band and class, uniform inside a band and exactly GBP 20,000 at the top; the both class splits each person''s amount by their balance shares, shifted so the class''s stocks-and-shares total is Table 9.4''s 2023-24 stocks-and-shares amount less the stocks-and-shares-only payments. People grow to 2024-25 by Table 9.4''s account growth (an upper bound on people, as from 6 April 2024 a person may pay into several ISAs of one type), and amounts scale to Table 9.4''s 2024-25 amounts per type, holding subscribers already at the GBP 20,000 overall limit and moving combined amounts between GBP 18,000 (Table 9.7''s last band floor below the limit) and the limit on a line up to it, so a factor below one leaves no empty range under the limit (María''s rulings of 29 October 2026): Table 9.7''s cash bands sit above Table 9.4, and growing that excess by Table 9.4''s ratio against the limit pushed the subscribers below it far up. Every Lifetime ISA holder aged 18 to 47 (the ages that can hold one in 2024-25) with room under the overall limit pays in, amounts in (0, GBP 4,000] drawn to Table 9.4''s 2024-25 Lifetime ISA amount. No ISA fact is a calibration target: binding the cells the stage draws to would be circular, so the reviewed fence keeps them measured only.' + notes: 'Imputes ISA subscriptions per adult by type, and the per-person ISA balances, on the final spine (microcosm#1415). Each household''s was_wealth cash and stocks-and-shares ISA totals are split across its adults by the holding pattern of a WAS round-8 donor household with the same number of adults, so person balances sum exactly to the household totals, which are unchanged. Subscribers in 2023-24 are adults aged 18 or over in households holding the type, raked by class (cash only, stocks and shares only, both) to Table 9.7''s income bands, Table 9.8''s age and sex and Table 9.9''s regions and selected by keyed walks, the both class first. Amounts follow Table 9.7''s subscription bands within each income band and class, uniform inside a band and exactly GBP 20,000 at the top; the both class splits each person''s amount by their balance shares, shifted so the class''s stocks-and-shares total is Table 9.4''s 2023-24 stocks-and-shares amount less the stocks-and-shares-only payments. Amounts go in the order of a key that blends each subscriber''s weighted rank on the class''s balance (cash, stocks and shares, or the two together) with a keyed uniform at balance_rank_weight; tied balances share one weighted mid-rank, so the keyed uniform alone orders them. People grow to 2024-25 by Table 9.4''s account growth (an upper bound on people, as from 6 April 2024 a person may pay into several ISAs of one type), and amounts scale to Table 9.4''s 2024-25 amounts per type, holding subscribers already at the GBP 20,000 overall limit and moving combined amounts between GBP 18,000 (Table 9.7''s last band floor below the limit) and the limit on a line up to it, so a factor below one leaves no empty range under the limit (María''s rulings of 29 October 2026): Table 9.7''s cash bands sit above Table 9.4, and growing that excess by Table 9.4''s ratio against the limit pushed the subscribers below it far up. Every Lifetime ISA holder aged 18 to 47 (the ages that can hold one in 2024-25) with room under the overall limit pays in, amounts in (0, GBP 4,000] drawn to Table 9.4''s 2024-25 Lifetime ISA amount and ordered by the same blend, on the Lifetime ISA balance over every holder aged 18 to 47, with tied balances again sharing one mid-rank; the Lifetime ISA operation declares no rank weight and reuses the amounts operation''s balance_rank_weight. No ISA fact is a calibration target: binding the cells the stage draws to would be circular, so the reviewed fence keeps them measured only.' - stage: pip_points survey: DWP's Pathways to Work evidence pack, chapter 2, Tables 2.9 and 2.10 (PIP points by age group, England and Wales, end of January 2025), on the spine's stored PIP awards source: pip_dl_category and pip_m_category (frs_disability, and hmrc_spi_income_spine for the SPI rows) and the stored age; dwp_pip_points_facts.json, vendored from the pinned feed diff --git a/packages/microcosm-build/src/microcosm/build/uk/take_up_contract.json b/packages/microcosm-build/src/microcosm/build/uk/take_up_contract.json index 7192c0905b..96ad35ff45 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/take_up_contract.json +++ b/packages/microcosm-build/src/microcosm/build/uk/take_up_contract.json @@ -2,7 +2,7 @@ "version": 1, "country": "uk", "build_year": 2024, - "policy": "UK stochastic assignment contract for the FRS 2024-25 source spine. Date-keyed values use the incumbent parameter loader semantics: choose the latest value with year less than or equal to the build year. Reported-receipt anchors use an unweighted full-population target denominator, target = int(rate * n_units), and accept anchor overshoot. Random streams are intentionally identity-keyed by seed, output variable name, and entity id; this is a signed difference from the incumbent UK pipeline sequential RNG stream. Incumbent provenance: ebf733c.", + "policy": "UK stochastic assignment contract for the FRS 2024-25 source spine. Date-keyed values use the incumbent parameter loader semantics: choose the latest value with year less than or equal to the build year. Reported-receipt anchors use an unweighted full-population target denominator, target = int(rate * n_units), counted within each geographic clone by frs_take_up's anchored draws (spi_benefit_coherence's SPI-channel Universal Credit redraw still counts over every clone's units at once), and accept anchor overshoot. Random streams are intentionally identity-keyed by seed, output variable name, and entity id; this is a signed difference from the incumbent UK pipeline sequential RNG stream. Incumbent provenance: ebf733c.", "incumbent_provenance": { "commit": "ebf733c", "description": "incumbent UK pipeline stochastic layer", diff --git a/packages/microcosm-build/src/microcosm/build/uk/uk_data_target_parity.json b/packages/microcosm-build/src/microcosm/build/uk/uk_data_target_parity.json index d124bd49af..2144b309b9 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/uk_data_target_parity.json +++ b/packages/microcosm-build/src/microcosm/build/uk/uk_data_target_parity.json @@ -1183,7 +1183,7 @@ "covers": [ "policyengine_uk_data.targets.sources.ons_land_values" ], - "evidence": "uk-data targets/sources/ons_land_values.py and _land.py; Microcosm target ids ons.land.{land_value,household_land_value,corporate_land_value}.", + "evidence": "uk-data targets/sources/ons_land_values.py and _land.py; Microcosm target ids ons.land.{land_value,household_land_value,corporate_land_value}, of which ons.land.land_value and ons.land.corporate_land_value are measure exclusions tracking microcosm#1521 since the #931 move.", "status": "ported_national" }, { diff --git a/packages/microcosm-build/src/microcosm/build/uk/uk_population_targets.json b/packages/microcosm-build/src/microcosm/build/uk/uk_population_targets.json index 132571e893..9fa27c846b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/uk_population_targets.json +++ b/packages/microcosm-build/src/microcosm/build/uk/uk_population_targets.json @@ -16800,7 +16800,7 @@ "policyengine": { "metric_name": "ons/land_value", "value_variable": "land_value", - "notes": "National balance sheet land series for the total_economy sector; uk-data carries the last estimate flat to later years (populace-side declaration, uk-data targets/sources/_land.py:48-52 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." + "notes": "National balance sheet land series for the total_economy sector; uk-data carries the last estimate flat to later years (populace-side declaration, uk-data targets/sources/_land.py:48-52 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3). Held as a measured diagnostic (calibration_measure_exclusions.json, María's ruling of 2026-11-06, microcosm#1521 workstream 2): E44R also counts the land of financial corporations, government and NPISH, which neither ons.land.household_land_value (E44N) nor ons.land.corporate_land_value (E43B) covers, and land_value is the sum of the two columns those rows read, so the three rows could not all be met." }, "axiom": { "metric_name": "ons/land_value", @@ -16834,7 +16834,7 @@ "policyengine": { "metric_name": "ons/household_land_value", "value_variable": "household_land_value", - "notes": "National balance sheet land series for the households_and_npish sector; uk-data carries the last estimate flat to later years (populace-side declaration, uk-data targets/sources/_land.py:54-59 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." + "notes": "National balance sheet land series for the households sector (ONS E44N, households alone: NPISH land, like financial corporations' and government's, counts only in the total-economy row, ons.land.land_value); uk-data carries the last estimate flat to later years (populace-side declaration, uk-data targets/sources/_land.py:54-59 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "ons/household_land_value", @@ -16868,7 +16868,7 @@ "policyengine": { "metric_name": "ons/corporate_land_value", "value_variable": "corporate_land_value", - "notes": "National balance sheet land series for the non_financial_corporations sector; uk-data carries the last estimate flat to later years (populace-side declaration, uk-data targets/sources/_land.py:60-65 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." + "notes": "National balance sheet land series for the non_financial_corporations sector; uk-data carries the last estimate flat to later years (populace-side declaration, uk-data targets/sources/_land.py:60-65 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3). Held as a measured diagnostic (calibration_measure_exclusions.json, María's ruling of 2026-11-06, microcosm#1521 workstream 2): policyengine-uk's corporate_land_value spreads the engine's own aggregate by each household's share of the weighted corporate-sector wealth, so the column meets that aggregate at any weights and, bound, the row pins corporate-wealth mass, not land." }, "axiom": { "metric_name": "ons/corporate_land_value", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py index 90411f338a..e16b066d6c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py @@ -402,10 +402,6 @@ validate_uk_national_frame, write_uk_national_frame, ) -from microcosm.build.uk_runtime.national_sampling import ( - sample_uk_spine_frame, - uk_spine_source_family_units, -) from microcosm.build.uk_runtime.oa_ladder_sources import ( LADDER_OA_COLUMNS, assemble_uk_oa_ladder, @@ -623,7 +619,6 @@ "rowwise_calibration_mass_reason", "runtime_provenance", "rotated_uk_local_holdout", - "sample_uk_spine_frame", "uk_cgt_annual_exempt_amount", "UK_CGT_REQUIRED_COLUMNS", "UK_CGT_TARGET_COVERAGE_REQUIREMENTS", @@ -937,7 +932,6 @@ "uk_calibration_diagnostics_payload", "uk_fit_by_family", "uk_support_limited_misses", - "uk_spine_source_family_units", "uk_frame_content_identity", "uk_dsa_policy", "uk_dwp_disability_category_rates", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/annuity_trust_column.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/annuity_trust_column.py new file mode 100644 index 0000000000..32c0ba4fa1 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/annuity_trust_column.py @@ -0,0 +1,25 @@ +"""The name of the UK spine's annuity and trust income descriptor. + +FRS rows keep the payments of an annuity not bought with pension funds and of +a trust or covenant in private_pension_income, where policyengine-uk's means +tests, household income and tax read them. SPI Table 3.7 counts both as other +income (note 6), so the spine stores the part they make of +private_pension_income apart, under this name: a calibration-only descriptor +that the Table 3.7 other-income measure adds (microcosm#1428 review, finding +1). frs_spine writes it, the SPI income stage zeros it on the rows it draws, +and measure_simulation's resolver reads it. + +The resolver reads zeros where a frame lacks the column, so a renamed constant +would change the measures without re-keying them, and every node whose code +reads the name binds this module. It holds the name alone, nothing that +computes, so the dense measures and the national problem key on the name +without keying on frs_spine, every edit to which would otherwise re-solve the +local and the national line on a bound spine (microcosm#931, ruled 9 Nov +(late)). +""" + +from __future__ import annotations + +#: The descriptor: each person's annual annuity and trust payments, already +#: inside private_pension_income. No engine input reads it. +ANNUITY_AND_TRUST_INCOME_COLUMN = "annuity_and_trust_income" diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/area_home_values.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/area_home_values.py new file mode 100644 index 0000000000..7b58ae0d9a --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/area_home_values.py @@ -0,0 +1,546 @@ +"""The WAS chain's home values off each owner's constituency, G8 (microcosm#931). + +Form F (C-15). After segment 2 of the WAS chain, each England and Wales owner +with a positive drawn main residence ``v2`` reads its assigned constituency's +knots (``high_value_home_value_knots.json``) at its weighted mid-rank within +its FRS region, divided by the factor ``regional_property_uprating`` computes +on ``v2``; that stage, declaring the kept ``v2`` +(``main_residence_value_rank_basis``) its denominator, returns it to the knot +value to one rounding, and a home read at the knots' cap to the cap or below. +A region without a factor keeps the knot value, as that stage leaves it. Every other row keeps ``v2`` bit for bit, and the release +boundary drops the kept column. Ties break on household-id keys over a fixed +stream with seed source NONE (C-43). The evidence counts clone 0's distinct +true sources, rounded or banded, with no value, sum or factor (C-16, C-44); +the module's own refusals name columns and declared constants only. It reads +the areas the spine's area stages assign; the packaged WAS stage declares it, +and the regional property uprating the kept draw as its denominator +(:func:`area_home_values_declared`), in place of the retired +``regional_home_values`` stage (microcosm#931). +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping +from dataclasses import dataclass, field +from importlib.resources import files +from types import MappingProxyType +from typing import Any + +import numpy as np +import pandas as pd + +from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.uk_runtime.geographic_clones import ( + geographic_clone_count, + ladder_clone_index_column, +) +from microcosm.build.uk_runtime.property_localisation import ( + RANK_BASIS_COLUMN, + ValueKnots, + _england_wales, + load_value_knots, + localise_main_residence_values, + owner_mask, + owner_uniforms, +) +from microcosm.build.uk_runtime.receipt_counts import ( + distinct_count, + rounded_count, + source_household_ids, + support_band, +) +from microcosm.build.uk_runtime.regional_uprating import ( + UK_REGIONAL_LAND_VALUES_RESOURCE, + regional_reference_factors, +) +from microcosm.build.uk_runtime.rowwise_geography import ( + AREA_TYPE_TO_CROSSWALK_COLUMN, + FRS_REGION_TO_REGION_CODE, +) +from microcosm.build.uk_runtime.spi_support import support_channel_column +from microcosm.frame import Frame + +__all__ = [ + "AREA_HOME_VALUES_EXEMPT_COLUMNS", + "AREA_HOME_VALUES_INPUTS", + "AREA_HOME_VALUES_OPERATION_KIND", + "AREA_HOME_VALUES_STREAM", + "FRS_REGION_TO_KNOTS_REGION", + "KNOTS_ARTIFACT_ROLE", + "KNOTS_LEVEL", + "KNOTS_RESOURCE", + "METHOD_VERSION", + "REGIONAL_REFERENCE_ARTIFACT_ROLE", + "UKAreaHomeValues", + "UKAreaHomeValuesLocaliser", + "UKAreaHomeValuesResources", + "area_home_values_declared", + "area_home_values_operation_parameters", + "clone_zero_rows", + "load_area_home_values_resources", + "load_committed_area_home_values_resources", + "localise_area_home_values", + "support_clip_exemptions", +] + +#: The operation the WAS stage declares to localise its home values. +AREA_HOME_VALUES_OPERATION_KIND = "localise_main_residence_value" +#: The keyed stream the rank's tie-breaks draw on: seed source NONE and a +#: fixed stream, as ruled for ``regional_home_values`` on 9 Nov (C-43). It is +#: this step's own; the stage's operation declares it, so its contract hash +#: covers it. +AREA_HOME_VALUES_STREAM = ( + "sha256-u53-v1", + "uk-spine-area-home-values-v1", + 0, + 0, +) +METHOD_VERSION = "uk-spine-area-home-values-v1" +#: The two artifacts the WAS stage declares for this step (role, resource): +#: the constituency knots and the regional reference the factor reads. +KNOTS_ARTIFACT_ROLE = "value_knots" +KNOTS_RESOURCE = "high_value_home_value_knots.json" +REGIONAL_REFERENCE_ARTIFACT_ROLE = "regional_reference" +_ARTIFACTS = ( + (KNOTS_ARTIFACT_ROLE, KNOTS_RESOURCE), + (REGIONAL_REFERENCE_ARTIFACT_ROLE, UK_REGIONAL_LAND_VALUES_RESOURCE), +) +#: The knots' level the values are read at. +KNOTS_LEVEL = "constituency" +#: The knots' region codes for the FRS regions they cover. Wales is the +#: country code the knots use, not the placeholder region code the area +#: assignment keeps for it (``W99999999``). The evidence counts by these +#: codes, which the home-value ranking gate declares. +FRS_REGION_TO_KNOTS_REGION: Mapping[str, str] = MappingProxyType( + { + **{ + region: code + for region, code in FRS_REGION_TO_REGION_CODE.items() + if code.startswith("E12") + }, + "WALES": "W92000004", + } +) +#: The WAS columns the support clip leaves to this step: a localised home can +#: pass the donor's largest main residence, and the property total with it +#: (C-18). The WAS stage's support clip declares them exempt. +AREA_HOME_VALUES_EXEMPT_COLUMNS = ("main_residence_value", "property_wealth") +_CONSTITUENCY_COLUMN = AREA_TYPE_TO_CROSSWALK_COLUMN["constituency"] +_CHANNEL_COLUMN = support_channel_column("household") +#: The household columns the step reads, besides ``source_household_id``, +#: the true source its evidence counts by, the support channel the factor's +#: mean leaves SPI rows out of, and the clone index the evidence counts clone +#: 0 by, where the frame carries one. +AREA_HOME_VALUES_INPUTS = ( + "household_id", + "region", + "tenure_type", + _CONSTITUENCY_COLUMN, +) +_CLONE_INDEX_COLUMN = ladder_clone_index_column("household") +_WHAT = "The area home values" + + +def area_home_values_operation_parameters() -> dict[str, object]: + """The reviewed operation the WAS stage must declare verbatim. + + The draw it reads and where, the rows it rewrites, the rank and its + stream, the knots and their level, the value and the factor it is divided + by, the region map its evidence counts by, the rank basis it keeps and the + rows it leaves. + """ + + return { + "value_column": "main_residence_value", + "after_segment": 2, + "rows": ( + "owner households (tenure owned outright or with a mortgage) with a " + "positive drawn main_residence_value in the FRS regions of England " + "and Wales" + ), + "rank": ( + "weighted mid-rank of the drawn main_residence_value among the owner " + "households of the FRS region with a positive draw, both support " + "channels, at the stage's household weights, ties broken by a draw " + "keyed on the household id" + ), + "stream": list(AREA_HOME_VALUES_STREAM), + "knots": KNOTS_RESOURCE, + "knots_level": KNOTS_LEVEL, + "constituency_column": _CONSTITUENCY_COLUMN, + "regional_reference": UK_REGIONAL_LAND_VALUES_RESOURCE, + "value": ( + "np.interp(rank, grid, the assigned constituency's knots) divided by " + "the FRS region's factor, a home at the knots' cap taking the largest " + "quotient the factor multiplies back to the cap or below" + ), + "factor": ( + "avg_house_price / unweighted_mean(drawn main_residence_value | " + "region, FRS-base owners, >0), the regional property uprating's " + "factor on the draw" + ), + "region_map": dict(sorted(FRS_REGION_TO_KNOTS_REGION.items())), + "rank_basis_column": RANK_BASIS_COLUMN, + "kept": ( + "Scotland, Northern Ireland, renters and owners without a positive " + "draw keep the draw" + ), + "method": METHOD_VERSION, + } + + +def support_clip_exemptions(stage: SourceStageSpec) -> frozenset[str]: + """The columns the WAS stage's ``support_clip`` declares exempt. + + The declared ``exempt`` list, as ``etb_services.support_clip_exempt`` + reads its stage's: none, unless the stage declares the area home values, + whose two columns :func:`area_home_values_declared` holds it to. + """ + + exempt: set[str] = set() + for operation in stage.operations: + if operation.kind == "support_clip": + declared = operation.parameters.get("exempt") + if declared: + exempt |= {str(column) for column in declared} + return frozenset(exempt) + + +def area_home_values_declared(stage: SourceStageSpec) -> bool: + """Whether the WAS stage declares this step, held to its reviewed form. + + Declared, the stage carries :data:`AREA_HOME_VALUES_OPERATION_KIND` once, + with :func:`area_home_values_operation_parameters` verbatim; the knots and + the regional reference as artifacts, each with + ``runtime_sha256_required``; ``main_residence_value_rank_basis`` among its + outputs; and its support clip exempting exactly + :data:`AREA_HOME_VALUES_EXEMPT_COLUMNS`. Undeclared, it carries neither + the output nor a support clip exemption. Anything between is refused, + since a half declaration would clip a localised home back to the donor's + range or drop the rank basis the regional uprating divides by. + """ + + operations = [ + operation + for operation in stage.operations + if operation.kind == AREA_HOME_VALUES_OPERATION_KIND + ] + exempt = support_clip_exemptions(stage) + if not operations: + if RANK_BASIS_COLUMN in stage.outputs or exempt: + raise ValueError( + f"The WAS stage declares {RANK_BASIS_COLUMN!r} or a support clip " + f"exemption without {AREA_HOME_VALUES_OPERATION_KIND!r}; both " + "belong to the area home values (microcosm#931)." + ) + return False + if len(operations) != 1: + raise ValueError( + f"The WAS stage declares {AREA_HOME_VALUES_OPERATION_KIND!r} more " + "than once." + ) + if dict(operations[0].parameters) != area_home_values_operation_parameters(): + raise ValueError( + f"The WAS stage's {AREA_HOME_VALUES_OPERATION_KIND!r} differs from the " + "reviewed operation area_home_values states (microcosm#931)." + ) + declared = sorted( + (str(artifact.get("role")), str(artifact.get("resource"))) + for artifact in stage.artifacts + if artifact.get("runtime_sha256_required") is True + and (str(artifact.get("role")), str(artifact.get("resource"))) in _ARTIFACTS + ) + if declared != sorted(_ARTIFACTS): + raise ValueError( + f"The area home values read the artifacts {list(_ARTIFACTS)} (role, " + "resource), each once with runtime_sha256_required: true." + ) + if RANK_BASIS_COLUMN not in stage.outputs: + raise ValueError( + f"The area home values keep the draw as {RANK_BASIS_COLUMN!r}, which " + "the WAS stage must declare among its outputs." + ) + if exempt != set(AREA_HOME_VALUES_EXEMPT_COLUMNS): + raise ValueError( + "The area home values need the WAS support clip to exempt exactly " + f"{list(AREA_HOME_VALUES_EXEMPT_COLUMNS)}, which a localised home " + "can lift past the donor's range (C-18)." + ) + return True + + +@dataclass(frozen=True) +class UKAreaHomeValuesResources: + """The constituency knots and the regional reference, parsed, with their + bytes' SHA-256.""" + + knots: ValueKnots = field(repr=False) + regional_reference: Mapping[str, Any] = field(repr=False) + knots_sha256: str + regional_reference_sha256: str + + +def load_area_home_values_resources( + *, knots: bytes, regional_reference: bytes +) -> UKAreaHomeValuesResources: + """The two resources from their bytes: the knots through the property + loader, the regional reference as the regional uprating reads it.""" + + return UKAreaHomeValuesResources( + knots=load_value_knots(knots), + regional_reference=json.loads(regional_reference.decode("utf-8")), + knots_sha256=hashlib.sha256(knots).hexdigest(), + regional_reference_sha256=hashlib.sha256(regional_reference).hexdigest(), + ) + + +def load_committed_area_home_values_resources() -> UKAreaHomeValuesResources: + """The committed knots and regional reference the stage declares.""" + + package = files("microcosm.build.uk") + return load_area_home_values_resources( + knots=package.joinpath(KNOTS_RESOURCE).read_bytes(), + regional_reference=package.joinpath( + UK_REGIONAL_LAND_VALUES_RESOURCE + ).read_bytes(), + ) + + +@dataclass(frozen=True) +class UKAreaHomeValues: + """The step on one household table: every row's value and rank basis, the + rows rewritten, and what the evidence counts among clone-0 rows. The + source counts are exact and print only rounded or banded; nothing here + prints in the repr but the clone count, the knots' basis year and the + digests.""" + + value: np.ndarray = field(repr=False) + rank_basis: np.ndarray = field(repr=False) + rewritten: np.ndarray = field(repr=False) + clone_count: int + sources_rewritten_by_region: Mapping[str, int] = field(repr=False) + sources_at_cap: int = field(repr=False) + ranking: str = field(repr=False) + knots_basis_year: int + knots_sha256: str + regional_reference_sha256: str + + def evidence(self) -> dict[str, object]: + """The step's evidence block (C-16): rounded and banded counts of + clone-0 sources, digests and declared constants; no value, sum or + factor.""" + + return { + "clone_count": int(self.clone_count), + "counted_among": "clone_0", + "sources_rewritten_by_region": { + region: rounded_count(count) + for region, count in sorted(self.sources_rewritten_by_region.items()) + }, + "sources_at_cap": support_band(self.sources_at_cap), + "knots_basis_year": int(self.knots_basis_year), + "knots_sha256": self.knots_sha256, + "knots_level": KNOTS_LEVEL, + "regional_reference_sha256": self.regional_reference_sha256, + "ranking": {"rank_basis": self.ranking}, + "rank_basis_column": RANK_BASIS_COLUMN, + "stream": list(AREA_HOME_VALUES_STREAM), + "method": METHOD_VERSION, + } + + +def _require_columns(household: pd.DataFrame) -> None: + missing = [ + column + for column in (*AREA_HOME_VALUES_INPUTS, "source_household_id") + if column not in household.columns + ] + if missing: + raise ValueError( + f"{_WHAT} read household {missing}, which the earlier spine stages " + "write; the frame lacks them (microcosm#931)." + ) + + +def clone_zero_rows(household: pd.DataFrame) -> np.ndarray: + """Whether each household row is geographic clone 0's: its + ``household_clone_index`` is 0, or the table carries no index (one + clone). A count taken among these rows does not grow with the clone count + (C-44); the WAS stage's donor-range cap counts by it too.""" + + if _CLONE_INDEX_COLUMN not in household.columns: + return np.ones(len(household), dtype=bool) + return household[_CLONE_INDEX_COLUMN].to_numpy(dtype=np.int64) == 0 + + +def localise_area_home_values( + household: pd.DataFrame, + weights: np.ndarray, + value: np.ndarray, + *, + resources: UKAreaHomeValuesResources, +) -> UKAreaHomeValues: + """Every household row's main residence, England and Wales owners' + localised (G8, form F). + + ``value`` is the WAS chain's segment-2 draw in ``household``'s row order, + ``weights`` the stage's household weights in that order. Each England and + Wales owner with a positive draw reads its assigned constituency's knots + at its weighted mid-rank within its FRS region, divided by its region's + factor on the draw; a home read at the knots' cap takes the largest + quotient the factor multiplies back to the cap or below. Every other row + keeps its draw bit for bit. A row's + value depends on its own key and its region's pooled ranks and factor, + never on the table's row order. + + The evidence counts a source among clone 0's rows of positive weight: a + row of zero weight carries no localised value into a weighted figure + (``receipt_counts.distinct_count``, microcosm#1262). + + Refused, by name: a missing input column; weights or draws that do not + align or are not finite; an England and Wales owner with a positive draw + whose ``constituency_code`` is not in England and Wales, or another owner + with a positive draw whose code is; a constituency without knots; a + localised value that is not positive; and the refusals of the rank and the + source reader. + """ + + _require_columns(household) + drawn = np.asarray(value, dtype=float) + weight = np.asarray(weights, dtype=float) + if drawn.shape != (len(household),) or weight.shape != (len(household),): + raise ValueError( + f"{_WHAT} read one drawn value and one weight per household row." + ) + if not np.isfinite(drawn).all(): + raise ValueError(f"{_WHAT} read finite drawn main residence values.") + sources = source_household_ids(household, stage=_WHAT) + region = household["region"].astype(str).to_numpy() + tenure = household["tenure_type"].to_numpy() + codes = np.asarray(household[_CONSTITUENCY_COLUMN].to_numpy(dtype=object)).astype( + str + ) + in_england_wales = np.isin(region, list(FRS_REGION_TO_KNOTS_REGION)) + owners = owner_mask(tenure) & (drawn > 0) + if not np.array_equal(owners & in_england_wales, owners & _england_wales(codes)): + raise ValueError( + f"{_WHAT} read an owner's {_CONSTITUENCY_COLUMN!r} in its FRS region's " + "nation: every owner with a drawn home value in England and Wales " + "needs a constituency there, and no other owner may hold one; the " + "area draw places each household in its own FRS region " + "(microcosm#931)." + ) + rank, receipt = owner_uniforms( + spine_key=household["household_id"].astype(str).to_numpy(dtype=object), + region=region, + tenure_type=tenure, + value=drawn, + weight=weight, + stream=AREA_HOME_VALUES_STREAM, + ) + knot_value, rewritten = localise_main_residence_values( + u=rank, + constituency_code=codes, + tenure_type=tenure, + value=drawn, + knots=resources.knots, + ) + if np.any(knot_value[rewritten] <= 0): + raise ValueError( + f"{_WHAT} read a knot value that is not positive: the regional " + "uprating's owners are the rows with a positive main residence, so " + "every owner the step ranks must keep one (microcosm#931)." + ) + factor_table = household.loc[ + :, [column for column in ("region", _CHANNEL_COLUMN) if column in household] + ].copy() + factor_table["main_residence_value"] = drawn + factors = regional_reference_factors(factor_table, resources.regional_reference) + localised = knot_value.copy() + at_knots_cap = knot_value >= resources.knots.cap + for name, factor in factors.items(): + rows = rewritten & (region == name) + quotient = knot_value[rows] / factor + # The regional uprating multiplies the quotient back by this factor. + # A home read at the knots' cap takes the largest quotient whose + # product does not pass the cap, so the uprating never returns it one + # rounding above uk_support's price-paid bound (microcosm#931). + capped = at_knots_cap[rows] + over = capped & (quotient * factor > knot_value[rows]) + while over.any(): + quotient[over] = np.nextafter(quotient[over], 0.0) + over = capped & (quotient * factor > knot_value[rows]) + localised[rows] = quotient + if not np.isfinite(localised).all() or np.any(localised[rewritten] <= 0): + raise ValueError(f"{_WHAT} must be finite and positive where they localise.") + counted = clone_zero_rows(household) + knots_region = np.asarray( + [FRS_REGION_TO_KNOTS_REGION.get(name, "") for name in region], dtype=object + ) + rewritten_by_region = { + str(code): distinct_count( + sources, rewritten & counted & (knots_region == code), weight + ) + for code in sorted(set(knots_region[rewritten & counted])) + } + at_cap = distinct_count( + sources, rewritten & counted & (knot_value >= resources.knots.cap), weight + ) + return UKAreaHomeValues( + value=localised, + rank_basis=drawn.copy(), + rewritten=rewritten, + clone_count=geographic_clone_count(household, stage=_WHAT), + sources_rewritten_by_region=MappingProxyType(rewritten_by_region), + sources_at_cap=int(at_cap), + ranking=str(receipt["rank_basis"]), + knots_basis_year=int(resources.knots.basis_year), + knots_sha256=resources.knots_sha256, + regional_reference_sha256=resources.regional_reference_sha256, + ) + + +@dataclass(frozen=True) +class UKAreaHomeValuesLocaliser: + """The step bound to one WAS stage frame, for ``impute_was_wealth``. + + Built from the frame the WAS stage runs on, before its chain draws: + the household columns the step reads and the stage's household weights. + Called with the segment-2 draw, it returns + :func:`localise_area_home_values`' result. + """ + + household: pd.DataFrame = field(repr=False) + weights: np.ndarray = field(repr=False) + resources: UKAreaHomeValuesResources = field(repr=False) + + @classmethod + def from_frame( + cls, frame: Frame, *, resources: UKAreaHomeValuesResources + ) -> UKAreaHomeValuesLocaliser: + if not isinstance(frame, Frame): + raise TypeError(f"{_WHAT} read the households of a Frame.") + household = frame.table("household") + _require_columns(household) + columns = [ + column + for column in ( + *AREA_HOME_VALUES_INPUTS, + "source_household_id", + _CHANNEL_COLUMN, + _CLONE_INDEX_COLUMN, + ) + if column in household.columns + ] + return cls( + household=household.loc[:, columns].copy(), + weights=np.asarray(frame.weights_for("household").values, dtype=float), + resources=resources, + ) + + def __call__(self, value: np.ndarray) -> UKAreaHomeValues: + return localise_area_home_values( + self.household, self.weights, value, resources=self.resources + ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_area_conditioning.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_area_conditioning.py new file mode 100644 index 0000000000..1488f3568c --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_area_conditioning.py @@ -0,0 +1,682 @@ +"""The UK's conditioned area draw: its definition, inputs and fallback ladder (microcosm#931). + +G2 of the clone-first plan draws each household's atomic area within its FRS +region, conditioned on what the FRS records about the real address (O14 iv +and v): the address's IMD decile within its nation, and the household's +council-tax band where the survey gives it a band its region uses. The shared +operator takes both (:mod:`microcosm.build.atomic_geography`: a code +constraint that may be missing, and ``weight_by``, a weight column chosen per +household), and the shared chooser +(:func:`~microcosm.build.atomic_support_conditioning.choose_fallback_levels`) +picks the level each household is drawn at. This module declares the UK's use +of both; it runs no stage: + +- :func:`uk_conditioned_assignment_definition` is the operator's declaration + over the three conditioned supports. It is today's region-only declaration + (``atomic_area_support.uk_atomic_assignment_definition``) with the + conditioned sources; the region (required) and the transient decile input + (``required: false``) as constraints; England and Wales and Scotland + sampling by the household's band weight, or by households where it has no + band; Northern Ireland, which has domestic rates, by households; the ten + ladder columns as its only layers; and the spine's own stream. +- :data:`UK_ATOMIC_FALLBACK_LEVELS` is the ladder, in the plan's order: L0 + the decile and the band's weight, L1 the decile and households, L2 the + region and the band's weight, L3 the region and households. Northern + Ireland, with no band weights, takes L1 and L3 only. +- :func:`uk_draw_inputs` is the narrow transient frame the draw reads: each + household's geography key and region, its decile as "1" to "10" (missing + for 0, no observed decile) and its band where ``council_tax_band_observed`` + (missing otherwise). Rows the SPI support channel copied get both inputs + missing, so they draw at L3, the level the SPI-copy area stage draws them + at, though a copy keeps its parent's decile and band flag (C-5, C-10). +- :func:`uk_draw_levels` gives each household its level: the first whose + inputs are present and whose cell has mass in the weight it samples by + there, the chooser's rule, with the chooser's arguments read from the + definition itself, so the chooser and the draw agree. +- :func:`uk_draw_frame` sets every input a household's level drops to + missing, so one definition serves every level. +- :func:`uk_fallback_reasons` says, for the evidence, why each household is + drawn where it is. + +The module is its own, not part of :mod:`atomic_area_support`, whose hash the +national problem, the dense measures, the inspection export and the geography +identity bind: it imports that module's constants and two of its helpers and +leaves it as it is (C-28). + +Refusals give columns and declared constants only, never a key, an id, a code +of a household or a count. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from numbers import Integral +from types import MappingProxyType + +import numpy as np +import pandas as pd + +from microcosm.build.atomic_geography import ( + AtomicSupport, + decode_atomic_support, + validate_assignment_spec, +) +from microcosm.build.atomic_support_conditioning import ( + FallbackLevel, + choose_fallback_levels, +) +from microcosm.build.source_lineage import copy_paths +from microcosm.build.uk_runtime.atomic_area_support import ( + ASSIGNMENT_COLUMNS, + BAND_WEIGHT_COLUMNS, + CONDITIONED_SOURCES, + IMD_DECILE_COLUMN, + IMD_DECILES, + SYSTEMS, + _profile, + uk_conditioned_support_base, +) +from microcosm.build.uk_runtime.council_tax_constants import UK_COUNCIL_TAX_BANDS +from microcosm.build.uk_runtime.geographic_clones import ( + GEOGRAPHY_HOUSEHOLD_KEY_COLUMN, +) +from microcosm.build.uk_runtime.household_lineage import UK_COPY_LINEAGE + +__all__ = [ + "BAND_INPUT_COLUMN", + "COUNCIL_TAX_BAND_COLUMN", + "COUNCIL_TAX_BAND_OBSERVED_COLUMN", + "DECILE_INPUT_COLUMN", + "DEFAULT_WEIGHT_COLUMN", + "DRAW_INPUT_COLUMNS", + "IDENTITY_COLUMN", + "REGION_COLUMN", + "SUPPORT_CHANNEL_BRANCH", + "UK_ATOMIC_FALLBACK_LEVELS", + "UK_DRAW_CONSTRAINTS", + "UK_FALLBACK_STATUSES", + "UK_SPINE_AREA_COLUMNS", + "UK_SPINE_ATOMIC_AREA_DRAW_STREAM", + "UK_SPINE_ATOMIC_AREA_SEED", + "UK_SPINE_ATOMIC_AREA_STREAM", + "UK_SPINE_LADDER_COLUMNS", + "UK_SPINE_LAYERS", + "UNOBSERVED_IMD_DECILE", + "uk_band_weight_map", + "uk_conditioned_assignment_definition", + "uk_conditioned_supports", + "uk_draw_frame", + "uk_draw_inputs", + "uk_draw_levels", + "uk_draw_systems", + "uk_fallback_reasons", + "uk_support_channel_copies", + "uk_system_fallback_levels", +] + +#: The key every draw is keyed on: each clone's geography key, which the +#: spine's geographic_identity stage writes and the SPI-copy area stage +#: rewrites on copies (named in the geographic_clones leaf; +#: ``atomic_area_support.IDENTITY_COLUMN`` is the same name). +IDENTITY_COLUMN = GEOGRAPHY_HOUSEHOLD_KEY_COLUMN +#: The spine's own stream for the area draw, and its seed: 0, declared by the +#: stage (C-3), one constant for both lines, so clone 0 nests at every K. +UK_SPINE_ATOMIC_AREA_STREAM = "uk-spine-atomic-area-v1" +UK_SPINE_ATOMIC_AREA_SEED = 0 +_STREAM_ALGORITHM = "sha256-u53-v1" +#: The definition's stream at that seed, the draw's ``stream`` entry +#: (:func:`uk_conditioned_assignment_definition`), which a spine's geography +#: binding records with the seed at its index 3 (microcosm#931). +UK_SPINE_ATOMIC_AREA_DRAW_STREAM = ( + _STREAM_ALGORITHM, + UK_SPINE_ATOMIC_AREA_STREAM, + 0, + UK_SPINE_ATOMIC_AREA_SEED, +) + +#: The household columns the inputs are read from. The band flag and the +#: unobserved decile are frs_spine's (``FRS_COUNCIL_TAX_BAND_OBSERVED_COLUMN`` +#: and ``FRS_IMD_DECILE_UNOBSERVED``), spelled here so that an edit to that +#: module does not re-key the area stages; a test holds the spellings equal. +REGION_COLUMN = "region" +COUNCIL_TAX_BAND_COLUMN = "council_tax_band" +COUNCIL_TAX_BAND_OBSERVED_COLUMN = "council_tax_band_observed" +UNOBSERVED_IMD_DECILE = 0 +#: The transient inputs the draw reads: never stage outputs, they live on the +#: narrow frame :func:`uk_draw_inputs` builds and go with it. +DECILE_INPUT_COLUMN = "atomic_draw_imd_decile" +BAND_INPUT_COLUMN = "atomic_draw_council_tax_band" +DRAW_INPUT_COLUMNS = ( + IDENTITY_COLUMN, + REGION_COLUMN, + DECILE_INPUT_COLUMN, + BAND_INPUT_COLUMN, +) +#: Each constraint input and the support code column it must match. +UK_DRAW_CONSTRAINTS: Mapping[str, str] = MappingProxyType( + {REGION_COLUMN: "frs_region", DECILE_INPUT_COLUMN: IMD_DECILE_COLUMN} +) +#: The weight a household samples by where it has no band, or its level +#: drops it. +DEFAULT_WEIGHT_COLUMN = "households" +#: The lineage branch whose copies draw without inputs: the SPI support +#: channel, which copies households and makes the income band donors. +SUPPORT_CHANNEL_BRANCH = "spi_support_channel" + +#: The ten ladder code columns the draw writes: the geography ladder's columns +#: without the engine's ``local_authority``, which the local-authority stage +#: resolves from ``local_authority_code``. Spelled here, as +#: ``geography_ladder.UK_GEOGRAPHY_LADDER_COLUMNS`` orders them, so that an +#: edit to that module does not re-key the area stages; a test holds them +#: equal. +UK_SPINE_LADDER_COLUMNS = ( + "oa_code", + "lsoa_code", + "msoa_code", + "local_authority_code", + "ward_code", + "constituency_code", + "region_code", + "itl3_code", + "itl2_code", + "itl1_code", +) +#: Each layer's support column and output: the atomic area itself for +#: ``oa_code``, the column of the same name otherwise. The nation-native +#: aliases (``atomic_area_support.UK_NATIVE_ALIAS_COLUMNS``) are left out: +#: they are missing outside their own nation and dropped at the export +#: boundary. +UK_SPINE_LAYERS = tuple( + ("area" if column == "oa_code" else column, column) + for column in UK_SPINE_LADDER_COLUMNS +) +#: Every column the draw writes: the three assignment columns, then the ladder. +UK_SPINE_AREA_COLUMNS = (*ASSIGNMENT_COLUMNS, *UK_SPINE_LADDER_COLUMNS) + +#: The fallback ladder, in the plan's order. A household takes the first level +#: whose inputs are present and whose cell has mass in the weight it samples +#: by there; a level reads every input it does not keep as missing. +UK_ATOMIC_FALLBACK_LEVELS = ( + FallbackLevel("L0", (REGION_COLUMN, DECILE_INPUT_COLUMN), True), + FallbackLevel("L1", (REGION_COLUMN, DECILE_INPUT_COLUMN), False), + FallbackLevel("L2", (REGION_COLUMN,), True), + FallbackLevel("L3", (REGION_COLUMN,), False), +) +_LEVELS = {level.name: level for level in UK_ATOMIC_FALLBACK_LEVELS} +_BAND_WEIGHT_PREFIX = "households_band_" +#: The statuses :func:`uk_fallback_reasons` gives each input, in order. +UK_FALLBACK_STATUSES: Mapping[str, tuple[str, ...]] = MappingProxyType( + { + "decile": ("kept", "unobserved", "dropped"), + "band": ("kept", "unobserved", "dropped", "none"), + } +) + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise ValueError("UK conditioned area draw: " + message) + + +def uk_band_weight_map(system: str) -> dict[str, str]: + """Each council-tax band the system's support weighs, and its column. + + ``{"A": "households_band_a", ...}``: A to I in England and Wales, A to H + in Scotland, none in Northern Ireland (``BAND_WEIGHT_COLUMNS``). + """ + + _require(system in SYSTEMS, "unknown area system") + columns = BAND_WEIGHT_COLUMNS[system] + mapping = { + column.removeprefix(_BAND_WEIGHT_PREFIX).upper(): column for column in columns + } + _require( + all(column.startswith(_BAND_WEIGHT_PREFIX) for column in columns) + and set(mapping) <= set(UK_COUNCIL_TAX_BANDS), + "band weight columns must name council-tax bands", + ) + return mapping + + +def uk_system_fallback_levels(system: str) -> tuple[FallbackLevel, ...]: + """The levels a system's households take: all four where the support + weighs bands, L1 and L3 in Northern Ireland, which has none.""" + + _require(system in SYSTEMS, "unknown area system") + if BAND_WEIGHT_COLUMNS[system]: + return UK_ATOMIC_FALLBACK_LEVELS + return tuple(level for level in UK_ATOMIC_FALLBACK_LEVELS if not level.weight_by) + + +def _seed(seed: object) -> int: + _require( + isinstance(seed, Integral) + and not isinstance(seed, (bool, np.bool_)) + and 0 <= int(seed) < 2**32, + "seed must be a bounded integer", + ) + return int(seed) + + +def uk_conditioned_supports(supports: Mapping[str, bytes]) -> dict[str, AtomicSupport]: + """The three conditioned supports, decoded, each held to canonical bytes. + + Each payload must be exactly what + ``atomic_area_support.assemble_uk_conditioned_area_support`` makes of its + system's canonical support, as the definition requires. + """ + + _require( + isinstance(supports, Mapping) and set(supports) == set(SYSTEMS), + "the three systems' supports are required", + ) + decoded = {} + for system in SYSTEMS: + payload = supports[system] + _require(type(payload) is bytes, "immutable support bytes required") + # Refuses bytes that are not the canonical conditioned output. + uk_conditioned_support_base(payload) + support = decode_atomic_support(payload) + _require(support.metadata["system"] == system, "support system differs") + decoded[system] = support + return decoded + + +def uk_conditioned_assignment_definition( + supports: Mapping[str, bytes], + *, + seed: int, +) -> dict: + """Declare the spine's conditioned area draw over the three supports. + + As ``atomic_area_support.uk_atomic_assignment_definition`` declares the + region-only draw, but over the conditioned supports + (``CONDITIONED_SOURCES``, canonical bytes only), with the region + (required) and :data:`DECILE_INPUT_COLUMN` (``required: false``) as + constraints, one stage at the atomic area that samples by + ``weight_by`` (input :data:`BAND_INPUT_COLUMN`, each band to its + ``households_band_*`` column, default ``households``) where the system + weighs bands and by ``households`` in Northern Ireland, the ten ladder + columns as layers (:data:`UK_SPINE_LAYERS`), and the stream + ``("sha256-u53-v1", "uk-spine-atomic-area-v1", 0, seed)``. + + A missing input is skipped (the decile) or samples by the default weight + (the band), so one definition draws every level of the ladder: a level's + draw sets the inputs it drops to missing (:func:`uk_draw_frame`). + """ + + seed = _seed(seed) + decoded = uk_conditioned_supports(supports) + systems = [] + for system in SYSTEMS: + support = decoded[system] + level, vintage, regions, _ = _profile(system) + _require( + all( + support.metadata[key] == value + for key, value in ( + ("system", system), + ("level", level), + ("code_system", "uk_gss"), + ("vintage", vintage), + ) + ), + "support system identity differs", + ) + bands = uk_band_weight_map(system) + stage = ( + { + "level": "area", + "weight_by": { + "input": BAND_INPUT_COLUMN, + "map": bands, + "default": DEFAULT_WEIGHT_COLUMN, + }, + } + if bands + else {"level": "area", "weight": DEFAULT_WEIGHT_COLUMN} + ) + columns = support.metadata["columns"] + systems.append( + { + "id": system, + "level": level, + "code_system": "uk_gss", + "vintage": vintage, + "source": CONDITIONED_SOURCES[system], + "selector": {REGION_COLUMN: list(regions)}, + "constraints": [ + { + "input": column, + "support": matched, + "required": column == REGION_COLUMN, + } + for column, matched in UK_DRAW_CONSTRAINTS.items() + ], + "observed_area": None, + "stages": [stage], + "layers": [ + { + "input": column, + "output": output, + **{ + key: columns[column][key] + for key in ("source", "vintage", "relation") + }, + } + for column, output in UK_SPINE_LAYERS + ], + } + ) + return validate_assignment_spec( + { + "version": 1, + "identity": [IDENTITY_COLUMN], + "stream": [_STREAM_ALGORITHM, UK_SPINE_ATOMIC_AREA_STREAM, 0, seed], + "outputs": dict( + zip(("area", "system", "basis"), ASSIGNMENT_COLUMNS, strict=True) + ), + "systems": systems, + } + ) + + +def uk_support_channel_copies(household: pd.DataFrame) -> np.ndarray: + """Whether each row is a copy the SPI support channel made. + + Read from the lineage columns through the shared reader on the UK's copy + lineage, never from ids or values: a row whose copy path takes the + support channel's branch, an SPI copy or an income band donor or any copy + a later stage makes of one. A table without the channel's columns has no + such row. + """ + + branch = next( + b for b in UK_COPY_LINEAGE.branches if b.name == SUPPORT_CHANNEL_BRANCH + ) + if not any(column in household.columns for column in branch.columns): + return np.zeros(len(household), dtype=bool) + _, paths = copy_paths(household, UK_COPY_LINEAGE, require=(SUPPORT_CHANNEL_BRANCH,)) + return np.fromiter( + (any(step == SUPPORT_CHANNEL_BRANCH for step, _ in path) for path in paths), + dtype=bool, + count=len(paths), + ) + + +def _strings(series: pd.Series, column: str) -> list[str]: + values = series.to_numpy(dtype=object) + _require( + not series.isna().any() + and all( + isinstance(value, str) and value and value.strip() == value + for value in values + ), + f"{column} must hold a nonempty code on every row", + ) + return [str(value) for value in values] + + +def uk_draw_inputs(household: pd.DataFrame) -> pd.DataFrame: + """The narrow transient frame the area draw reads, one row per household. + + Columns :data:`DRAW_INPUT_COLUMNS`, in the household table's row order + and index, each in the graph's string dtype (``StringDtype("python")``): + + - the geography key and the region, as the table holds them; + - :data:`DECILE_INPUT_COLUMN`: the household's ``imd_decile`` as "1" to + "10", missing where it is 0, no observed decile (O14 v); + - :data:`BAND_INPUT_COLUMN`: its ``council_tax_band`` where + ``council_tax_band_observed`` is true, missing otherwise (O14 iv). + + A row the SPI support channel copied (:func:`uk_support_channel_copies`) + gets both inputs missing, whatever its decile and flag, which it keeps + from its parent (C-5, C-10). + + Refused, by column: a missing column, a missing or blank key or region, a + decile column that is not integer or holds codes outside 0 to 10, a flag + column that is not a complete boolean, and an observed band that is not a + band A to I. + """ + + _require(isinstance(household, pd.DataFrame), "the household table is a DataFrame") + required = ( + IDENTITY_COLUMN, + REGION_COLUMN, + IMD_DECILE_COLUMN, + COUNCIL_TAX_BAND_COLUMN, + COUNCIL_TAX_BAND_OBSERVED_COLUMN, + ) + missing = [column for column in required if column not in household.columns] + _require(not missing, f"the household table lacks {missing}") + keys = _strings(household[IDENTITY_COLUMN], IDENTITY_COLUMN) + regions = _strings(household[REGION_COLUMN], REGION_COLUMN) + decile = household[IMD_DECILE_COLUMN] + _require( + pd.api.types.is_integer_dtype(decile.dtype) + and not pd.api.types.is_bool_dtype(decile.dtype) + and not decile.isna().any(), + f"{IMD_DECILE_COLUMN} must be a complete integer column", + ) + deciles = decile.to_numpy(dtype=np.int64) + _require( + bool( + np.all((deciles >= UNOBSERVED_IMD_DECILE) & (deciles <= len(IMD_DECILES))) + ), + f"{IMD_DECILE_COLUMN} holds codes outside 0 to 10", + ) + observed = household[COUNCIL_TAX_BAND_OBSERVED_COLUMN] + _require( + pd.api.types.is_bool_dtype(observed.dtype) and not observed.isna().any(), + f"{COUNCIL_TAX_BAND_OBSERVED_COLUMN} must be a complete boolean column", + ) + flags = observed.to_numpy(dtype=bool) + bands = household[COUNCIL_TAX_BAND_COLUMN].to_numpy(dtype=object) + _require( + all(band in UK_COUNCIL_TAX_BANDS for band in bands[flags]), + f"an observed {COUNCIL_TAX_BAND_COLUMN} is not a band A to I", + ) + copies = uk_support_channel_copies(household) + decile_inputs = [ + pd.NA if copy or value == UNOBSERVED_IMD_DECILE else str(int(value)) + for value, copy in zip(deciles.tolist(), copies.tolist(), strict=True) + ] + band_inputs = [ + str(band) if flag and not copy else pd.NA + for band, flag, copy in zip( + bands.tolist(), flags.tolist(), copies.tolist(), strict=True + ) + ] + string = pd.StringDtype(storage="python") + return pd.DataFrame( + { + IDENTITY_COLUMN: pd.Series(keys, index=household.index, dtype=string), + REGION_COLUMN: pd.Series(regions, index=household.index, dtype=string), + DECILE_INPUT_COLUMN: pd.Series( + decile_inputs, index=household.index, dtype=string + ), + BAND_INPUT_COLUMN: pd.Series( + band_inputs, index=household.index, dtype=string + ), + }, + index=household.index, + ) + + +def uk_draw_systems(inputs: pd.DataFrame, definition: Mapping) -> np.ndarray: + """Each row's area system, by the definition's region selectors. + + Refuses a row that no system, or more than one, selects, as the shared + operator does. + """ + + spec = validate_assignment_spec(definition) + systems = np.full(len(inputs), "", dtype=object) + claimed = np.zeros(len(inputs), dtype=np.int64) + for system in spec["systems"]: + mask = np.ones(len(inputs), dtype=bool) + for column, accepted in system["selector"].items(): + _require(column in inputs.columns, f"the draw inputs lack {column!r}") + mask &= inputs[column].isin(accepted).to_numpy(dtype=bool) + systems[mask] = system["id"] + claimed += mask + _require( + bool(np.all(claimed == 1)), + "every household must fall in exactly one area system by its region", + ) + return systems + + +def _system_rules(system: Mapping) -> tuple[dict | None, str]: + """A system's ``weight_by`` rule without its default, and the default.""" + + stages = system["stages"] + _require( + len(stages) == 1 and stages[0]["level"] == "area", + "the conditioned draw samples one stage, at the atomic area", + ) + (stage,) = stages + rule = stage.get("weight_by") + if rule is None: + return None, stage["weight"] + return {"input": rule["input"], "map": dict(rule["map"])}, rule["default"] + + +def uk_draw_levels( + inputs: pd.DataFrame, + definition: Mapping, + supports: Mapping[str, AtomicSupport], +) -> np.ndarray: + """Each household's fallback level, by the shared chooser, in row order. + + ``inputs`` is :func:`uk_draw_inputs`' frame and ``definition`` + :func:`uk_conditioned_assignment_definition`'s. Each system's households + go to the chooser with its support and the definition's own arguments: + its constraints as the input-to-support mapping, its stage's + ``weight_by`` rule without the default, and the default as + ``default_weight``; a system without ``weight_by`` (Northern Ireland) + takes its unweighted levels only. So a household's level is the first of + :data:`UK_ATOMIC_FALLBACK_LEVELS` whose inputs are present and whose cell + has mass in the weight it samples by there, and a household with neither + input, an SPI-channel row among them, takes L3. Returns level names. + """ + + spec = validate_assignment_spec(definition) + _require( + isinstance(supports, Mapping) + and set(supports) == {system["id"] for system in spec["systems"]} + and all(isinstance(support, AtomicSupport) for support in supports.values()), + "a decoded support for every system of the definition is required", + ) + systems = uk_draw_systems(inputs, spec) + names = np.full(len(inputs), "", dtype=object) + for system in spec["systems"]: + rows = np.flatnonzero(systems == system["id"]) + if not len(rows): + continue + weight_by, default = _system_rules(system) + levels = ( + UK_ATOMIC_FALLBACK_LEVELS + if weight_by is not None + else tuple( + level for level in UK_ATOMIC_FALLBACK_LEVELS if not level.weight_by + ) + ) + names[rows] = choose_fallback_levels( + supports[system["id"]], + inputs.iloc[rows], + constraint_columns={ + c["input"]: c["support"] for c in system["constraints"] + }, + weight_by=weight_by, + default_weight=default, + levels=levels, + ) + return np.asarray(names.tolist(), dtype=str) + + +def _level_names(inputs: pd.DataFrame, levels: Sequence[str]) -> list[str]: + names = [str(name) for name in np.asarray(levels, dtype=object).tolist()] + _require(len(names) == len(inputs), "one level per household") + _require(set(names) <= set(_LEVELS), "a level outside the UK fallback ladder") + return names + + +def uk_draw_frame(inputs: pd.DataFrame, levels: Sequence[str]) -> pd.DataFrame: + """The draw's inputs at each household's level. + + A copy of ``inputs`` with every input the household's level does not + keep set to missing: the decile where the level keeps the region alone, + the band where the level samples by households. The definition skips a + missing decile and samples a missing band by households, so the draw of + this frame draws each household at its level. + """ + + names = _level_names(inputs, levels) + frame = inputs.copy() + constraints = dict.fromkeys( + column for level in UK_ATOMIC_FALLBACK_LEVELS for column in level.constraints + ) + kept = { + column: np.asarray( + [column in _LEVELS[name].constraints for name in names], dtype=bool + ) + for column in constraints + } + kept[BAND_INPUT_COLUMN] = np.asarray( + [_LEVELS[name].weight_by for name in names], dtype=bool + ) + for column, keep in kept.items(): + if not keep.all(): + frame[column] = frame[column].mask(~keep) + return frame + + +def uk_fallback_reasons( + inputs: pd.DataFrame, levels: Sequence[str], definition: Mapping +) -> pd.DataFrame: + """Why each household is drawn at its level: one status per input. + + Column ``decile``: ``kept`` where the level keeps the decile, + ``unobserved`` where the input is missing, ``dropped`` where it is + present but no level that keeps it has mass. Column ``band`` likewise for + the band's weight, or ``none`` where the household's system weighs no + band (Northern Ireland). An L0 household reads ``kept`` twice. An + SPI-channel row's inputs are missing by rule, so it reads ``unobserved``. + On ``inputs``' index, in its row order. + """ + + names = _level_names(inputs, levels) + spec = validate_assignment_spec(definition) + systems = uk_draw_systems(inputs, spec) + banded = { + system["id"]: _system_rules(system)[0] is not None for system in spec["systems"] + } + decile_present = inputs[DECILE_INPUT_COLUMN].notna().to_numpy(dtype=bool) + band_present = inputs[BAND_INPUT_COLUMN].notna().to_numpy(dtype=bool) + deciles, bands = [], [] + for name, system, has_decile, has_band in zip( + names, + systems.tolist(), + decile_present.tolist(), + band_present.tolist(), + strict=True, + ): + level = _LEVELS[name] + if DECILE_INPUT_COLUMN in level.constraints: + deciles.append("kept") + else: + deciles.append("dropped" if has_decile else "unobserved") + if not banded[system]: + bands.append("none") + elif level.weight_by: + bands.append("kept") + else: + bands.append("dropped" if has_band else "unobserved") + return pd.DataFrame( + { + "decile": pd.Series(deciles, index=inputs.index, dtype=object), + "band": pd.Series(bands, index=inputs.index, dtype=object), + }, + index=inputs.index, + ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_area_support.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_area_support.py index 26a98b9eaa..40c07e958e 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_area_support.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/atomic_area_support.py @@ -22,6 +22,14 @@ validate_assignment_spec, ) from microcosm.build.atomic_support_conditioning import extend_atomic_support + +# The household key the assignment draws on. The clone-first spine's +# geographic_identity stage writes it too, so the name lives in the +# geographic_clones leaf, which that stage hashes; re-exported here, so a +# kernel that reads it here is held to the leaf (microcosm#931). +from microcosm.build.uk_runtime.geographic_clones import ( + GEOGRAPHY_HOUSEHOLD_KEY_COLUMN as IDENTITY_COLUMN, +) from microcosm.build.uk_runtime.rowwise_geography import FRS_REGION_TO_REGION_CODE from microcosm.graph.canonical import canonical_json @@ -31,7 +39,6 @@ "uk_ni_data_zone_2021", ) SOURCES = {system: system + "_support" for system in SYSTEMS} -IDENTITY_COLUMN = "geography_household_key" #: Assignment outputs the shared operators write on the household table. ASSIGNMENT_COLUMNS = ("atomic_area_code", "atomic_area_system", "atomic_area_basis") #: Nation-native aliases of derived layers; NA outside their own nation and diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py index 2f68553bf6..f97e3753f9 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/battery_bindings.py @@ -28,12 +28,13 @@ from __future__ import annotations import hashlib +import importlib import json from collections.abc import Callable, Iterable, Mapping from dataclasses import dataclass, replace from datetime import date, datetime from importlib.resources import files -from types import MappingProxyType +from types import MappingProxyType, ModuleType from typing import Any import numpy as np @@ -83,8 +84,8 @@ metric_names, ) from microcosm.build.uk_runtime.national_frame import ( - UK_RELEASE_EXPORT_DROPPED_COLUMNS, _uk_gate_surface, + uk_release_export_dropped_columns, uk_release_export_frame, ) from microcosm.build.uk_runtime.release_input_coverage import ( @@ -133,7 +134,9 @@ __all__ = [ "UK_GATE_REGISTRY", + "UK_GATE_REGISTRY_MODULES", "UKGateBinding", + "uk_gate_registry_modules", ] @@ -261,8 +264,17 @@ def _evaluate_release_input_coverage( # The required inputs are the ones the cut's reference build carries # (microcosm#1095); the artifact selector marks the gate evidence_absent # when no reference surface arrived. + # The export of the context's line: on the national line the geographic + # clone indices leave too (microcosm#931). The line is read raw and the + # export refuses one it does not know: this binding also runs the + # preflight's manifest check, and _release_line would bring + # weighted_integrity into that phase's reach. return uk_release_input_coverage_gate( - _uk_gate_surface(uk_release_export_frame(context.frame)), + _uk_gate_surface( + uk_release_export_frame( + context.frame, line=context.artifacts.get("release_line") + ) + ), engine, reference_surface=context.artifacts["reference_surface"], manifest=manifest, @@ -320,6 +332,25 @@ def _evaluate_calibration_reference_coverage( ) +#: The stage-health checks that read the phase's frame beside their stage's +#: receipt: the band donors' check recomputes a capped seating's seats from +#: the transferred phase's frame, the copy area check compares each copy with +#: its parent there, and the council-tax check holds each household's +#: reductions within its bill there (microcosm#931). Every other check reads +#: its receipt alone and runs without a frame. +_STAGE_HEALTH_FRAME_CHECKS = frozenset( + { + "spi_income_band_donor_support", + "copy_area_agreement", + "council_tax_reduction_within_bill", + } +) + + +def _stage_health_requires_frame(parameters: Mapping[str, Any]) -> bool: + return parameters.get("check") in _STAGE_HEALTH_FRAME_CHECKS + + def _evaluate_stage_health( context: EvidenceContext, parameters: Mapping[str, Any] ) -> GateResult: @@ -335,6 +366,7 @@ def _evaluate_stage_health( stage=stage, check=str(parameters["check"]), parameters=parameters, + frame=context.frame if _stage_health_requires_frame(parameters) else None, ) @@ -393,26 +425,27 @@ def _export_dropped_columns_from_spine( ) -> dict[str, Any]: """Columns the release export drops, read from the certifier's spine frame. - ``UK_RELEASE_EXPORT_DROPPED_COLUMNS`` leave at the release boundary - (microcosm#1063 c9 and the salary-sacrifice pre-conversion pay carrier), - so a gate that checks every declared stage output cannot find them on - the release candidate. The certifier passes the spine frame as the - ``spine_frame`` artifact; this returns each requested column that is an - export-dropped column present on that frame, keyed by column name. Any - other requested column, or any column when no spine frame is supplied, - is left to the caller's missing-column path. + The columns ``national_frame.uk_release_export_dropped_columns`` lists + for the context's release line leave at the release boundary + (microcosm#1063 c9 and the salary-sacrifice pre-conversion pay carrier + on every line, and on the national line the geographic clone indices of + its one-clone spine, microcosm#931), so a gate that checks every declared + stage output cannot find them on the release candidate. The certifier + passes the spine frame as the ``spine_frame`` artifact; this returns each + requested column that the context's line drops at export and that is + present on that frame, keyed by column name. Any other requested column, + or any column when no spine frame is supplied, is left to the caller's + missing-column path. """ - from microcosm.build.uk_runtime.national_frame import ( - UK_RELEASE_EXPORT_DROPPED_COLUMNS, - ) - spine_frame = context.artifacts.get("spine_frame") if spine_frame is None: return {} dropped = { column: entity - for entity, names in UK_RELEASE_EXPORT_DROPPED_COLUMNS.items() + for entity, names in uk_release_export_dropped_columns( + _release_line(context) + ).items() for column in names } found: dict[str, Any] = {} @@ -799,7 +832,10 @@ def _evaluate_enum_domain( (table,) = tables by_nation = _nation_partitioned_domains(column, parameters) if by_nation is None: - return enum_domain_gate({column: table[column]}, {column: domain}) + return _count_free_enum_domain_result( + enum_domain_gate({column: table[column]}, {column: domain}), + partitioned=False, + ) from microcosm.build.uk_runtime.council_tax_constants import UK_NATION_BY_REGION @@ -807,13 +843,54 @@ def _evaluate_enum_domain( raise ValueError(f"{column} is partitioned by region, which its table lacks.") nations = table["region"].astype(str).map(UK_NATION_BY_REGION) if nations.isna().any(): - unknown = sorted(set(table.loc[nations.isna(), "region"].astype(str))) - raise ValueError(f"{column} rows carry regions with no nation: {unknown}.") - return enum_domain_gate( - {column: table[column]}, - {column: domain}, - partitions={column: nations}, - partition_domains={column: by_nation}, + # The battery prints a raised refusal as the gate's failure, so it + # names the column, never the rows' values (C-32, microcosm#931). + raise ValueError(f"{column} rows carry regions with no nation.") + return _count_free_enum_domain_result( + enum_domain_gate( + {column: table[column]}, + {column: domain}, + partitions={column: nations}, + partition_domains={column: by_nation}, + ), + partitioned=True, + ) + + +def _count_free_enum_domain_result( + result: GateResult, *, partitioned: bool +) -> GateResult: + """The shared enum gate's verdict, rebuilt without counts or examples (C-32). + + The UK's enum gates run on frames whose rows are SPI copies and geographic + clones of survey households (microcosm#931), so the shared gate's count of + values outside the domain and its examples of them could say something of + a few records. The verdict is the shared gate's: each failure names the + column and its domain, and the details keep the columns checked, the + allowed values (and each partition's), and the columns with a value + outside the domain. ``microcosm.build.gates`` is untouched. + """ + + details = dict(result.details) + outside = sorted(str(column) for column in dict(details.get("invalid_counts", {}))) + domain = "its nation's domain" if partitioned else "the engine's enum domain" + rebuilt: dict[str, object] = { + "columns_checked": details.get("columns_checked", 0), + "allowed_values": details.get("allowed_values", {}), + } + if "partition_allowed_values" in details: + rebuilt["partition_allowed_values"] = details["partition_allowed_values"] + rebuilt["columns_outside_domain"] = outside + failures: tuple[str, ...] = () + if not result.passed: + failures = tuple( + f"{column}: a value lies outside {domain}." for column in outside + ) or (f"A value lies outside {domain}.",) + return GateResult( + name=result.name, + passed=result.passed, + failures=failures, + details=rebuilt, ) @@ -1223,8 +1300,11 @@ def _evaluate_degenerate_release_surface( now=_exclusion_clock(context), # The release boundary drops these before writing; the certifier never # sees them on the exported H5, so a gate fed the pre-export frame - # must not report them (found by the first graph dense build). - dropped_at_export=UK_RELEASE_EXPORT_DROPPED_COLUMNS, + # must not report them (found by the first graph dense build). What + # it drops is the context's line's: on the national line, the + # geographic clone indices too, constant on its one-clone spine + # (microcosm#931). + dropped_at_export=uk_release_export_dropped_columns(_release_line(context)), **kwargs, ) return _with_out_of_line(result, context, out_of_line) @@ -1826,12 +1906,13 @@ def _evaluate_input_mass_parity( reviewed_exclusions=exclusions, ) # The reference is compared against the surface the release ships: the - # export frame, with the reviewed export exclusions dropped. The release + # export frame, with the reviewed export exclusions dropped (and on the + # national line the geographic clone indices, microcosm#931). The release # cut already loads that frame; at the national seam the raw frame still # carries the spine's WAS property_wealth, which the reference also # stores, so comparing it there would call the signed exclusion stale # (microcosm#1106). - export_frame = uk_release_export_frame(context.frame) + export_frame = uk_release_export_frame(context.frame, line=_release_line(context)) result = uk_input_mass_parity_gate( uk_input_mass_totals(export_frame), reference, @@ -2132,9 +2213,12 @@ def _ledger_compile_parity_required_artifacts( "zeroed_columns", "restored_columns", "restored_within_earnings_limit_columns", - # microcosm#1063 child_benefit_take_up stage-health check. + # microcosm#1063 child_benefit_take_up stage-health check; its + # per-age standard errors from María's ruling of 2026-11-10 + # (microcosm#931). "maximum_claim_rate_deviation", "maximum_age_claim_rate_deviation", + "maximum_age_claim_rate_standard_errors", "minimum_age_child_rows", "maximum_opt_out_share_deviation", "minimum_eligible_family_units", @@ -2273,10 +2357,31 @@ def _ledger_compile_parity_required_artifacts( # uk-data#629 spi_hourly_pay check: the drawn employees' share # below the minimum wage may exceed the FRS rows' by this much. "maximum_share_above_frs", + # The spine geography checks (microcosm#931): the area home + # values' knots level, beside the ranking check's keys above; + # the mapping-integrity check's accepted outcomes, validation + # scope and support systems; and each copy layer's columns the + # copy area check holds to the copy's parent. + "knots_level", + "accepted_outcomes", + "validation_scope", + "support_systems", + "columns_by_copy_layer", + # The council_tax_reduction_within_bill check (microcosm#931): + # the bill, the household and person reduction columns it + # holds within the bill, and the regions whose households it + # leaves out; its tolerance is absolute_tolerance above. + "bill_column", + "household_reduction_columns", + "person_reduction_columns", + "excluded_regions", } ), artifact_keys=frozenset({"stage_evidence"}), needs_frame=False, + # The band donors', the copy area and the council-tax checks read the + # phase's frame (microcosm#931). + frame_predicate=_stage_health_requires_frame, evidence=_stage_health_evidence, ), "nonnegative_columns": UKGateBinding( @@ -2463,3 +2568,77 @@ def _ledger_compile_parity_required_artifacts( evidence=_ledger_compile_parity_evidence, ), } + +#: Every first-party module the registry's bindings compute through: each +#: binding's evaluator and hooks, what they call, to a fixed point, this module +#: among them. A kernel that evaluates whichever gates its node names through +#: the registry keys on them all: the national gates, whose node's gate scope +#: names their gates (microcosm#931 ruling 6). The gate batteries, whose +#: phases declare their gates, key each phase on its own gates' share of +#: these instead. test_uk_graph derives the set from the registry and holds +#: this list to it exactly. +UK_GATE_REGISTRY_MODULES: tuple[str, ...] = ( + "microcosm.build.bounded_amounts", + "microcosm.build.chronicle_epoch", + "microcosm.build.copy_agreement", + "microcosm.build.gate_battery", + "microcosm.build.gates", + "microcosm.build.input_mass", + "microcosm.build.ledger_targets", + "microcosm.build.source_lineage", + "microcosm.build.stochastic_assignment", + "microcosm.build.uk_runtime.battery_bindings", + "microcosm.build.uk_runtime.bus_fare_pricing", + "microcosm.build.uk_runtime.bus_support_pricing", + "microcosm.build.uk_runtime.bus_use_incidence", + "microcosm.build.uk_runtime.cgt_projection", + "microcosm.build.uk_runtime.chronicle_feed", + "microcosm.build.uk_runtime.copy_id_table", + "microcosm.build.uk_runtime.council_tax_constants", + "microcosm.build.uk_runtime.cut_reference", + "microcosm.build.uk_runtime.dfe_spring_term", + "microcosm.build.uk_runtime.diagnostics", + "microcosm.build.uk_runtime.donor_uprating", + "microcosm.build.uk_runtime.energy_pricing", + "microcosm.build.uk_runtime.etb_services", + "microcosm.build.uk_runtime.frs_annual_income", + "microcosm.build.uk_runtime.frs_take_up", + "microcosm.build.uk_runtime.gambling_participation", + "microcosm.build.uk_runtime.geography_ladder", + "microcosm.build.uk_runtime.high_value_home_gates", + "microcosm.build.uk_runtime.high_value_homes", + "microcosm.build.uk_runtime.hmrc_capital_gains", + "microcosm.build.uk_runtime.household_lineage", + "microcosm.build.uk_runtime.lcfs_consumption", + "microcosm.build.uk_runtime.ledger_fact_vendoring", + "microcosm.build.uk_runtime.ledger_targets", + "microcosm.build.uk_runtime.local_authority_input", + "microcosm.build.uk_runtime.local_targets", + "microcosm.build.uk_runtime.national_frame", + "microcosm.build.uk_runtime.nts_bus_travel", + "microcosm.build.uk_runtime.receipt_counts", + "microcosm.build.uk_runtime.reference_surface", + "microcosm.build.uk_runtime.release_identity", + "microcosm.build.uk_runtime.release_input_coverage", + "microcosm.build.uk_runtime.rowwise_geography", + "microcosm.build.uk_runtime.source_runtime", + "microcosm.build.uk_runtime.stage_health", + "microcosm.build.uk_runtime.take_up_contract", + "microcosm.build.uk_runtime.terminal_gates", + "microcosm.build.uk_runtime.uc_capital_coherence", + "microcosm.build.uk_runtime.weighted_integrity", + "microcosm.calibrate.geography_constants", + "microcosm.calibrate.hierarchy", + "microcosm.calibrate.registry", + "microcosm.calibrate.solve", + "microcosm.calibrate.target", + "microcosm.frame.adapters.policyengine_uk", + "microcosm.frame.materialize", +) + + +def uk_gate_registry_modules() -> tuple[ModuleType, ...]: + """The modules :data:`UK_GATE_REGISTRY_MODULES` names, imported, for a + kernel's implementation hash.""" + + return tuple(importlib.import_module(name) for name in UK_GATE_REGISTRY_MODULES) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/brma_area_layer.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/brma_area_layer.py index 01543433a4..eade32e6f0 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/brma_area_layer.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/brma_area_layer.py @@ -1,20 +1,22 @@ """The area-to-BRMA layer and each placed household's BRMA (microcosm#1262). -The spine's ``frs_brma`` stage draws a household's Broad Rental Market Area -within its FRS region before any household has an area, so every geographic -clone of a household used to carry the same BRMA wherever it was placed, and -its Local Housing Allowance rate described a different place from its local -authority and constituency. +The spine's ``frs_brma`` stage drew a household's Broad Rental Market Area +within its FRS region before any household had an area, so no household's +BRMA followed the area it was placed in, and its Local Housing Allowance rate +described a different place from its local authority and constituency. +``area_brma`` took its place (microcosm#931). ``uk/brma_area_layer.json`` places every 2021 LSOA of England and Wales and every 2022 output area of Scotland in one BRMA, by population-weighted centroid in the published BRMA polygons (Scotland corrected by Rent Service Scotland's postcode lookup); ``tools/build_uk_brma_area_layer.py`` rebuilds -it from Chronicle's raw registrations. The post-geography node -``uk.full.geography.brma`` reads a placed household's BRMA off its assigned -area, as its local authority and constituency are read. Northern Ireland has -no open BRMA geography and keeps the spine's draw. The national line has no -areas and keeps the spine's draw everywhere. +it from Chronicle's raw registrations. The clone-first spine's BRMA stage +(``spine_geography.UKAreaBRMAStageTransform``, G4 of microcosm#931) reads each +Great Britain household's BRMA off the area the spine draws it, as its local +authority and constituency are read. Northern Ireland has no open BRMA +geography: its households keep the BRMA ``frs_brma``'s draw gives them from +the census counts within their region, keyed on each benefit unit +(``frs_brma.draw_household_brmas``). Everything here is pure and depends on the standard library, numpy and pandas only, so the build tool can load this module by path. @@ -32,7 +34,6 @@ __all__ = [ "BRMA_AREA_LAYER_SOURCE", - "BRMA_LOCALISATION_MODES", "GRAINS", "KIND", "METHOD_VERSION", @@ -52,9 +53,6 @@ METHOD_VERSION = "uk-brma-area-layer-v1" #: Graph source name of the committed layer bytes. BRMA_AREA_LAYER_SOURCE = "uk_brma_area_layer" -#: ``area`` reads each placed Great Britain household's BRMA off its assigned -#: area; ``none`` leaves the spine's regional draw on every clone. -BRMA_LOCALISATION_MODES = ("none", "area") NI_SYSTEM = "uk_ni_data_zone_2021" diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py index 3d1406da30..fce711cd4b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/calibration_run.py @@ -5,11 +5,13 @@ shared executable graph (:mod:`.graph_national`, the driver's national path in :mod:`.full_build_cli`). What stays here is what both lines and the release-cut certifier still consume: the Logbook pipeline and attempt-id -prefix, the four gate scopes and their closed-world exclusions, the bound -spine sidecar and checkpoint authentication, the spine provenance blocks, -the scoped gate manifest, the CGT projection and admin-anchor artifacts, -the Ledger and runtime provenance blocks, the register census and doctrine -payloads, and the scoped-report signing. +prefix, the gate scopes and their closed-world exclusions, the bound spine +sidecar and checkpoint authentication, the spine provenance blocks, the CGT +projection and admin-anchor artifacts, the Ledger and runtime provenance +blocks, the register census and doctrine payloads, and the scoped-report +signing. The spine and local gate scopes and the scoped gate manifest live +in :mod:`.gate_scopes`, which the full build's gate battery hashes for them +in place of this module, and are re-exported here. """ from __future__ import annotations @@ -47,6 +49,20 @@ UK_NHS_SPENDING_COMPONENT_COLUMNS, ) +# Moved to gate_scopes, which the full build's gate battery hashes, so the +# full gate kernels' own hashes and the preflight's own code leave this +# module out (microcosm#931, as microcosm#1492 moved the two functions +# below): in the in-graph build an edit elsewhere in it re-keys only the +# calibrated gates and what follows them. On a bound spine the checkpoint, +# whose kernel hashes this module, still re-keys the preflight and the solve. +# Kept importable from here for this module's other callers. +from microcosm.build.uk_runtime.gate_scopes import ( + UK_LOCAL_GATE_SCOPE, + UK_SPINE_GATE_SCOPE, + uk_scoped_gate_manifest, +) +from microcosm.build.uk_runtime.geographic_clones import geographic_clone_count + # Moved to ledger_surface, which the UK target kernels hash, so an edit # elsewhere in this module no longer re-keys them (microcosm#1492). Kept # importable from here for this module's other callers. @@ -87,16 +103,6 @@ "uk_calibration_reference_coverage", ) -UK_LOCAL_GATE_SCOPE = ( - "uk_local_geography_ladder_post_calibration", - "uk_local_brma_enum_domain", - "uk_local_area_support", - "uk_local_target_fit", - "uk_local_per_family_fit", - "uk_local_weight_ratio", - "uk_local_weight_ess", -) - #: Report-only gates the dense battery evaluates on the calibrated pool beside #: the local battery; they are excluded from certification like the local scope. UK_LOCAL_DIAGNOSTIC_GATE_SCOPE = ( @@ -104,79 +110,6 @@ "uk_high_value_homes_support", ) -UK_SPINE_GATE_SCOPE = ( - "uk_stage_was_wealth_support", - "uk_stage_was_wealth_coherence", - "uk_stage_was_lisa_support", - "uk_stage_was_lisa_anchor", - "uk_stage_high_value_home_support_mass", - "uk_stage_property_components", - "uk_stage_nts_bus_travel_support", - "uk_stage_nts_bus_travel_facts", - "uk_stage_uc_deduction_attributes", - "uk_stage_lcfs_consumption_support", - "uk_stage_lcfs_consumption_energy_rake", - "uk_stage_lcfs_consumption_bus_pricing", - "uk_stage_lcfs_consumption_road_fuel_incidence", - "uk_stage_lcfs_consumption_road_fuel_level", - "uk_stage_lcfs_consumption_recomposed_totals", - "uk_stage_lcfs_consumption_excise_quantities", - "uk_stage_lcfs_consumption_gambling_participation", - "uk_stage_lcfs_consumption_basis", - "uk_stage_etb_vat_support", - "uk_stage_etb_services_support", - "uk_stage_etb_services_support_pricing", - "uk_stage_frs_hmrc_spine_leaves_signal", - "uk_stage_frs_annual_income_spells", - "uk_stage_spi_support_channel_mass", - "uk_stage_hmrc_spi_income_spine_identity", - "uk_stage_hmrc_spi_income_spine_hourly_pay", - "uk_stage_spi_benefit_coherence", - "uk_stage_uc_take_up", - "uk_stage_pension_credit_take_up", - "uk_stage_child_benefit_take_up", - "uk_stage_cgt_incidence_clone_mass", - "uk_stage_cgt_support_split_mass", - "uk_stage_cgt_residential_split_mass", - "uk_stage_spi_income_band_donors_support", - "uk_stage_hmrc_cgt_gains_spine_summary", - "uk_stage_hmrc_cgt_asset_type_spine_summary", - "uk_stage_cgt_incidence_anchor_composition", - "uk_stage_salary_sacrifice_realization", - "uk_stage_student_loans_realization", - "uk_stage_age_tail_targets", - "uk_stage_frs_relationships_composition", - "uk_stage_regional_home_values_ranking", - "uk_stage_property_purchases_fit", - "uk_stage_uc_legacy_closure_take_up_contract", - "uk_stage_frs_person_draws_apprentices", - # The take-up flags at their drawing stages, before calibration (María's - # ruling of 2026-10-29, microcosm#1095). - "uk_stage_frs_take_up_signal", - "uk_stage_frs_person_draws_take_up_signal", - "uk_stage_frs_household_draws_take_up_signal", - "uk_stage_childcare_copy_take_up_signal", - "uk_stage_isa_subscriptions_invariants", - "uk_stage_isa_subscriptions", - "uk_stage_company_cars_invariants", - "uk_stage_spi_employment_shell_fit", - "uk_stage_aps_occupation_fit", - # Weight-independent like the BRMA enum below, and its column exists from - # frs_relationships onward (#791). - "uk_ons_household_type_enum_domain", - # Weight-independent; the column exists only from hmrc_cgt_asset_type_spine - # onward, so it is checked at the transferred boundary, and its domain is - # declared by the stage module (microcosm#725). - "uk_capital_gains_asset_type_enum_domain", - # Weight-independent; spi_housing_shell is the last spine stage that writes - # the band, so the transferred boundary checks every row's nation - # (microcosm#1278). - "uk_council_tax_band_domain", - # Weight-independent, and its column exists from frs_brma onward, so the - # spine checks it at the assembled boundary instead of the release end. - "uk_brma_enum_domain", -) - UK_NATIONAL_GATE_SCOPE = ( "uk_release_input_coverage_manifest_current", "uk_release_family_build_stages", @@ -298,7 +231,7 @@ def uk_local_gate_scope_exclusions() -> dict[str, str]: elif gate_id in UK_LOCAL_DIAGNOSTIC_GATE_SCOPE: reason = ( "local diagnostic gate; evaluated by the dense battery but outside " - "the seven-gate local release contract (microcosm#1240, #1262)." + "the six-gate local release contract (microcosm#1240, #1262, #931)." ) else: # pragma: no cover - _scope_exclusions enforces closed-world ownership raise RuntimeError(f"UK gate {gate_id!r} belongs to no declared scope.") @@ -494,6 +427,14 @@ def _assert_spine_sidecar_binds_frame( sidecar: Mapping[str, object], frame: Frame, ) -> None: + """Refuse a sidecar that describes another frame than the one bound. + + The row counts, the household weight kind and total, and the geographic + clone count K (microcosm#931): a sidecar written before the spine recorded + its K describes a spine of one clone, and the frame's K is counted from + its clone index (``geographic_clone_count``). The refusals print counts, + the weight kind and total and K, never an id or a value of a row. + """ expected_counts = sidecar.get("entity_row_counts") actual_counts = {entity: int(len(frame.table(entity))) for entity in frame.entities} if expected_counts != actual_counts: @@ -517,6 +458,15 @@ def _assert_spine_sidecar_binds_frame( "input H5 build sidecar household_weight_total mismatch: " f"sidecar {expected_total!r}, frame {actual_total!r}" ) + expected_clones = sidecar.get("geographic_clone_count", 1) + actual_clones = geographic_clone_count( + frame.table("household"), stage="The bound spine" + ) + if type(expected_clones) is not int or expected_clones != actual_clones: + raise ValueError( + "input H5 build sidecar geographic_clone_count mismatch: " + f"sidecar {expected_clones!r}, frame {actual_clones!r}" + ) def _spine_gate_report_path(sidecar_path: Path) -> Path: @@ -607,6 +557,16 @@ def spine_provenance_from_sidecar( # Empty for a spine built before the marker; the release assemblers # rederive it from the stages either way (microcosm#1095). "spi_public_use_tape": dict(sidecar.get("spi_public_use_tape") or {}), + # What the dense line takes from the spine it binds (microcosm#931): + # its clone count K (one where a sidecar predates the record), its + # geography binding (empty before the spine assigned areas), its + # sampling receipt (none at f100) and its release posture. + "geographic_clone_count": sidecar.get("geographic_clone_count", 1), + "geography": dict(sidecar.get("geography") or {}), + "sampling": ( + None if sidecar.get("sampling") is None else dict(sidecar["sampling"]) + ), + "release_posture": sidecar.get("release_posture"), } @@ -618,36 +578,6 @@ def _calibration_gate_manifest() -> GatesManifest: ) -def uk_scoped_gate_manifest( - scope: tuple[str, ...] | frozenset[str], - *, - phases: tuple[str, ...], - policy_suffix: str, - source: GatesManifest | None = None, -) -> GatesManifest: - """Filter the declared UK gate spec to one battery's scope. - - The one scope-filtering implementation every scoped producer shares - (spine build, calibration seam, release cut). ``source`` lets a caller - that already holds a loaded spec — or a hermetic test that stubs one — - supply it; the default is the committed package spec. - """ - - if source is None: - source = load_country_spec("uk").gates - entries = tuple(entry for entry in source.gates if entry.id in scope) - missing = sorted(set(scope) - {entry.id for entry in entries}) - if missing: - raise RuntimeError(f"UK gate scope names undeclared gate id(s): {missing}.") - return GatesManifest( - country=source.country, - version=source.version, - policy=f"{source.policy}; {policy_suffix}", - phases=phases, - gates=entries, - ) - - #: Admin anchors published against a concept the frame carries only in parts. #: The anchor keeps the publisher's shape (one NHS budget line); our stages #: carry the spend split by point of delivery. Composing is the translation, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_asset_type.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_asset_type.py index 1a2ffcecf4..f32270e81c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_asset_type.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_asset_type.py @@ -34,31 +34,38 @@ weighted mean log gain of the liable population, ``h`` one for a gainer who shows residential stock (another residential property in the household, or property income) and ``s`` the declared stock log-odds - (microcosm#1014). ``(a, b)`` are solved by nested bisection so the - household-weighted expected count and gains of flagged persons equal the - individuals-basis Table 8a totals: residential gainers are many and - small relative to the whole (a fifth of the gains on a third of the - taxpayers), which the negative slope carries. Flags are then realised by - a weighted systematic walk in ascending gain order with one seeded - offset: the walk flags a person whenever the expected weight owed so far - reaches that person's weight, so the flagged weight tracks the expected - weight within one person's weight everywhere along the gains axis, and - both the realised count and the realised gains sit close to their - expectations on frames whose liable rows are few and heavy. The - Bernoulli sigma of each is reported as the envelope a gate can bound - with. A flagged person's whole net - gain is attributed to residential property (``capital_gains_residential_property``); - a taxpayer with both residential and other disposals is not split. + (microcosm#1014). ``(a, b)`` are solved by nested bisection + (``solve_residential_logistic``) so the household-weighted expected + count and gains of flagged persons equal the individuals-basis Table 8a + totals: residential gainers are many and small relative to the whole (a + fifth of the gains on a third of the taxpayers), which the negative + slope carries. The flag is not drawn. Since microcosm#1063 the + ``cgt_residential_split`` stage, which runs before this one and makes + that solve, carries ``p`` as weight: it splits every household carrying + liable gainers into arms at product weights, one per subset of + residential gainers, so at design weights the residential count and + gains are mass identities. This stage reads the arms: a person whose + ``capital_gains_residential_property`` is positive is a residential + gainer on that arm, and the whole net gain of a flagged person is + residential; a taxpayer with both residential and other disposals is + not split. 2. **BADR and Investors' Relief claims.** Among liable gainers not flagged residential, each Table 4.1 band draws its claimants from the persons whose net gain falls inside the band's range. Below the lifetime limit a claimant's whole net gain qualifies, so the band's claimants are solved like the residential flag: a logistic in centred log gains whose intercept matches the band's published claimant count and whose slope - matches its published qualifying gains, realised by the same weighted - systematic walk with one seeded offset per band. Each band's logistic - carries the same stock shift for a gainer who shows business stock - (household corporate wealth, or self-employment income). The open top + matches its published qualifying gains. Each band's logistic carries the + same stock shift for a gainer who shows business stock (household + corporate wealth, or self-employment income). The claims are realised + by a weighted systematic walk (``_weighted_systematic_flags``) in + ascending gain order, persons tied on gain in the order of a uniform + keyed on their person_id, with one identity-keyed offset per band, + keyed on its bounds: the walk flags a person whenever the expected + weight owed so far reaches that person's weight, so the flagged weight + tracks the expected weight within one person's weight along the gains + axis. The Bernoulli sigma of each band's count and gains is reported as + the envelope a gate can bound with. The open top band holds claimants at the lifetime limit: its probability is flat in gains and varies only with the stock shift, its intercept matches the count, the count is the published gains over the limit (the published @@ -67,7 +74,7 @@ Table 4.1's tax column to within half a percent; Table 4.1 counts gains before losses while the frame carries net gains, and Investors' Relief is merged with BADR as in the engine's input. -3. **Main asset type.** Every liable gainer not flagged residential draws +3. **Main asset type.** Every liable gainer not flagged residential takes one of five Table 7 types from a size-tilted categorical: the type weight times a log-normal kernel in log gains centred on the type's Table 7 mean gain per disposal (listed shares near £5,500, unlisted @@ -75,15 +82,21 @@ declared stock floor where the type's stock signal is absent (listed shares: a stocks-and-shares ISA or dividends; unlisted shares: business stock; other financial assets: financial wealth). A BADR or - Investors' Relief claimant draws only from the three types the reliefs - apply to (unlisted shares, business land and buildings, other + Investors' Relief claimant takes only one of the three types the + reliefs apply to (unlisted shares, business land and buildings, other non-financial assets), so the type follows the claim. The type weights - are fitted, with that restriction, by multiplicative updates until the - household-weighted gains shares by type reproduce Table 7's - non-residential gains shares, and the fit refuses if it does not - converge; one seeded uniform per person realises the draw. The - composition by gain band is reported for the evidence note; nothing is - fitted to it. + are fitted once per claimant status, with that restriction, by + multiplicative updates until each status's household-weighted gains + shares by type reach its share targets, which together restore Table + 7's non-residential gains shares, and each fit refuses if it does not + converge. The types are then assigned by error diffusion + (``_diffuse_types``) in descending gain order, persons tied on gain in + the order of a uniform keyed on their person_id, the assignment's only + draw: each person adds its stake (weight times gain) times its type + probabilities to a running balance per type and takes the type it may + hold whose balance is largest, so the realised gains by type track the + fit. The composition by gain band is reported for the evidence note; + nothing is fitted to it. Non-gainers carry ``none``; positive gains at or below the annual exempt amount carry ``sub_aea`` (Table 8 describes taxpayers with a liability and @@ -105,6 +118,10 @@ import pandas as pd from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.stochastic_assignment import ( + keyed_tie_order, + stable_identity_uniform, +) from microcosm.build.uk_runtime.cgt_imputation import ( UKCGTPolicyParameters, uk_cgt_policy_parameters, @@ -256,8 +273,10 @@ #: The BADR seed is combined with the build period, as the amounts stage #: does; distinct from the amounts stage's 552 so the two stages never share -#: a stream. The residential flag draws nothing since microcosm#1063: the -#: ``cgt_residential_split`` stage carries its probability as weight. +#: a draw. It also keys the tie-breaks of the BADR walk and the main asset +#: type walk, each under its own salt (microcosm#931). The residential flag +#: draws nothing since microcosm#1063: the ``cgt_residential_split`` stage +#: carries its probability as weight. CGT_BADR_FLAG_SEED = 555 _BISECTION_ITERATIONS = 200 @@ -1005,6 +1024,35 @@ def _json_number(value: float) -> float | None: return float(value) if np.isfinite(value) else None +def _badr_band_offset(band: HMRCCGTBADRBand, *, seed: int, time_period: int) -> float: + """A Table 4.1 band's walk offset: an identity-keyed uniform on its bounds. + + Keyed on the band's lower and upper edges, salted with the build period, + not on its position in the roster, so a facts revision that adds or + drops a band leaves every other band's offset alone (microcosm#931). + The key formats the edges, so an edge read as the float ``50000.0`` + would key ``50000.0`` and draw the band's offset anew, though the roster + check, which compares edges by value, accepts it; an edge that is not an + integer is refused, as ``integer_record_keys`` refuses float record ids. + """ + + edges = [band.lower_bound] + if band.upper_bound is not None: + edges.append(band.upper_bound) + for edge in edges: + if isinstance(edge, bool) or not isinstance(edge, (int, np.integer)): + raise TypeError( + "Table 4.1 band edges must be integers to key a band's walk " + f"offset, not {type(edge).__name__}." + ) + upper = "open" if band.upper_bound is None else int(band.upper_bound) + return stable_identity_uniform( + f"{int(band.lower_bound)}|{upper}", + seed=seed, + salt=f"assign_badr_qualifying_gains:offset:{int(time_period)}", + ) + + def _assign_badr_claims( *, gains: np.ndarray, @@ -1021,9 +1069,12 @@ def _assign_badr_claims( """Draw each Table 4.1 band's claimants and their qualifying gains. Returns the qualifying gain of every person (0 where not a claimant) and - one receipt row per band. One seeded offset per declared band is drawn - in band order whether or not the band is skipped, so the stream does - not depend on which bands carry a target. + one receipt row per band. Each band's walk offset is an identity-keyed + uniform on its bounds (``_badr_band_offset``), so it depends neither on + which bands carry a target nor on which other bands the facts declare. + The walk runs in ascending gain order, persons tied on gain in the order + of a uniform keyed on their person_id, salted with the build period, then + of their ids (``keyed_tie_order``; microcosm#931). """ qualifying = np.zeros(gains.shape) @@ -1031,11 +1082,9 @@ def _assign_badr_claims( if not limit > 0.0: raise ValueError(f"BADR lifetime limit must be positive, got {limit}.") bands = facts.table4_bands - rng = np.random.default_rng((seed, int(time_period))) - offsets = rng.random(len(bands)) pool_gains = gains[pool_index] receipts: list[dict[str, object]] = [] - for position, band in enumerate(bands): + for band in bands: label = f"BADR band {band.label}" open_top = band.upper_bound is None if open_top: @@ -1109,9 +1158,17 @@ def _assign_badr_claims( np.log(band_gains) - centre, intercept, slope, band_offset ) amounts = band_gains - order = np.lexsort((person_id[rows], band_gains)) + order = keyed_tie_order( + band_gains, + person_id[rows], + seed=seed, + salt=f"assign_badr_qualifying_gains:tie:{int(time_period)}", + ) flags = _weighted_systematic_flags( - probabilities, band_weights, order, float(offsets[position]) + probabilities, + band_weights, + order, + _badr_band_offset(band, seed=seed, time_period=time_period), ) claimants = rows[flags] qualifying[claimants] = amounts[flags] @@ -1527,8 +1584,15 @@ def assign_uk_cgt_asset_types( ) stakes = person_weight[rows] * gains[rows] # Largest gain first, so the rows that could move the composition most - # are placed while every later row can still repay the balance. - order = np.lexsort((person_id[rows], -gains[rows])) + # are placed while every later row can still repay the balance; persons + # tied on gain take the order of a uniform keyed on their person_id, + # then of their ids (microcosm#931). + order = keyed_tie_order( + -gains[rows], + person_id[rows], + seed=badr_seed, + salt=f"assign_main_asset_type:tie:{int(time_period)}", + ) choice, balances = _diffuse_types(type_probabilities, stakes, order) remainder_types[in_status] = choice mass = float(stakes.sum()) @@ -1660,7 +1724,10 @@ def assign_uk_cgt_asset_types( "log_sigma": CGT_ASSET_TYPE_LOG_SIGMA, "median_anchor_gbp": medians, "composition_by_claimant_status": status_reports, - "realization": "error diffusion in descending (gain, person_id) order", + "realization": ( + "error diffusion in descending gain order, ties in ascending order of " + "a uniform keyed on person_id, then person_id ascending" + ), "achieved_median_gain_gbp": { asset_type_name: _weighted_median( gains[asset_type == asset_type_name], @@ -1930,9 +1997,16 @@ def cgt_asset_type_operation_parameters() -> dict[str, dict[str, Any]]: "Table 4.1's tax column to within half a percent" ), "realization": ( - "weighted systematic sampling within each band in ascending " - "(gain, person_id) order with one seeded offset per band, drawn " - "in band order whether or not the band carries a target" + "weighted systematic sampling within each band in ascending gain " + "order, persons tied on gain in ascending order of a uniform " + "keyed on person_id under seed and salted " + "assign_badr_qualifying_gains:tie:, then person_id " + "ascending, with one identity-keyed offset per band, keyed on " + "its bounds (|, open for the " + "top band) and salted assign_badr_qualifying_gains:offset:, " + "with no stream order, so a band's offset depends neither on " + "which bands carry a target nor on which other bands the facts " + "declare" ), "seed": CGT_BADR_FLAG_SEED, "seed_mixing": "seed combined with the build period", @@ -2002,13 +2076,15 @@ def cgt_asset_type_operation_parameters() -> dict[str, dict[str, Any]]: "weight_fitting_iterations": _SHARE_FIT_ITERATIONS, "weight_fitting_tolerance": _SHARE_FIT_TOLERANCE, "realization": ( - "error diffusion per claimant status in descending (gain, " - "person_id) order: each person adds weight x gain x type " - "probability to a running balance per type and takes the type it " - "may hold with the largest balance, which is drawn down by the " - "whole weight x gain, so the realised gains by type track the " - "fitted composition instead of following the heaviest rows; " - "deterministic, no seed" + "error diffusion per claimant status in descending gain order, " + "persons tied on gain in ascending order of a uniform keyed on " + "person_id under the assign_badr_qualifying_gains seed and salted " + "assign_main_asset_type:tie:, then person_id ascending: " + "each person adds weight x gain x type probability to a running " + "balance per type and takes the type it may hold with the largest " + "balance, which is drawn down by the whole weight x gain, so the " + "realised gains by type track the fitted composition instead of " + "following the heaviest rows; that tie-break is its only draw" ), "output_column": CGT_ASSET_TYPE_COLUMN, "value_domain": list(CGT_ASSET_TYPE_DOMAIN), diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py index 360c4d6fb3..a5fad7b30c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_imputation.py @@ -99,6 +99,10 @@ import pandas as pd from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.stochastic_assignment import ( + keyed_tie_uniforms, + stable_identity_uniform, +) from microcosm.build.uk_runtime.advani_summers import ( ADVANI_SUMMERS_RESOURCE, ADVANI_SUMMERS_VINTAGE, @@ -168,13 +172,15 @@ #: the certified June H5) is retired; its family left the release contract. UK_CGT_SPINE_STAGE_NAME = "hmrc_cgt_gains_spine" -#: Base seed for the stage's draws. Combined with the build period so two -#: periods draw differently while each build is reproducible. +#: Base seed for the stage's draws. Combined with the build period, which +#: salts every walk offset, plan jitter and rank-tie uniform, so two periods +#: draw differently while each build is reproducible. UK_CGT_IMPUTATION_SEED = 552 #: How the sub-AEA remainder takes its amounts (microcosm#970): the #: Advani-Summers within-band quantile function restricted to the exempt -#: range, rank-preserving within each total-income band, no seeded draw. +#: range, rank-preserving within each total-income band, no seeded draw of +#: its own (ties on the prior gain take the allocation's rank-tie uniforms). UK_CGT_REMAINDER_POLICY = "advani_summers_within_band_exempt_range" #: The reviewed mass-conservation receipt the spine stage records. The @@ -1224,12 +1230,30 @@ def margin_errors( return targets, report +class _KeyedOffset: + """One identity-keyed uniform in the shape the walks read (``random()``). + + ``_walk_bands`` reads its rounding offset, and ``_draw_plan_amounts`` its + quantile jitter, through one ``rng.random()`` call each. The redraw + passes each call one of these, holding a ``stable_identity_uniform`` + keyed on that walk's or plan's own label, in place of a shared + Generator, whose n-th draw belonged to whichever call came n-th + (microcosm#931). The ISA subscriptions walk their bands with it too. + """ + + def __init__(self, value: float) -> None: + self._value = float(value) + + def random(self) -> float: + return self._value + + def _walk_bands( *, ranked: np.ndarray, weights: np.ndarray, boundaries: Sequence[tuple[int, float]], - rng: np.random.Generator, + rng: np.random.Generator | _KeyedOffset, members: dict[int, list[np.ndarray]], ) -> tuple[np.ndarray, dict[int, float]]: """Assign ranked gainers to bands, highest first, and draw their amounts. @@ -1272,8 +1296,9 @@ def _draw_plan_amounts( person_id: np.ndarray, existing: np.ndarray, weights: np.ndarray, - rng: np.random.Generator, + rng: np.random.Generator | _KeyedOffset, new_gains: np.ndarray, + tie: np.ndarray | None = None, ) -> None: """Draw one plan's amounts on weight-proportional quantile strata. @@ -1287,12 +1312,13 @@ def _draw_plan_amounts( third of whose weight sat in the bottom strata). The open band takes each stratum's conditional mean, so its weighted mean is the published mean exactly; a bounded band takes the quantile inside each stratum, which - reproduces the mean to the width of the widest stratum. + reproduces the mean to the width of the widest stratum. ``tie`` orders + members tied on the key as :func:`_ranked` does, reversed. """ if members.size == 0: return - ordered = _ranked(members, person_id=person_id, existing=existing)[::-1] + ordered = _ranked(members, person_id=person_id, existing=existing, tie=tie)[::-1] cumulative = np.cumsum(weights[ordered]) total = float(cumulative[-1]) if not total > 0.0: @@ -1316,12 +1342,27 @@ def _draw_plan_amounts( def _ranked( - indices: np.ndarray, *, person_id: np.ndarray, existing: np.ndarray + indices: np.ndarray, + *, + person_id: np.ndarray, + existing: np.ndarray, + tie: np.ndarray | None = None, ) -> np.ndarray: - # Rank by the given key, largest first: the allocation's wealth-blended - # rank key, or the prior gain for the sub-AEA remainder. person_id breaks - # ties so the ordering, and with it every draw, is deterministic. - order = np.lexsort((person_id[indices], -existing[indices])) + """``indices`` by the given key, largest first. + + The key is the allocation's wealth-blended rank key, or the prior gain + for the sub-AEA remainder. Persons tied on it take the order of ``tie`` + ascending, one identity-keyed uniform per person + (:func:`microcosm.build.stochastic_assignment.keyed_tie_uniforms`, the + uniforms its ``keyed_tie_order`` sorts on), then of person_id, so no id + range, the copies or clones of one household among them, wins every tie + (microcosm#931). With no ``tie``, person_id alone breaks ties. + """ + + if tie is None: + order = np.lexsort((person_id[indices], -existing[indices])) + else: + order = np.lexsort((person_id[indices], tie[indices], -existing[indices])) return indices[order] @@ -1335,6 +1376,7 @@ def _map_remainder_amounts( rows: Sequence[Mapping[str, object]], annual_exempt_amount: float, new_gains: np.ndarray, + tie: np.ndarray | None = None, ) -> dict[str, object]: """Place the sub-AEA remainder on the A&S within-band surface (microcosm#970). @@ -1347,14 +1389,17 @@ def _map_remainder_amounts( its crossing of the annual exempt amount, so the band's weighted remainder reproduces the published conditional shape on (0, AEA]. The mapping is a monotone transform of the existing gains inside each band, - consumes no seeded draw, and fails closed if any amount leaves the - range. Returns the receipt for the allocation report. + makes no draw of its own (``tie``, the allocation's rank-tie uniforms, + orders persons tied on their prior gain, as :func:`_ranked` reversed), + and fails closed if any amount leaves the range. Returns the receipt for + the allocation report. """ band_rows: list[dict[str, object]] = [] for band in np.unique(band_index[remainder]): members = remainder[band_index[remainder] == band] - ordered = _ranked(members, person_id=person_id, existing=existing)[::-1] + ranked = _ranked(members, person_id=person_id, existing=existing, tie=tie) + ordered = ranked[::-1] knots = advani_summers_knots(rows[int(band)]) lower_quantile, upper_quantile = exempt_range_quantiles( knots, annual_exempt_amount @@ -1526,7 +1571,7 @@ def _placement_receipts( Receipts only: every input is what the two passes, the plan draws and the remainder mapping already produced, and nothing here touches - ``new_gains`` or the seeded stream. Placed rows are the gainers a pass + ``new_gains`` or draws. Placed rows are the gainers a pass assigned to a plan; demoted rows are the gainers left to the sub-exempt remainder. The initial band is the Table 2.1a band of the prior gain; the redrawn band is the band of the new amount, which for a placed row is the @@ -1766,8 +1811,8 @@ def impute_uk_capital_gains_with_report( ) if missing_households: raise ValueError( - "Person rows reference households with no weight: " - f"{sorted(missing_households)[:5]}." + f"Person rows reference {len(missing_households)} household(s) with " + "no weight." ) person_weight = ( person["person_household_id"].map(weights_by_household).to_numpy(dtype=float) @@ -1784,10 +1829,29 @@ def impute_uk_capital_gains_with_report( _, age_group, _, region_group = _person_conditioning_cells(person, household) rank_key, investable_wealth = _allocation_rank_key(person, household, existing) + # Persons tied on a rank take the order of a uniform keyed on their own + # person_id, salted with the build period, then of their ids: one draw per + # person for every walk, plan and the remainder, so the same person wins + # a tie throughout and no id range, the copies or clones of one household + # among them, wins every tie (microcosm#931). + rank_tie = keyed_tie_uniforms( + person_id, + seed=seed, + salt=f"{UK_CGT_SPINE_STAGE_NAME}:rank_tie:{int(time_period)}", + ) # Receipts only (microcosm#1045): the walk never reads the row type. support_family, support_column_present = _support_family_rows(person, household) - rng = np.random.default_rng((seed, int(time_period))) + # Each walk's rounding offset and each plan's quantile jitter is an + # identity-keyed uniform on its own label, salted with the build period, + # so none depends on which other walks run or skip, or in what order + # (microcosm#931). + offset_salt = f"{UK_CGT_SPINE_STAGE_NAME}:offset:{int(time_period)}" + jitter_salt = f"{UK_CGT_SPINE_STAGE_NAME}:jitter:{int(time_period)}" + + def keyed(salt: str, key: str) -> _KeyedOffset: + return _KeyedOffset(stable_identity_uniform(key, seed=seed, salt=salt)) + new_gains = existing.copy() is_gainer = existing > 0 assigned = np.zeros(len(person), dtype=bool) @@ -1846,7 +1910,10 @@ def impute_uk_capital_gains_with_report( carry = -deferred continue ranked = _ranked( - np.flatnonzero(in_cell), person_id=person_id, existing=rank_key + np.flatnonzero(in_cell), + person_id=person_id, + existing=rank_key, + tie=rank_tie, ) weights = person_weight[ranked] # When the cell holds less gainer mass than its target, @@ -1863,7 +1930,11 @@ def impute_uk_capital_gains_with_report( ranked=ranked, weights=weights, boundaries=boundaries, - rng=rng, + rng=keyed( + offset_salt, + f"cell:{income_lower}:{UK_CGT_AGE_GROUP_LOWER_BOUNDS[a]}:" + f"{UK_CGT_REGION_GROUP_LABELS[r]}", + ), members=cell_members, ) for gain_lower, chunks in cell_members.items(): @@ -1902,7 +1973,9 @@ def impute_uk_capital_gains_with_report( gain_lower: value * net_shortfall / positive_total for gain_lower, value in positive.items() } - ranked = _ranked(np.flatnonzero(pool), person_id=person_id, existing=rank_key) + ranked = _ranked( + np.flatnonzero(pool), person_id=person_id, existing=rank_key, tie=rank_tie + ) weights = person_weight[ranked] scale = min(1.0, float(weights.sum()) / net_shortfall) boundaries = [ @@ -1914,7 +1987,7 @@ def impute_uk_capital_gains_with_report( ranked=ranked, weights=weights, boundaries=boundaries, - rng=rng, + rng=keyed(offset_salt, f"pool:{income_lower}"), members=pool_members, ) for gain_lower, chunks in pool_members.items(): @@ -1937,14 +2010,15 @@ def impute_uk_capital_gains_with_report( person_id=person_id, existing=rank_key, weights=person_weight, - rng=rng, + rng=keyed(jitter_salt, f"plan:{gain_lower}:{income_lower}"), new_gains=new_gains, + tie=rank_tie, ) # Below the published taxpayer mass: sub-AEA gainers take Advani-Summers # within-band amounts restricted to (0, AEA], rank-preserving within each - # A&S total-income band (microcosm#970). Nothing here touches rng, so the - # walk and plan draw order above is unchanged. + # A&S total-income band (microcosm#970). Nothing here draws; ties on the + # prior gain take the rank-tie uniforms. remainder_receipt = _map_remainder_amounts( remainder=np.flatnonzero(is_gainer & ~assigned), existing=existing, @@ -1956,6 +2030,7 @@ def impute_uk_capital_gains_with_report( rows=remainder_rows, annual_exempt_amount=parameters.annual_exempt_amount, new_gains=new_gains, + tie=rank_tie, ) if not np.isfinite(new_gains).all(): @@ -2434,10 +2509,14 @@ def _assert_cgt_spine_stage_parameters(stage: SourceStageSpec) -> None: "band's pooled unassigned gainers" ), "ordering": ( - "a rank key descending, person_id ascending on ties: among " - "gainers, the household investable-wealth rank times the wealth " - "rank weight plus the prior-gain rank times its complement, " - "both as average ranks over the gainers" + "a rank key descending, then the rank-tie uniform ascending, then " + "person_id ascending: among gainers, the household " + "investable-wealth rank times the wealth rank weight plus the " + "prior-gain rank times its complement, both as average ranks " + "over the gainers; the rank-tie uniform is one identity-keyed " + "uniform per person, keyed on person_id under seed_base and " + "salted hmrc_cgt_gains_spine:rank_tie:, which every " + "walk, plan and the remainder read" ), "wealth_rank_weight": UK_CGT_WEALTH_RANK_WEIGHT, "investable_wealth_columns": list(UK_CGT_INVESTABLE_WEALTH_COLUMNS), @@ -2512,10 +2591,16 @@ def _assert_cgt_spine_stage_parameters(stage: SourceStageSpec) -> None: "bottom_band_floor": "annual exempt amount plus one pound", "seed_base": UK_CGT_IMPUTATION_SEED, "seed_mixing": ( - "seed combined with the build period; each walk draws its " - "rounding offset in cell order then pooled-fallback order, then " - "each plan draws its quantile jitter, income bands ascending and " - "gain bands highest first" + "seed combined with the build period, which salts every draw; " + "each walk's rounding offset and each plan's quantile jitter is " + "an identity-keyed uniform on its own label, with no stream " + "order, so none depends on which other walks run or skip: a cell " + "walk keys on cell::: and " + "an income band's pooled walk on pool:, salted " + "hmrc_cgt_gains_spine:offset:, and a plan on " + "plan::, salted " + "hmrc_cgt_gains_spine:jitter:, bands and age groups by " + "their lower bounds and region groups by their labels" ), "deterministic": True, "cell_means": ( @@ -2561,14 +2646,14 @@ def _assert_cgt_spine_stage_parameters(stage: SourceStageSpec) -> None: "remainder gainers take the midpoints of the weight-proportional " "quantile strata that partition the range, evaluated on the " "band's spline; zero-weight persons take a nominal stratum of " - "the band's smallest positive weight and carry no mass; no " - "seeded draw is consumed, so the walk and plan draw order is " - "unchanged" + "the band's smallest positive weight and carry no mass; the " + "mapping makes no draw of its own" ), "ordering": ( - "existing gains ascending, person_id descending on ties (the " - "prior-gain ranking reversed; the allocation and the within-band " - "draws rank on the wealth-blended key)" + "existing gains ascending, then the rank-tie uniform descending, " + "then person_id descending (the prior-gain ranking reversed; the " + "allocation and the within-band draws rank on the wealth-blended " + "key)" ), "quantile_points": list(CGT_QUANTILE_POINTS), "spline_degree": 1, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_residential_split.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_residential_split.py index 2a4db3f39f..cb38dc94c0 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_residential_split.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_residential_split.py @@ -25,8 +25,9 @@ and ``_gains``; the by-band rows stay fenced), decides between the arms afterwards exactly as it decides between the incidence clone's halves. Every household's mass and composition and the total household mass are conserved; arm ``j`` takes every entity id plus -``j`` times the frame's id multiplier, the lineage rule the executor and the -geography identity kernel read for the support split and the clone. +``j`` times the declared radix of the copy-id table's top field, +``cgt_residential_split`` (microcosm#931), the lineage rule the executor +and the identity tool read for the support split and the clone too. """ from __future__ import annotations @@ -39,6 +40,7 @@ import pandas as pd from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.uk_runtime import copy_id_table from microcosm.build.uk_runtime.cgt_asset_type import ( CGT_RESIDENTIAL_GAINS_COLUMN, CGT_RESIDENTIAL_STOCK_SIGNAL, @@ -64,7 +66,6 @@ uk_time_period, validate_uk_national_frame, ) -from microcosm.build.uk_runtime.rowwise_geography import id_multiplier_for_values from microcosm.build.uk_runtime.spi_support import ( _exact_total_correction, _importance_weights_with_exact_total, @@ -86,6 +87,9 @@ #: A household with more liable gainers than this would need more than eight #: arms; none exists on the FRS spine, so the stage refuses rather than grows. CGT_RESIDENTIAL_MAXIMUM_LIABLE_GAINERS = 3 +#: The copy-id field arm ``j`` takes ``j`` in: the table's top field, so every +#: arm id exceeds every incoming id (microcosm#931). +CGT_RESIDENTIAL_COPY_ID_FIELD = "cgt_residential_split" #: Concentration diagnostic: the share of the residential gains mass on the #: largest residential arm stakes (arm weight times gain). CGT_RESIDENTIAL_TOP_ARM_ROWS = 10 @@ -227,7 +231,8 @@ def split_cgt_residential_households( shift); a household with ``k`` liable gainers becomes ``2 ** k`` arms at product weights, the arm with no residential gainer keeping the household's ids and every other arm ``j`` taking every entity id plus - ``j`` times the frame's id multiplier. The exact-total correction lands on + ``j`` times the declared radix of the copy-id table's + ``cgt_residential_split`` field. The exact-total correction lands on the heaviest unsplit incumbent so every arm weight stays bitwise its product. Refuses a frame the stage has already split, one the asset-type stage has already classified, or a household with more liable gainers @@ -321,13 +326,17 @@ def split_cgt_residential_households( f"{CGT_RESIDENTIAL_MAXIMUM_LIABLE_GAINERS} liable gainers; the " "residential split refuses to enumerate their arms." ) - multiplier = id_multiplier_for_values( - person["person_id"], - person["person_household_id"], - person["person_benunit_id"], - benunit["benunit_id"], - household["household_id"], - ) + for table, columns in ( + (person, _PERSON_ID_COLUMNS), + (benunit, ("benunit_id",)), + (household, ("household_id",)), + ): + for column in columns: + copy_id_table.require_below( + table[column], + CGT_RESIDENTIAL_COPY_ID_FIELD, + what=f"incoming {column} values", + ) household_position = pd.Series(np.arange(len(household)), index=household_ids) incumbent_weights = weights.copy() arm_person: list[pd.DataFrame] = [] @@ -365,10 +374,11 @@ def split_cgt_residential_households( member_positions = np.flatnonzero(members) member_rank = rank_by_position.reindex(member_positions).to_numpy() for j in range(1, 2**k): - id_offset = j * multiplier copy_person = family_person.copy() for column in _PERSON_ID_COLUMNS: - copy_person[column] = copy_person[column].astype("int64") + id_offset + copy_person[column] = copy_id_table.mint( + copy_person[column], CGT_RESIDENTIAL_COPY_ID_FIELD, j + ) residential = np.zeros(len(copy_person), dtype=bool) for m in range(k): if (j >> m) & 1: @@ -377,12 +387,12 @@ def split_cgt_residential_households( residential, gains[member_positions], 0.0 ) copy_benunit = family_benunit.copy() - copy_benunit["benunit_id"] = ( - copy_benunit["benunit_id"].astype("int64") + id_offset + copy_benunit["benunit_id"] = copy_id_table.mint( + copy_benunit["benunit_id"], CGT_RESIDENTIAL_COPY_ID_FIELD, j ) copy_household = family_household.copy() - copy_household["household_id"] = ( - copy_household["household_id"].astype("int64") + id_offset + copy_household["household_id"] = copy_id_table.mint( + copy_household["household_id"], CGT_RESIDENTIAL_COPY_ID_FIELD, j ) copy_household[HOUSEHOLD_IS_CGT_RESIDENTIAL_CLONE] = True copy_household[CGT_RESIDENTIAL_CLONE_INDEX_COLUMN] = np.full( @@ -392,10 +402,10 @@ def split_cgt_residential_households( arm_benunit.append(copy_benunit) arm_household.append(copy_household) arm_weights.append(weights[positions] * factors[:, j]) - # Every arm id exceeds every incoming id (the multiplier is larger than - # any id), so the arms follow the incumbents; within the arms the rows are - # ordered by id so the group tables stay sorted whatever the household - # order the arms were built in. + # Every arm id exceeds every incoming id (the arms' field is the copy-id + # table's top field, above every incoming id), so the arms follow the + # incumbents; within the arms the rows are ordered by id so the group + # tables stay sorted whatever the household order the arms were built in. arms_person = pd.concat(arm_person, ignore_index=True) if arm_person else None arms_benunit = pd.concat(arm_benunit, ignore_index=True) if arm_benunit else None arms_household = ( @@ -716,9 +726,9 @@ def cgt_residential_split_operation_parameters() -> tuple[ ), "gainer_order": "person_id ascending within the household", "id_remapping": ( - "id_multiplier_for_values; arm j takes every entity id + j x " - "multiplier, bit m of j marking the household's m-th liable " - "gainer residential" + "copy-id table field cgt_residential_split, the top field; arm " + "j takes every entity id + j x the field's radix (10**18), bit " + "m of j marking the household's m-th liable gainer residential" ), "flag_column": HOUSEHOLD_IS_CGT_RESIDENTIAL_CLONE, "index_column": CGT_RESIDENTIAL_CLONE_INDEX_COLUMN, @@ -782,6 +792,7 @@ def _assert_cgt_residential_split_stage_parameters(stage: SourceStageSpec) -> No __all__ = [ "CGT_RESIDENTIAL_CLONE_INDEX_COLUMN", + "CGT_RESIDENTIAL_COPY_ID_FIELD", "CGT_RESIDENTIAL_MAXIMUM_LIABLE_GAINERS", "CGT_RESIDENTIAL_PROBABILITY_COLUMN", "CGT_RESIDENTIAL_SPLIT_MASS_CHANGE_REASON", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_structure.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_structure.py index 2923099276..0d123feafc 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_structure.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_structure.py @@ -19,6 +19,7 @@ from microcosm.build.source_manifest import SourceStageSpec from microcosm.build.stochastic_assignment import stable_identity_uniforms +from microcosm.build.uk_runtime import copy_id_table from microcosm.build.uk_runtime.advani_summers import ( ADVANI_SUMMERS_RESOURCE, CGT_QUANTILE_POINTS, @@ -46,16 +47,20 @@ uk_time_period, validate_uk_national_frame, ) -from microcosm.build.uk_runtime.rowwise_geography import ( - clone_entity_frame, - id_multiplier_for_values, -) +from microcosm.build.uk_runtime.rowwise_geography import clone_entity_frame from microcosm.build.uk_runtime.spi_support import ( _importance_weights_with_exact_total, ) from microcosm.frame import Frame, MassChangeRecord, WeightKind CGT_CLONE_MASS_SPLIT = 0.5 +#: The copy-id field the incidence twin takes ordinal 1 in (microcosm#931). +CGT_CLONE_COPY_ID_FIELD = "cgt_incidence_clone" +#: The id rule the clone and anchor manifests restate. +CGT_CLONE_ID_REMAPPING = ( + "copy-id table field cgt_incidence_clone; the twin takes every entity id + " + "the field's radix (10**17)" +) CGT_PRIOR_SEED = 0 CGT_PRIOR_SALT = "cgt_prior_amount" CGT_ADULT_MINIMUM_AGE = 16 @@ -249,13 +254,19 @@ def clone_cgt_incidence( person = frame.table("person").copy() benunit = frame.table("benunit").copy() household = frame.table("household").copy() - multiplier = id_multiplier_for_values( - person["person_id"], - person["person_household_id"], - person["person_benunit_id"], - benunit["benunit_id"], - household["household_id"], - ) + # The twin takes every id plus the declared radix of the copy-id table's + # cgt_incidence_clone field, which sits above every field filled before + # it (microcosm#931). + for table, columns in ( + (person, ("person_id", "person_household_id", "person_benunit_id")), + (benunit, ("benunit_id",)), + (household, ("household_id",)), + ): + for column in columns: + copy_id_table.require_below( + table[column], CGT_CLONE_COPY_ID_FIELD, what=f"incoming {column} values" + ) + multiplier = copy_id_table.radix(CGT_CLONE_COPY_ID_FIELD) cloned_person = clone_entity_frame( person, id_columns=("person_id", "person_household_id", "person_benunit_id"), @@ -343,12 +354,14 @@ def pair_clone_households( ) -> tuple[np.ndarray, np.ndarray, int]: """Positions of every clone household and of its paired original. - The clone stage offsets ids by ``id_multiplier_for_values`` over the rows - it saw; those rows are exactly the households whose clone flag is false, - the support split's copies included (they are pre-clone rows with their - own clones, microcosm#1045). Recomputing the multiplier from them - reproduces the executor's lineage rule, and the pairing must be a - bijection: an unpaired clone or original fails closed. + The clone stage offsets ids by the declared radix of the copy-id table's + ``cgt_incidence_clone`` field over the rows it saw; those rows are exactly + the households whose clone flag is false, the support split's copies + included (they are pre-clone rows with their own clones, microcosm#1045). + A clone's original is its id less that radix, the executor's lineage rule, + whatever the frame's largest id (microcosm#931), and the pairing must be + a bijection: an unpaired clone or original fails closed, printing counts + only. """ if HOUSEHOLD_IS_CGT_CLONE not in household.columns: @@ -365,31 +378,22 @@ def pair_clone_households( ) if len(np.unique(household_ids)) != len(household_ids): raise ValueError("CGT incidence anchor requires unique household ids.") - original_ids = set(household_ids[original].tolist()) - person_original = person["person_household_id"].isin(original_ids).to_numpy() - benunit_original = ( - benunit["benunit_id"] - .isin(set(person.loc[person_original, "person_benunit_id"].tolist())) - .to_numpy() - ) - multiplier = id_multiplier_for_values( - person.loc[person_original, "person_id"], - person.loc[person_original, "person_household_id"], - person.loc[person_original, "person_benunit_id"], - benunit.loc[benunit_original, "benunit_id"], - household_ids[original], - ) + multiplier = copy_id_table.radix(CGT_CLONE_COPY_ID_FIELD) position_by_id = {int(value): index for index, value in enumerate(household_ids)} clone_positions = np.flatnonzero(clone) original_positions = np.empty(len(clone_positions), dtype=np.int64) + unpaired = 0 for slot, position in enumerate(clone_positions): source = position_by_id.get(int(household_ids[position]) - multiplier) if source is None or not original[source]: - raise ValueError( - "CGT incidence anchor found a clone household without a paired " - f"original: household_id {int(household_ids[position])}." - ) + unpaired += 1 + continue original_positions[slot] = source + if unpaired: + raise ValueError( + f"CGT incidence anchor found {unpaired} clone household(s) without a " + "paired original at the cgt_incidence_clone field's radix." + ) if len(np.unique(original_positions)) != len(original_positions): raise ValueError( "CGT incidence anchor found two clone households paired to one original." @@ -714,8 +718,8 @@ def _oldest_adult_indices( missing_households = household_ids - set(candidates.person_household_id) if missing_households: raise ValueError( - "Every CGT household requires an adult carrier; missing household " - f"id(s): {sorted(missing_households)[:5]}." + "Every CGT household requires an adult carrier; " + f"{len(missing_households)} household(s) have none." ) candidates["_row"] = candidates.index candidates = candidates.sort_values( @@ -808,7 +812,7 @@ def _assert_cgt_incidence_stage_parameters(stage: SourceStageSpec) -> None: "mass_split": CGT_CLONE_MASS_SPLIT, "weight_kind_out": WeightKind.IMPORTANCE.value, "conservation": "exact_total", - "id_remapping": "id_multiplier_for_values", + "id_remapping": CGT_CLONE_ID_REMAPPING, "declared_factor": 1.0, "reason": CGT_CLONE_MASS_CHANGE_REASON, }, @@ -845,12 +849,11 @@ def cgt_incidence_anchor_operation_parameters() -> tuple[ "pair_clone_households_to_originals", { "clone_flag_column": HOUSEHOLD_IS_CGT_CLONE, - "id_remapping": "id_multiplier_for_values", + "id_remapping": CGT_CLONE_ID_REMAPPING, "pairing": ( - "clone household_id minus the clone stage's id multiplier, " - "recomputed from the pre-clone rows (clone flag false; the " - "support split's copies are pre-clone rows with their own " - "clones)" + "clone household_id minus the cgt_incidence_clone field's " + "declared radix (the support split's copies are pre-clone " + "rows with their own clones)" ), "requirement": ( "bijection between clone and original households; an " diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py index dc9d7c555b..cf4ddbdef4 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/cgt_support.py @@ -10,15 +10,17 @@ descending investable wealth until their weight covers the band's support mass (the published count of gainers at or above GBP 250,000 in that income column, doubled for the clone split and scaled by the declared headroom of -1.25) and divides every selected household into ``ceil(weight / 60)`` -identical copies at equal weight, each carrying its index ``k`` in +1.25) and divides every selected household into ``ceil(K x weight / 60)`` +identical copies at equal weight, ``K`` the frame's geographic clone count +(1 on a national spine; microcosm#931), each carrying its index ``k`` in ``cgt_support_copy_index`` (0 on roots and unsplit households), the explicit -lineage the geography identity kernel keys on. No value moves, no draw is -made, and every household's mass and the total household mass are conserved -exactly; the clone stage then -gives each copy's clone its own identity-keyed prior, and the redraw's -wealth-ranked walk places the light rows where it was already placing heavy -ones. +lineage the geography identity kernel keys on. No value moves, and every +household's mass and the total household mass are conserved exactly; the +one draw is the walk's tie-break, a uniform keyed on each household's own id +that orders households tied on wealth (microcosm#931). The incidence clone +stage (``cgt_incidence_clone``) then gives each copy's clone its own +identity-keyed prior, and the redraw's wealth-ranked walk places the light +rows where it was already placing heavy ones. The support masses are read from the vendored ``hmrc_cgt_conditioning_facts.json`` (HMRC Table 3, individuals by size of gain and taxable income) through @@ -27,7 +29,7 @@ from __future__ import annotations -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass, field from typing import Any @@ -35,6 +37,8 @@ import pandas as pd from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.stochastic_assignment import keyed_tie_order +from microcosm.build.uk_runtime import copy_id_table from microcosm.build.uk_runtime.cgt_imputation import ( UK_CGT_INVESTABLE_WEALTH_COLUMNS, UK_CGT_TAXABLE_INCOME_PROXY_COMPONENTS, @@ -48,6 +52,11 @@ _assert_closed_world_operations, _oldest_adult_indices, ) + +# Moved to geographic_clones, beside the clone index columns' names; this +# stage's node hashes that module (microcosm#931). Kept importable from here +# for this module's other callers. +from microcosm.build.uk_runtime.geographic_clones import geographic_clone_count from microcosm.build.uk_runtime.hmrc_capital_gains import ( HMRC_CGT_CONDITIONING_RESOURCE, HMRC_CGT_INCOME_BAND_LOWER_BOUNDS, @@ -61,7 +70,6 @@ uk_time_period, validate_uk_national_frame, ) -from microcosm.build.uk_runtime.rowwise_geography import id_multiplier_for_values from microcosm.build.uk_runtime.spi_support import ( _exact_total_correction, _importance_weights_with_exact_total, @@ -80,16 +88,36 @@ #: roots and unsplit households: the explicit lineage the geography identity #: kernel keys on (microcosm#932), so no consumer recovers ``k`` from ids. CGT_SUPPORT_COPY_INDEX_COLUMN = "cgt_support_copy_index" -#: The clone stage halves every household and only the clone twin gains. +#: The incidence clone stage (``cgt_incidence_clone``) halves every household +#: and only the clone twin gains. CGT_SUPPORT_CLONE_SPLIT_FACTOR = 2 #: Covers clones whose prior draws at or below zero and the quarter of the #: redraw's rank key the prior carries. CGT_SUPPORT_HEADROOM = 1.25 -#: A copy weighs at most this before the clone (half of it after), so the -#: GBP 5m+ band seats at least a hundred rows. +#: A copy weighs at most this divided by the geographic clone count before +#: the incidence clone (half of that after): at most 60 on a national spine, +#: so the GBP 5m+ band seats at least a hundred rows. CGT_SUPPORT_MAXIMUM_COPY_WEIGHT = 60.0 +#: The copy-id field copy ``k`` takes ``k`` in: ordinals 1 to 9,999, so a +#: household is divided into at most 10,000 rows, root included +#: (microcosm#931). +CGT_SUPPORT_COPY_ID_FIELD = "cgt_support_split" +#: The refusal when a household needs more copies than the field holds. It is +#: fixed: it never names the household, its weight or its copy count. +CGT_SUPPORT_COPY_BOUND_REFUSAL = ( + "The CGT support split would divide a household into more copies than the " + "copy-id table's cgt_support_split field holds (ordinals up to 9,999; " + "microcosm#931). The field is declared, never sized to the frame: a " + "household this heavy needs a reviewed wider field." +) #: The Table 3 bands of gains the support is sized on. CGT_SUPPORT_MINIMUM_GAIN_BAND_LOWER = 250_000 +#: The seed and salt of the selection walk's tie-break: households tied on +#: investable wealth take the order of a uniform keyed on their own +#: household_id, then of their ids, not of their ids alone, which always put +#: a household's lowest copy or clone ordinal first (microcosm#931). +CGT_SUPPORT_TIE_SEED = 0 +CGT_SUPPORT_TIE_SALT = "cgt_support_split:wealth_tie" #: The redraw's income bands, so classification matches its walk. CGT_SUPPORT_INCOME_BAND_LOWER_BOUNDS: tuple[int, ...] = ( HMRC_CGT_INCOME_BAND_LOWER_BOUNDS @@ -107,7 +135,56 @@ "HMRC Table 3 income band into light copies at equal weight; every " "household's mass and the total household mass are conserved." ) +#: The exact-total correction is tried on at most this many of the households +#: a support split left whole, heaviest first, before spi_support's shared +#: fallback (microcosm#931; :func:`_exact_support_weights`, which the +#: high-value home support split shares). Each candidate that cannot carry the +#: correction costs a few dozen sums of the frame's weights. At the CGT split +#: a selected household left at one copy that can carry it also costs a rerun +#: of the selection walk (``keeps_the_split``), so 64 such candidates that the +#: check passes over would cost 64 walks, about 30 to 45 s at the 0.5 to 0.7 s +#: one walk takes on a synthetic vector the size of a ``K = 15`` spine, the +#: more of it tied on wealth the slower, since the walk's tie-break draws the +#: tied households' uniforms and checks the ids it keys them on. A national +#: frame does not reach that, since its 64 heaviest whole households are +#: unselected ones, far heavier than ``60 / K``, so the cap bounds the search +#: there to the sums. +CGT_SUPPORT_CORRECTION_CANDIDATES = 64 +#: The one exception to "every copy carries weight / copies" that the +#: ``copy_rule`` of both support splits declares, built on the cap so the +#: declarations restate it, with the order the candidates are tried in +#: (:func:`_exact_support_weights`; microcosm#931). The CGT split's adds +#: :data:`CGT_SUPPORT_ONE_COPY_CHECK` after it. +CGT_SUPPORT_EXACT_TOTAL_EXCEPTION = ( + f"except where none of the {CGT_SUPPORT_CORRECTION_CANDIDATES} heaviest " + "households left whole, tried heaviest first, tied weights in row order, " + "can carry the exact-total correction, when the shared fallback can put it " + "on a root or copy, off weight / copies by about one ulp of the total weight" +) +#: The clause the CGT split's ``copy_rule`` adds after that exception, built +#: on the cap too. Of those candidates, a selected household left at one copy +#: can carry the correction only where its corrected weight keeps its one +#: copy and the walk's selection (``keeps_the_split`` in +#: :func:`split_cgt_support_households`, kept as a reading of the +#: whole-households-first rule). The shared fallback does not make that +#: check, so it can still put the correction on a one-copy household the +#: check passed over (María's rulings of 2026-11-09; microcosm#931). The +#: high-value home support split makes no such check, so its ``copy_rule`` +#: ends at the exception. +CGT_SUPPORT_ONE_COPY_CHECK = ( + f"of those {CGT_SUPPORT_CORRECTION_CANDIDATES}, a selected household left " + "at one copy can carry the correction only where its corrected weight " + "keeps its one copy and the walk's selection, a check the fallback does " + "not make" +) _HOUSEHOLD_SUPPORT_CHANNEL_COLUMN = "household_support_channel" +#: The geographic clone index the clone-first spine gives every household +#: (``geographic_clones.ladder_clone_index_column("household")``, the +#: ``geographic_support`` branch of the UK copy lineage; microcosm#931). This +#: stage reads ``K`` through ``geographic_clones.geographic_clone_count``, +#: which reads that module's name, and its node hashes that module, so no code +#: here reads this spelling any more; a test holds it equal to the others. +_GEOGRAPHIC_CLONE_INDEX_COLUMN = "household_clone_index" _PERSON_ID_COLUMNS = ("person_id", "person_household_id", "person_benunit_id") _POLICY_PARAMETER_NAMES = ( "gov.hmrc.income_tax.allowances.personal_allowance.amount", @@ -122,8 +199,9 @@ class UKCGTSupportSplitResult: ``band_rows`` carry one mapping per Table 3 income band, in the order of :data:`CGT_SUPPORT_INCOME_BAND_LOWER_BOUNDS`; the totals, the mass pair, - the three parameters and the support-channel split of the selected - households make up the rest of :meth:`evidence`. + the four parameters (the geographic clone count the copies were counted + on among them) and the support-channel split of the selected households + make up the rest of :meth:`evidence`. """ frame: Frame @@ -141,6 +219,7 @@ class UKCGTSupportSplitResult: new_total: float frs_selected: int spi_selected: int + clone_count: int clone_split_factor: int = CGT_SUPPORT_CLONE_SPLIT_FACTOR headroom: float = CGT_SUPPORT_HEADROOM maximum_copy_weight: float = CGT_SUPPORT_MAXIMUM_COPY_WEIGHT @@ -170,6 +249,7 @@ def evidence(self) -> dict[str, object]: "clone_split_factor": int(self.clone_split_factor), "headroom": float(self.headroom), "maximum_copy_weight": float(self.maximum_copy_weight), + "clone_count": int(self.clone_count), }, "support_channel_split": { "frs": int(self.frs_selected), @@ -185,8 +265,10 @@ class UKCGTSupportSplitStageTransform: ``distribution`` defaults to the vendored Table 3 joint; ``parameters`` default to the policyengine-uk allowance parameters at the frame's build period, read at apply time so the base package never imports the engine - at import time. Constructing the transform binds the manifest to the - reviewed operation dictionary and the vendored pins. + at import time. The geographic clone count the copies are counted on is + read from the frame (:func:`geographic_clone_count`). Constructing the + transform binds the manifest to the reviewed operation dictionary and the + vendored pins. """ stage: SourceStageSpec @@ -206,7 +288,12 @@ def __call__(self, frame: Frame) -> Frame: if parameters is None: parameters = uk_cgt_policy_parameters(uk_time_period(frame)) result = split_cgt_support_households( - frame, distribution=distribution, parameters=parameters + frame, + distribution=distribution, + parameters=parameters, + clone_count=geographic_clone_count( + frame.table("household"), stage=CGT_SUPPORT_SPLIT_STAGE_NAME + ), ) object.__setattr__(self, "last_result", result) return result.frame @@ -336,17 +423,24 @@ def select_cgt_support_households( income_band: Sequence[int] | np.ndarray, wealth: Sequence[float] | np.ndarray, support_mass_by_band: Mapping[int, float], + *, + seed: int, + salt: str, ) -> tuple[np.ndarray, tuple[dict[str, object], ...]]: """Select the wealthiest households of each income band, whole. Within each band of ``support_mass_by_band`` (income band lower bound to support mass, walked in mapping order) the positive-weight households of - that band are ordered by wealth descending then household id ascending, + that band are ordered by wealth descending, households tied on wealth by + a uniform keyed on their own household id under ``seed`` and ``salt``, + then by id (:func:`microcosm.build.stochastic_assignment.keyed_tie_order`), and taken whole until the cumulative weight reaches the support mass; the household that crosses it is taken. A pool whose mass falls short is taken entirely and recorded as exhausted, never refused; zero-weight rows are outside every pool. Returns the selection mask aligned to the - inputs and one receipt row per band. + inputs and one receipt row per band. ``seed`` and ``salt`` have no + default: the stage, its rerun of the walk and the identity tool's + recompute each pass the declared pair (:func:`cgt_support_tie_break`). """ ids = np.asarray(household_ids) @@ -377,7 +471,9 @@ def select_cgt_support_households( ) pool = np.flatnonzero((bands == income_lower) & (values > 0.0)) pool_mass = float(values[pool].sum()) - order = pool[np.lexsort((ids[pool], -wealth_values[pool]))] + order = pool[ + keyed_tie_order(-wealth_values[pool], ids[pool], seed=seed, salt=salt) + ] cumulative = np.cumsum(values[order]) if target <= 0.0: take = 0 @@ -406,12 +502,48 @@ def select_cgt_support_households( return selected, tuple(rows) +def _require_clone_count(clone_count: object) -> int: + """``clone_count`` as an int, refused unless it is a positive integer.""" + + if ( + isinstance(clone_count, bool) + or not isinstance(clone_count, int | np.integer) + or clone_count < 1 + ): + raise ValueError( + "The CGT support split needs a positive integer geographic clone " + f"count, got {clone_count!r}." + ) + return int(clone_count) + + def cgt_support_copy_counts( weights: Sequence[float] | np.ndarray, maximum_copy_weight: float = CGT_SUPPORT_MAXIMUM_COPY_WEIGHT, + *, + clone_count: int, ) -> np.ndarray: - """``ceil(weight / maximum_copy_weight)`` per household, at least one.""" + """``max(1, ceil(clone_count x weight / maximum_copy_weight))`` per household. + + ``clone_count`` is the frame's geographic clone count ``K`` + (:func:`geographic_clone_count`; 1 on a national spine). A geographic + clone carries ``1 / K`` of its source household's weight, so ``K x w`` is + its weight scaled back to the source: a clone at exactly ``1 / K`` of its + source's weight gets the count its source gets at ``K = 1``, and each of + its copies weighs at most ``maximum_copy_weight / K`` (microcosm#931). + + For ``K > 1`` the quotient is snapped to the nearest integer when it lies + within ``K`` ulps of it. A clone's weight ``W / K`` is rounded, so when + ``W`` is an exact multiple of ``maximum_copy_weight`` the product + ``K x (W / K)`` can land an ulp above the multiple, and ``ceil`` would add + a copy that ``K = 1`` does not make (at ``K = 7``, for one). The snap + gives up the converse case: a quotient within ``K`` ulps above an integer, + which ``K = 1`` rounds up, counts one copy fewer. At ``K = 1`` no weight + was divided, so nothing is snapped and the count is ``ceil(w / + maximum_copy_weight)`` bit for bit. + """ + clone_count = _require_clone_count(clone_count) if not np.isfinite(maximum_copy_weight) or maximum_copy_weight <= 0.0: raise ValueError( f"CGT support maximum copy weight must be positive, got " @@ -420,31 +552,82 @@ def cgt_support_copy_counts( values = np.asarray(weights, dtype=float) if not np.isfinite(values).all() or (values < 0.0).any(): raise ValueError("CGT support copy counts need finite, non-negative weights.") - counts = np.ceil(values / float(maximum_copy_weight)).astype("int64") + if clone_count == 1: + quotient = values / float(maximum_copy_weight) + else: + quotient = (clone_count * values) / float(maximum_copy_weight) + nearest = np.rint(quotient) + quotient = np.where( + np.abs(quotient - nearest) <= clone_count * np.spacing(nearest), + nearest, + quotient, + ) + counts = np.ceil(quotient).astype("int64") return np.maximum(counts, 1) +def _refuse_copies_beyond_the_copy_id_field(copies: np.ndarray) -> None: + """Refuse a household divided into more rows than its copy-id field holds. + + ``copies`` counts each household's rows, root included, so a household + of ``bound`` rows takes ordinals up to ``bound - 1`` and fits; one more + row does not. The refusal is the fixed :data:`CGT_SUPPORT_COPY_BOUND_REFUSAL`. + """ + + bound = copy_id_table.ordinal_bound(CGT_SUPPORT_COPY_ID_FIELD) + if len(copies) and int(np.max(copies)) > bound: + raise ValueError(CGT_SUPPORT_COPY_BOUND_REFUSAL) + + def split_cgt_support_households( frame: Frame, *, distribution: HMRCCapitalGainsJointDistribution, parameters: UKCGTPolicyParameters, + clone_count: int, headroom: float = CGT_SUPPORT_HEADROOM, ) -> UKCGTSupportSplitResult: """Split the wealthiest households of each income band into light copies. - Deterministic and mass-conserving: no draw, no seed, no salt. Each - selected household of weight ``w`` becomes ``n = ceil(w / 60)`` copies at - ``w / n``; the root keeps its ids, copy ``k`` takes every entity id plus - ``k`` times the frame's id multiplier, records ``k`` in - ``cgt_support_copy_index`` and carries a full copy of the household's - person and benefit-unit rows. The exact-total correction lands on the - heaviest unselected incumbent so every copy weight stays bitwise - ``w / n``. Refuses a frame the clone stage has already seen or one this - stage has already split. + Deterministic and mass-conserving. Its one draw is the selection walk's + tie-break: households tied on wealth take the order of a uniform keyed + on their own household id under the declared seed and salt + (:func:`cgt_support_tie_break`), then of their ids, and the walk's rerun + below passes the same pair. Each + selected household of weight ``w`` becomes ``n = ceil(K x w / 60)`` + copies at ``w / n``, ``K`` the frame's geographic clone count + (``clone_count``, which the stage transform reads from the frame), so a + household at exactly ``1 / K`` of its source's weight gets the ``n`` its + source gets on a one-clone spine, and every copy weighs at most + ``60 / K`` (:func:`cgt_support_copy_counts`). The selection walk still + accumulates each row's own weight, so selection nests only + approximately: the household at which a band's walk reaches its support + mass, and households tied on wealth, can be selected in some clones and + not others, in their keyed order rather than lowest household id first, + so no clone index is favoured. The root keeps its ids, copy + ``k`` takes every entity id plus ``k`` times the declared radix of the + copy-id table's ``cgt_support_split`` field (so every incoming id must + sit below it, and ``n`` is at most the field's bound of 10,000), records + ``k`` in ``cgt_support_copy_index`` and carries a full copy of the + household's person and benefit-unit rows. The exact-total correction + lands on a household the split left whole (an unselected one, or a + selected one left at one copy): the first of the + :data:`CGT_SUPPORT_CORRECTION_CANDIDATES` heaviest that can carry it, so + every root and copy weighs bitwise ``w / n``. A selected one carries it + only where its corrected weight leaves the walk's selection and its one + copy as they were, so wherever one of those candidates carries it, the + rule rerun on the output divides the same households. Only where none of + them can does spi_support's shared fallback place it, unchanged and + without that check. Its row can then be a root or copy, off ``w / n`` by + about one ulp of the total, or a whole household, among them a one-copy + one the check passed over, after which the rule rerun on the output can + divide a different set (:func:`_exact_support_weights`). Refuses a frame + the incidence clone stage (``cgt_incidence_clone``) has already seen or + one this stage has already split. """ validate_uk_national_frame(frame) + clone_count = _require_clone_count(clone_count) person = frame.table("person").copy() benunit = frame.table("benunit").copy() household = frame.table("household").copy() @@ -479,20 +662,32 @@ def split_cgt_support_households( support_by_band = { int(row["income_lower_bound"]): float(row["support_mass"]) for row in mass_rows } + tie_seed, tie_salt = cgt_support_tie_break() selected, selection_rows = select_cgt_support_households( - household_ids, weights, income_band, wealth, support_by_band + household_ids, + weights, + income_band, + wealth, + support_by_band, + seed=tie_seed, + salt=tie_salt, ) copies = np.ones(len(household), dtype="int64") copies[selected] = cgt_support_copy_counts( - weights[selected], CGT_SUPPORT_MAXIMUM_COPY_WEIGHT - ) - multiplier = id_multiplier_for_values( - person["person_id"], - person["person_household_id"], - person["person_benunit_id"], - benunit["benunit_id"], - household["household_id"], + weights[selected], CGT_SUPPORT_MAXIMUM_COPY_WEIGHT, clone_count=clone_count ) + _refuse_copies_beyond_the_copy_id_field(copies) + for table, columns in ( + (person, _PERSON_ID_COLUMNS), + (benunit, ("benunit_id",)), + (household, ("household_id",)), + ): + for column in columns: + copy_id_table.require_below( + table[column], + CGT_SUPPORT_COPY_ID_FIELD, + what=f"incoming {column} values", + ) household[HOUSEHOLD_IS_CGT_SUPPORT_COPY] = False household[CGT_SUPPORT_COPIES_COLUMN] = copies household[CGT_SUPPORT_COPY_INDEX_COLUMN] = np.zeros(len(household), dtype="int64") @@ -510,14 +705,15 @@ def split_cgt_support_households( benunit["benunit_id"].isin(set(family_person["person_benunit_id"])) ].copy() family_household = household.loc[family].copy() - offset = k * multiplier for column in _PERSON_ID_COLUMNS: - family_person[column] = family_person[column].astype("int64") + offset - family_benunit["benunit_id"] = ( - family_benunit["benunit_id"].astype("int64") + offset + family_person[column] = copy_id_table.mint( + family_person[column], CGT_SUPPORT_COPY_ID_FIELD, k + ) + family_benunit["benunit_id"] = copy_id_table.mint( + family_benunit["benunit_id"], CGT_SUPPORT_COPY_ID_FIELD, k ) - family_household["household_id"] = ( - family_household["household_id"].astype("int64") + offset + family_household["household_id"] = copy_id_table.mint( + family_household["household_id"], CGT_SUPPORT_COPY_ID_FIELD, k ) family_household[HOUSEHOLD_IS_CGT_SUPPORT_COPY] = True family_household[CGT_SUPPORT_COPY_INDEX_COLUMN] = np.full( @@ -531,10 +727,45 @@ def split_cgt_support_households( final_benunit = pd.concat([benunit, *copy_benunit], ignore_index=True) final_household = pd.concat([household, *copy_household], ignore_index=True) final_weights = np.concatenate([incumbent_weights, *copy_weights]) + + def keeps_the_split(row: int, weight: float) -> bool: + """Whether whole household ``row`` at ``weight`` leaves the split as it was. + + An unselected household sits past its band's crossing, or in no + walked band, so its weight moves neither the walk nor its count of + one. A selected household left at one copy can move both, so it + carries the correction only where the walk rerun on its corrected + weight, under the stage's declared tie-break, selects the same + households and that weight still counts one copy: the rule rerun on + the stage's output then divides the same households the same way, as + the identity tool's recompute does. + """ + + if not selected[row]: + return True + counted = cgt_support_copy_counts( + [weight], CGT_SUPPORT_MAXIMUM_COPY_WEIGHT, clone_count=clone_count + ) + if int(counted[0]) != 1: + return False + moved = weights.copy() + moved[row] = weight + reselected, _ = select_cgt_support_households( + household_ids, + moved, + income_band, + wealth, + support_by_band, + seed=tie_seed, + salt=tie_salt, + ) + return bool(np.array_equal(reselected, selected)) + exact = _exact_support_weights( final_weights, target=old_total, - unselected=np.flatnonzero(~selected & (incumbent_weights > 0.0)), + whole=np.flatnonzero((copies == 1) & (incumbent_weights > 0.0)), + keeps_the_split=keeps_the_split, ) receipt = MassChangeRecord( entity="household", @@ -604,25 +835,56 @@ def split_cgt_support_households( new_total=float(exact.total), frs_selected=frs_selected, spi_selected=spi_selected, + clone_count=clone_count, ) def _exact_support_weights( - values: np.ndarray, *, target: float, unselected: np.ndarray + values: np.ndarray, + *, + target: float, + whole: np.ndarray, + keeps_the_split: Callable[[int, float], bool] | None = None, ) -> Weights: """Importance weights summing bitwise to ``target``. - The correction lands on the heaviest unselected incumbent so every split - weight stays exactly ``w / n``; a frame with no unselected positive - incumbent falls back to the shared multi-candidate correction. + ``whole`` gives the positions of the households a support split left + whole: the positive-weight rows it did not divide, whose weights it left + as they were (the unselected households, and at the CGT split the + selected ones left at one copy). The correction is tried on the + :data:`CGT_SUPPORT_CORRECTION_CANDIDATES` heaviest of them, heaviest + first and tied weights in row order, and lands on the first that can + carry it, so every root and copy of a divided household keeps bitwise + ``w / n`` (microcosm#931). A candidate carries it where the correction + is representable on it and ``keeps_the_split``, where given, accepts its + corrected weight: the CGT split refuses a selected household left at one + copy whose corrected weight would move the walk's selection or count a + second copy. The first candidate is the heaviest unselected household, + the one corrected before, wherever no selected household left at one + copy outweighs it or ties it from an earlier row; a total already exact + comes back unchanged. + + Only where none of them can carry it, or there is none, does + spi_support's shared multi-candidate correction place it, unchanged: its + candidates (the heaviest, last, first, lightest and middle positive rows) + can be a root or copy of a divided household, which is then off ``w / n`` + by about one ulp of the total, or a whole household, a one-copy household + that check refused among them, since it does not consult + ``keeps_the_split``. On the CGT nesting scenario at 7 and 13 clones, for + one, every whole household sits in the half of NumPy's pairwise sum whose + other half carries exactly half an ulp of the total, so no change to its + weight rounds the total onto the target. """ - if unselected.size: - heaviest = int(unselected[np.argmax(values[unselected])]) + rows = np.asarray(whole, dtype=np.int64) + heaviest_first = rows[np.lexsort((rows, -values[rows]))] + for index in heaviest_first[:CGT_SUPPORT_CORRECTION_CANDIDATES].tolist(): corrected = _exact_total_correction( - values, target=target, correction_index=heaviest + values, target=target, correction_index=index ) - if corrected is not None: + if corrected is not None and ( + keeps_the_split is None or keeps_the_split(index, float(corrected[index])) + ): return Weights(corrected, WeightKind.IMPORTANCE) return _importance_weights_with_exact_total(values, target) @@ -673,24 +935,36 @@ def cgt_support_split_operation_parameters() -> tuple[ "investable_wealth_columns": list(UK_CGT_INVESTABLE_WEALTH_COLUMNS), "selection": ( "within each income band walk households in investable wealth " - "descending, household_id ascending, accumulating positive " - "household weight until the cumulative weight reaches the " - "band's support mass; whole households; an exhausted pool is " - "recorded, not refused" + "descending, households tied on it in ascending order of a " + "uniform keyed on their own household_id under seed and salt, " + "then household_id ascending, accumulating positive household " + "weight until the cumulative weight reaches the band's support " + "mass; whole households; an exhausted pool is recorded, not " + "refused" ), + "seed": CGT_SUPPORT_TIE_SEED, + "salt": CGT_SUPPORT_TIE_SALT, "maximum_copy_weight": CGT_SUPPORT_MAXIMUM_COPY_WEIGHT, "copy_rule": ( - "copies = ceil(household weight / maximum_copy_weight); every " - "copy carries weight / copies" + "copies = ceil(geographic clone count x household weight / " + "maximum_copy_weight), the quotient snapped to the nearest " + "integer within clone-count ulps when the clone count " + "exceeds one; every copy carries weight / copies, " + f"{CGT_SUPPORT_EXACT_TOTAL_EXCEPTION}; " + f"{CGT_SUPPORT_ONE_COPY_CHECK}" ), "id_remapping": ( - "id_multiplier_for_values; copy k takes every entity id + k x " - "multiplier" + "copy-id table field cgt_support_split; copy k takes every " + "entity id + k x the field's radix (10**13), k up to 9,999, " + "and a household needing more copies is refused" ), "flag_column": HOUSEHOLD_IS_CGT_SUPPORT_COPY, "copies_column": CGT_SUPPORT_COPIES_COLUMN, "copy_index_column": CGT_SUPPORT_COPY_INDEX_COLUMN, - "values": "every column copied unchanged; no draw, no seed, no salt", + "values": ( + "every column copied unchanged; the selection's tie-break is " + "the stage's only draw" + ), "weight_kind_out": WeightKind.IMPORTANCE.value, "conservation": "exact_total", "declared_factor": 1.0, @@ -700,6 +974,19 @@ def cgt_support_split_operation_parameters() -> tuple[ ) +def cgt_support_tie_break() -> tuple[int, str]: + """The declared seed and salt of the selection walk's tie-break. + + Read from the reviewed operation the manifests restate verbatim + (:func:`cgt_support_split_operation_parameters`), so the stage's walk, its + rerun of the walk in ``keeps_the_split`` and the identity tool's + recompute all key their ties on the declared pair (microcosm#931). + """ + + ((_, operation),) = cgt_support_split_operation_parameters() + return int(operation["seed"]), str(operation["salt"]) + + def _assert_cgt_support_split_stage_parameters(stage: SourceStageSpec) -> None: """Bind every stage manifest parameter to reviewed code constants. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/childcare_copy_take_up.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/childcare_copy_take_up.py index 48cc10a4ad..741248a783 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/childcare_copy_take_up.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/childcare_copy_take_up.py @@ -1,7 +1,8 @@ """Re-draw the childcare take-up of support and CGT copies on their own units. ``frs_take_up`` draws the childcare flags and the extended-hours usage keyed on -each FRS benefit unit before any copy exists, so every copy the later stages +each FRS benefit unit, each geographic clone's on its own ids +(microcosm#931), before any other copy exists, so every copy the later stages make (the SPI support channel and its income-band donors, the high-value home support, the CGT support split, the CGT incidence clone and the residential arms) inherits its host's draws. A small cell of hosts then fixes the take-up @@ -22,8 +23,10 @@ id in the frame, so a copy's id is unique and differs from every native id: the copy's streams are independent of its host's, and a rebuild of the same frame draws the same values. Native benefit units keep their ``frs_take_up`` -draws untouched. Anchored flags (Child Benefit, Pension Credit, Universal -Credit) and ``would_claim_uc_childcare`` are left as they are. +draws untouched, every geographic clone's included, since a geographic +clone is not a copy here (``without_geographic_branch``). Anchored flags +(Child Benefit, Pension Credit, Universal Credit) and +``would_claim_uc_childcare`` are left as they are. A copy is a benefit unit whose household carries a non-empty clone path (``uk_household_clone_paths``: the SPI support channel, the CGT support split, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/copy_id_table.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/copy_id_table.py new file mode 100644 index 0000000000..20bbb57881 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/copy_id_table.py @@ -0,0 +1,251 @@ +"""Fixed decimal fields for the ids of UK copy rows (microcosm#931). + +Every UK copy layer used to offset its copies' ids by a power of ten one digit +above the largest id it saw. Once the spine clones households at its start, +that largest id grows with the clone count K, so every later offset would move +with K and the K=1 spine would stop being the clone-0 slice of the K=15 spine. +Here each layer owns a decimal field of the id instead, declared up front: +copy ``k`` of a row in field ``f`` takes ``id + k x radix(f)`` on all five id +columns (``household_id``; ``person_id``, ``person_household_id`` and +``person_benunit_id``; ``benunit_id``), so every copy id is a pure function of +its source id and its copy path. + +The fields are listed least significant first, in the order the spine applies +them, each with its ordinal bound (ordinals ``0 .. bound - 1``, 0 meaning "not +a copy in this field") and its radix: + +- ``vintage``: reserved for pooled FRS vintages (the plan's P3). The spine + reads one FRS release today, so nothing mints it; raw FRS ids + (``household_id`` the sernum, ``benunit_id`` sernum x 100 + b, + ``person_id`` sernum x 1000 + p) must stay below its radix. +- ``geographic_support``: the geographic clones the spine's + ``geographic_expand`` stage mints before any other copy layer runs (K at + most 100). +- ``spi_support_channel``: the SPI support copy (ordinal 1, stage 16) and the + income band donor (ordinal 2, stage 17). +- ``high_value_support``: the high-value home copies (stage 26). +- ``cgt_support_split``: the CGT support copies (stage 42), four digits. +- ``cgt_incidence_clone``: the CGT incidence twin (stage 43). +- ``cgt_residential_split``: the residential arms (stage 46), the top field, + so arms follow every incumbent in id order. + +A field must sit above every field filled before it runs, which keeps a copy's +parent below the field's radix: a copy's ordinal is then ``id // radix`` +modulo the field's width, and its parent ``id - ordinal x radix``. The table +is UK-local; a shared, country-neutral table comes with a later stage. This +module imports nothing from the UK runtime, so any module can read it. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from types import MappingProxyType + +import numpy as np + +__all__ = [ + "UK_COPY_ID_BASE_RADIX", + "UK_COPY_ID_FIELDS", + "UK_COPY_ID_LARGEST_ID", + "CopyIdField", + "mint", + "ordinal", + "ordinal_bound", + "parent", + "radix", + "require_below", +] + +_INT64_MAX = int(np.iinfo(np.int64).max) + + +@dataclass(frozen=True) +class CopyIdField: + """One decimal field of a copy id: ordinals ``0 .. bound - 1`` at ``radix``.""" + + name: str + bound: int + radix: int + + @property + def width(self) -> int: + """The field's span, ``10 ** digits(bound - 1)``.""" + + return 10 ** len(str(self.bound - 1)) + + +#: Raw ids sit below this radix, the first field's. +UK_COPY_ID_BASE_RADIX = 10**8 +#: The fields in the order the spine applies them, least significant first. +UK_COPY_ID_FIELDS: tuple[CopyIdField, ...] = ( + CopyIdField("vintage", 10, 10**8), + CopyIdField("geographic_support", 100, 10**9), + CopyIdField("spi_support_channel", 3, 10**11), + CopyIdField("high_value_support", 6, 10**12), + CopyIdField("cgt_support_split", 10_000, 10**13), + CopyIdField("cgt_incidence_clone", 2, 10**17), + CopyIdField("cgt_residential_split", 8, 10**18), +) + + +def _largest_id(fields: Sequence[CopyIdField], base_radix: int) -> int: + """The largest id the fields can mint: every ordinal at its bound less one.""" + + return sum((field.bound - 1) * field.radix for field in fields) + base_radix - 1 + + +def _validate_fields(fields: Sequence[CopyIdField], base_radix: int) -> int: + """Refuse a malformed table; return its largest mintable id. + + The first radix is the base radix; every later radix is the previous one + times the previous field's width, so the fields are contiguous decimal + digits and none overlaps another; the largest mintable id fits int64. + """ + + names = [field.name for field in fields] + if not names or len(set(names)) != len(names): + raise ValueError("The copy-id fields must be named, each once.") + expected = base_radix + for field in fields: + if type(field.bound) is not int or field.bound < 2: + raise ValueError( + f"Copy-id field {field.name!r} needs an integer bound of at least 2." + ) + if field.radix != expected: + raise ValueError( + f"Copy-id field {field.name!r} sits at radix {field.radix}, not " + f"{expected}, so the fields are not contiguous decimal digits." + ) + expected = field.radix * field.width + largest = _largest_id(fields, base_radix) + if largest >= _INT64_MAX: + raise ValueError( + f"The copy-id fields can mint ids up to {largest}, which int64 cannot hold." + ) + return largest + + +#: About 7.2e18, below int64's 2**63 - 1. +UK_COPY_ID_LARGEST_ID = _validate_fields(UK_COPY_ID_FIELDS, UK_COPY_ID_BASE_RADIX) +_FIELDS = MappingProxyType({field.name: field for field in UK_COPY_ID_FIELDS}) + + +def _field(name: str) -> CopyIdField: + try: + return _FIELDS[name] + except KeyError as error: + raise ValueError(f"No UK copy-id field is named {name!r}.") from error + + +def radix(field: str) -> int: + """The declared radix of ``field``.""" + + return _field(field).radix + + +def ordinal_bound(field: str) -> int: + """The declared bound of ``field``: its ordinals run below it.""" + + return _field(field).bound + + +def _ids(values: object, *, what: str) -> np.ndarray: + """Exact int64 ids, refusing floats, booleans and negative ids by count.""" + + array = np.asarray(values) + if array.dtype.kind == "O": + items = array.ravel().tolist() + if any( + isinstance(item, bool | np.bool_) or not isinstance(item, int | np.integer) + for item in items + ): + raise TypeError(f"{what} must be integers.") + if any(not -_INT64_MAX - 1 <= int(item) <= _INT64_MAX for item in items): + raise ValueError(f"{what} must fit int64.") + array = np.asarray([int(item) for item in items], dtype=np.int64).reshape( + array.shape + ) + elif array.dtype.kind == "u": + if array.size and int(array.max()) > _INT64_MAX: + raise ValueError(f"{what} must fit int64.") + elif array.dtype.kind != "i": + raise TypeError(f"{what} must be integers, not {array.dtype}.") + result = array.astype(np.int64, copy=False) + negative = int((result < 0).sum()) + if negative: + raise ValueError(f"{negative} of the {what} are negative.") + return result + + +def require_below(ids: object, field: str, *, what: str = "ids") -> None: + """Refuse any id at or above ``field``'s radix, printing only the count. + + A stage that mints ``field`` copies rows whose ids must all sit below it, + so that every copy's parent is recoverable from the copy's id; the raw + FRS ids sit below the first field, ``vintage``. + """ + + values = _ids(ids, what=what) + reached = int((values >= radix(field)).sum()) + if reached: + raise ValueError( + f"{reached} of the {what} reach the {field!r} copy-id field " + f"(radix {radix(field)}): ids from that radix up are reserved for " + "rows minted in that field or a later one (microcosm#931)." + ) + + +def ordinal(ids: object, field: str) -> np.ndarray: + """Each id's ordinal in ``field``: ``(id // radix) % width``.""" + + spec = _field(field) + return (_ids(ids, what="ids") // spec.radix) % spec.width + + +def parent(ids: object, field: str) -> np.ndarray: + """Each id with its ``field`` digits cleared: the row it was copied from.""" + + values = _ids(ids, what="ids") + spec = _field(field) + return values - ((values // spec.radix) % spec.width) * spec.radix + + +def mint(ids: object, field: str, ordinals: object) -> np.ndarray: + """``ids + ordinals x radix(field)`` as int64, refusing every malformed copy. + + Refuses an ordinal outside ``[1, bound)`` in a fixed sentence (it names + the field and its declared bound, never how many copies a row needed), + an id whose ``field`` digits are already set (a copy of a copy in the same + field), and a result int64 cannot hold. ``ordinals`` is one integer or one + per id. The other messages carry counts and declared constants only, + never an id. + """ + + spec = _field(field) + values = _ids(ids, what="ids") + steps = _ids(ordinals, what=f"{field!r} ordinals") + try: + steps = np.broadcast_to(steps, values.shape) + except ValueError as error: + raise ValueError( + f"{field!r} ordinals must be one integer or one per id." + ) from error + if ((steps < 1) | (steps >= spec.bound)).any(): + raise ValueError( + f"A {field!r} copy ordinal falls outside [1, {spec.bound}), the " + "field's declared bound (microcosm#931)." + ) + occupied = int((((values // spec.radix) % spec.width) != 0).sum()) + if occupied: + raise ValueError( + f"{occupied} id(s) already carry a {field!r} ordinal; a copy is minted " + "from a row that is not a copy in its own field (microcosm#931)." + ) + offsets = steps * spec.radix + overflow = int((values > _INT64_MAX - offsets).sum()) + if overflow: + raise ValueError( + f"{overflow} {field!r} copy id(s) would overflow int64 (microcosm#931)." + ) + return values + offsets diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/council_tax_band_d.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/council_tax_band_d.py new file mode 100644 index 0000000000..6daab608d7 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/council_tax_band_d.py @@ -0,0 +1,522 @@ +"""2024-25 Band D council tax for Great Britain, and the statutory band formula (microcosm#931). + +Council-tax localisation prices a household's bill at the local authority the +spine draws for it. That needs each authority's Band D for the build's fiscal +year and the statute that turns a Band D into the bill of a band. Band D comes +from two committed resources, both keyed by the April 2023 codes of the local +authority roster ``local_authority_names.json``, through which the spine +resolves every drawn area: + +- ``council_tax_band_d_england_wales_2024_25.json``: England's and Wales's + 2024-25 Band D, built from two raw-only Chronicle registrations by + ``tools/build_uk_council_tax_band_d.py``. England's amounts are at pence; + Wales's are the published decimals, unrounded. +- ``scotgov_band_d_council_tax_2024_25.json``: Scotland's 2024-25 average + Band D for its 32 council areas (CTAS 2026, the record set + ``scotgov.ctas2026.band_d.fy2024``), vendored from the pinned Chronicle feed + by ``tools/vendor_uk_ledger_facts.py``. Its rows are read through + ``vendored_rows``, which refuses a copy taken from another feed before it + reads a row. Each amount is the loaded binary number; it must lie within + 1e-9 of a whole number of pence and is kept as that pence. + +:func:`load_uk_council_tax_band_d` merges the two as exact rationals, each read +as a :class:`fractions.Fraction` through :class:`decimal.Decimal`. It refuses a +code outside the roster's Great Britain codes, a Northern Ireland code +(Northern Ireland levies domestic rates, not council tax), a missing or +repeated authority, and a value that is not a positive finite decimal. Every +value is used as published: none is overridden. + +The statute (:func:`uk_statutory_council_tax`) multiplies an authority's Band +D by its band's proportion, by nation. England and Wales take 6, 7, 8, 9, 11, +13, 15 and 18 ninths for bands A to H (Local Government Finance Act 1992, +section 5), and Wales 21 ninths for band I (the Council Tax (Valuation Bands) +(Wales) Order 2003, from 2005-06). Scotland takes 240, 280, 320, 360, 473, 585, +705 and 882 three-hundred-and-sixtieths for A to H (section 74, as the Council +Tax (Substitution of Proportion) (Scotland) Order 2016 substituted it from +2017-18). A dwelling with a single adult resident takes the 25 per cent +discount (sections 11 and 79). The nations and each nation's bands are +``council_tax_constants``' ``UK_COUNCIL_TAX_BANDS_BY_NATION``, read here and +never edited. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +from collections.abc import Mapping +from dataclasses import dataclass +from decimal import Decimal +from fractions import Fraction +from importlib.resources import files +from types import MappingProxyType +from typing import Any + +from microcosm.build.uk_runtime.council_tax_constants import ( + UK_COUNCIL_TAX_BANDS_BY_NATION, +) +from microcosm.build.uk_runtime.ledger_fact_vendoring import ( + VENDORED_RESOURCE_KIND, + load_vendored_resource, + vendored_rows, + verify_vendored_resource_feed_identity, +) +from microcosm.build.uk_runtime.local_authority_input import ( + UK_LOCAL_AUTHORITY_VINTAGE, + load_uk_local_authority_names_resource, +) + +__all__ = [ + "SCOTLAND_BAND_D_ARTIFACT_ROLE", + "SCOTLAND_BAND_D_CONCEPT", + "SCOTLAND_BAND_D_GEOGRAPHY_VINTAGE", + "SCOTLAND_BAND_D_RECORD_SETS", + "SCOTLAND_BAND_D_RESOURCE", + "UK_BAND_D_ARTIFACT_ROLE", + "UK_BAND_D_BUILD_PERIOD", + "UK_BAND_D_ENGLAND_WALES_KIND", + "UK_BAND_D_ENGLAND_WALES_METHOD", + "UK_BAND_D_ENGLAND_WALES_RESOURCE", + "UK_BAND_D_ENGLAND_WALES_SCHEMA_VERSION", + "UK_BAND_D_FISCAL_YEAR", + "UK_BAND_D_PENCE_TOLERANCE", + "UK_BAND_D_PENCE_TOLERANCE_TEXT", + "UK_COUNCIL_TAX_BAND_RATIOS", + "UK_COUNCIL_TAX_SINGLE_ADULT_DISCOUNT", + "UKCouncilTaxBandD", + "UKCouncilTaxBandDError", + "band_d_decimal", + "band_d_pence", + "check_scotland_band_d_declaration", + "load_uk_council_tax_band_d", + "uk_statutory_council_tax", +] + +#: England's and Wales's Band D: built by ``tools/build_uk_council_tax_band_d.py`` +#: from the raw-only registrations, never edited by hand. +UK_BAND_D_ENGLAND_WALES_RESOURCE = "council_tax_band_d_england_wales_2024_25.json" +UK_BAND_D_ENGLAND_WALES_KIND = "uk_council_tax_band_d_england_wales" +UK_BAND_D_ENGLAND_WALES_SCHEMA_VERSION = 1 +UK_BAND_D_ENGLAND_WALES_METHOD = "uk-council-tax-band-d-v1" +#: The artifact role under which a stage declares the England-and-Wales resource. +UK_BAND_D_ARTIFACT_ROLE = "council_tax_band_d" + +#: Scotland's Band D: vendored from the pinned Chronicle feed by +#: ``tools/vendor_uk_ledger_facts.py``, never edited by hand. +SCOTLAND_BAND_D_RESOURCE = "scotgov_band_d_council_tax_2024_25.json" +#: The artifact role under which a stage declares the Scottish resource. +SCOTLAND_BAND_D_ARTIFACT_ROLE = "scotland_band_d_facts" +#: Every Chronicle record set read from the Scottish resource, as a stage's +#: ``verify_vendored_fact_resource`` operation declares them. +SCOTLAND_BAND_D_RECORD_SETS: tuple[str, ...] = ("scotgov.ctas2026.band_d.fy2024",) +SCOTLAND_BAND_D_CONCEPT = "scotgov.council_tax.average_band_d" +#: The council-area vintage of the Scottish rows: the 2019 council areas, which +#: the April 2023 roster carries for Scotland. +SCOTLAND_BAND_D_GEOGRAPHY_VINTAGE = "ca_2019" +_SCOTLAND_UNIT = "gbp" +_SCOTLAND_LEVEL = "local_authority" + +#: The fiscal year of both resources, and the build period (its opening year) +#: the stages map them to. +UK_BAND_D_FISCAL_YEAR = "2024-25" +UK_BAND_D_BUILD_PERIOD = 2024 + +#: The farthest, in pounds, a stored binary amount may sit from a whole number +#: of pence and still be kept as that pence. +UK_BAND_D_PENCE_TOLERANCE_TEXT = "1e-9" +UK_BAND_D_PENCE_TOLERANCE = Fraction(Decimal(UK_BAND_D_PENCE_TOLERANCE_TEXT)) + +#: A GSS code's first letter names its nation. The England-and-Wales resource +#: covers E and W codes, the Scottish resource S codes, and Northern Ireland's N +#: codes have no Band D. +_ENGLAND_WALES_LETTERS = "EW" +_SCOTLAND_LETTERS = "S" +_NORTHERN_IRELAND_LETTER = "N" + +_DECIMAL = re.compile(r"[0-9]+(?:\.[0-9]+)?") + + +class UKCouncilTaxBandDError(ValueError): + """Raised when the Band D resources or the statute's inputs are refused.""" + + +# ---------------------------------------------------------------------------- +# The statute + + +def _proportions( + nation: str, numerators: tuple[int, ...], denominator: int +) -> Mapping[str, Fraction]: + """A nation's band proportions to Band D, over the bands it uses.""" + + return MappingProxyType( + { + band: Fraction(numerator, denominator) + for band, numerator in zip( + UK_COUNCIL_TAX_BANDS_BY_NATION[nation], numerators, strict=True + ) + } + ) + + +_NINTHS = (6, 7, 8, 9, 11, 13, 15, 18) + +#: Each nation's statutory band proportions to Band D, by the engine's +#: ``country`` member: England's and Wales's ninths (Wales's band I since +#: 2005-06), and Scotland's three-hundred-and-sixtieths since 2017-18. +UK_COUNCIL_TAX_BAND_RATIOS: Mapping[str, Mapping[str, Fraction]] = MappingProxyType( + { + "ENGLAND": _proportions("ENGLAND", _NINTHS, 9), + "WALES": _proportions("WALES", (*_NINTHS, 21), 9), + "SCOTLAND": _proportions( + "SCOTLAND", (240, 280, 320, 360, 473, 585, 705, 882), 360 + ), + } +) + +#: The discount on the bill of a dwelling with a single adult resident, in every +#: nation that levies council tax. +UK_COUNCIL_TAX_SINGLE_ADULT_DISCOUNT = Fraction(1, 4) + + +def _exact_band_d(value: object) -> Fraction: + exact = isinstance(value, int | Fraction | Decimal) and not isinstance(value, bool) + if not exact: + raise UKCouncilTaxBandDError( + "Band D must be an exact rational (an int, Fraction or Decimal), " + f"not a {type(value).__name__}." + ) + if isinstance(value, Decimal) and not value.is_finite(): + raise UKCouncilTaxBandDError("Band D must be finite.") + amount = Fraction(value) + if amount <= 0: + raise UKCouncilTaxBandDError("Band D must be positive.") + return amount + + +def uk_statutory_council_tax( + band_d: Fraction | int | Decimal, nation: str, band: str, *, single_adult: bool +) -> Fraction: + """A dwelling's annual council tax under the statute, before any reduction. + + ``band_d`` is its authority's Band D in pounds, as an exact rational; + ``nation`` is the engine's ``country`` member and ``band`` one of that + nation's bands. The bill is Band D times the band's proportion, less + ``UK_COUNCIL_TAX_SINGLE_ADULT_DISCOUNT`` of it where ``single_adult``. + """ + + proportions = UK_COUNCIL_TAX_BAND_RATIOS.get(nation) + if proportions is None: + if nation in UK_COUNCIL_TAX_BANDS_BY_NATION: + raise UKCouncilTaxBandDError( + f"{nation} levies domestic rates, not council tax." + ) + raise UKCouncilTaxBandDError(f"{nation!r} is not a council-tax nation.") + if band not in proportions: + raise UKCouncilTaxBandDError(f"{nation} has no council-tax band {band!r}.") + if type(single_adult) is not bool: + raise UKCouncilTaxBandDError("single_adult must be a bool.") + bill = _exact_band_d(band_d) * proportions[band] + if single_adult: + bill *= 1 - UK_COUNCIL_TAX_SINGLE_ADULT_DISCOUNT + return bill + + +# ---------------------------------------------------------------------------- +# Amounts + + +def band_d_decimal(text: object, *, where: str) -> Fraction: + """A Band D written as a decimal, as an exact rational. + + The text must be digits with an optional fraction (no sign, exponent or + separator), and the amount positive. ``where`` names the amount in a + refusal, which never quotes it. + """ + + if not isinstance(text, str) or not _DECIMAL.fullmatch(text): + raise UKCouncilTaxBandDError(f"{where} is not a positive decimal amount.") + amount = Fraction(Decimal(text)) + if amount <= 0: + raise UKCouncilTaxBandDError(f"{where} is not a positive decimal amount.") + return amount + + +def band_d_pence(value: object, *, where: str) -> tuple[str, bool]: + """A stored binary amount as its pence decimal, and whether it was exactly that. + + The amount must be a positive finite number within + ``UK_BAND_D_PENCE_TOLERANCE`` of a whole number of pence. ``where`` names + the amount in a refusal, which never quotes it. + """ + + if isinstance(value, bool) or not isinstance(value, int | float): + raise UKCouncilTaxBandDError(f"{where} is not a number.") + if not math.isfinite(value) or value <= 0: + raise UKCouncilTaxBandDError(f"{where} is not a positive finite amount.") + exact = Fraction(value) + pence = round(exact * 100) + if pence <= 0 or abs(exact - Fraction(pence, 100)) > UK_BAND_D_PENCE_TOLERANCE: + raise UKCouncilTaxBandDError( + f"{where} is not within {UK_BAND_D_PENCE_TOLERANCE_TEXT} of a whole " + "number of pence." + ) + return str(Decimal(pence).scaleb(-2)), exact == Fraction(pence, 100) + + +def _nation_code(code: object, letters: str, *, resource: str) -> str: + """A code the resource may carry: one of ``letters``' nations, never N.""" + + if not isinstance(code, str) or not code: + raise UKCouncilTaxBandDError( + f"{resource} carries a Band D without a local authority code." + ) + if code.startswith(_NORTHERN_IRELAND_LETTER): + raise UKCouncilTaxBandDError( + f"{resource} carries the Northern Ireland code {code!r}; Northern " + "Ireland levies domestic rates, not council tax." + ) + if code[0] not in letters: + raise UKCouncilTaxBandDError( + f"{resource} carries {code!r}, a code of a nation it does not cover." + ) + return code + + +# ---------------------------------------------------------------------------- +# The resources + + +def _unique_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + """A JSON object's members, refusing a key written twice.""" + + members: dict[str, Any] = {} + for key, value in pairs: + if key in members: + raise UKCouncilTaxBandDError( + f"{UK_BAND_D_ENGLAND_WALES_RESOURCE} repeats the key {key!r}." + ) + members[key] = value + return members + + +def _england_wales_payload( + payload: Mapping[str, Any] | None, +) -> tuple[Mapping[str, Any], str]: + if payload is not None: + return payload, "supplied" + data = ( + files("microcosm.build.uk") + .joinpath(UK_BAND_D_ENGLAND_WALES_RESOURCE) + .read_bytes() + ) + return ( + json.loads(data, object_pairs_hook=_unique_keys), + hashlib.sha256(data).hexdigest(), + ) + + +def _england_wales_band_d(payload: Mapping[str, Any]) -> dict[str, Fraction]: + resource = UK_BAND_D_ENGLAND_WALES_RESOURCE + for key, expected in ( + ("schema_version", UK_BAND_D_ENGLAND_WALES_SCHEMA_VERSION), + ("kind", UK_BAND_D_ENGLAND_WALES_KIND), + ("resource", UK_BAND_D_ENGLAND_WALES_RESOURCE), + ("fiscal_year", UK_BAND_D_FISCAL_YEAR), + ("local_authority_vintage", UK_LOCAL_AUTHORITY_VINTAGE), + ): + actual = payload.get(key) + if type(actual) is not type(expected) or actual != expected: + raise UKCouncilTaxBandDError(f"{resource}: {key} must be {expected!r}.") + band_d = payload.get("band_d") + if not isinstance(band_d, Mapping) or not band_d: + raise UKCouncilTaxBandDError(f"{resource}: band_d must be a non-empty mapping.") + amounts: dict[str, Fraction] = {} + for code, text in band_d.items(): + _nation_code(code, _ENGLAND_WALES_LETTERS, resource=resource) + where = f"{resource}: the Band D of {code!r}" + amounts[code] = band_d_decimal(text, where=where) + return amounts + + +def _scotland_rows( + payload: Mapping[str, Any] | None, *, verify_feed_identity: bool +) -> tuple[list[Mapping[str, Any]], str, Mapping[str, Any]]: + """The Scottish rows, the resource's sha256 and the feed it was taken from. + + The packaged resource is read through ``vendored_rows``, which checks its + feed against the committed pin before it reads a row; a supplied payload + (tests) is checked against the pin first (unless ``verify_feed_identity`` + is false), then for the vendored header. + """ + + resource = SCOTLAND_BAND_D_RESOURCE + if payload is not None: + if verify_feed_identity: + verify_vendored_resource_feed_identity(payload) + rows = payload.get("rows") + if ( + payload.get("kind") != VENDORED_RESOURCE_KIND + or payload.get("resource") != resource + or not isinstance(rows, list) + or payload.get("row_count") != len(rows) + ): + raise UKCouncilTaxBandDError( + f"The supplied payload is not the vendored resource {resource}." + ) + feed = payload.get("source_fact_feed") or {} + return rows, "supplied", MappingProxyType(dict(feed)) + if verify_feed_identity: + rows = vendored_rows(resource) + else: + rows = list(load_vendored_resource(resource)["rows"]) + data = files("microcosm.build.uk").joinpath(resource).read_bytes() + feed = json.loads(data)["source_fact_feed"] + return rows, hashlib.sha256(data).hexdigest(), MappingProxyType(dict(feed)) + + +def _scotland_band_d(rows: list[Mapping[str, Any]]) -> dict[str, Fraction]: + """Each council area's Band D, its row held to the declared record set.""" + + resource = SCOTLAND_BAND_D_RESOURCE + band_d: dict[str, Fraction] = {} + for row in rows: + record = str(row.get("source_record_id")) + layout = row.get("layout") or {} + period = row.get("period") or {} + geography = row.get("geography") or {} + if layout.get("record_set_id") not in SCOTLAND_BAND_D_RECORD_SETS: + raise UKCouncilTaxBandDError( + f"{resource}: {record!r} is not of the record sets " + f"{list(SCOTLAND_BAND_D_RECORD_SETS)}." + ) + if ( + row.get("concept") != SCOTLAND_BAND_D_CONCEPT + or row.get("unit") != _SCOTLAND_UNIT + ): + raise UKCouncilTaxBandDError( + f"{resource}: {record!r} is not {SCOTLAND_BAND_D_CONCEPT!r} in " + f"{_SCOTLAND_UNIT!r}." + ) + if period.get("type") != "fiscal_year" or str(period.get("value")) != str( + UK_BAND_D_BUILD_PERIOD + ): + raise UKCouncilTaxBandDError( + f"{resource}: {record!r} is not of fiscal year {UK_BAND_D_FISCAL_YEAR}." + ) + if ( + geography.get("level") != _SCOTLAND_LEVEL + or geography.get("vintage") != SCOTLAND_BAND_D_GEOGRAPHY_VINTAGE + ): + raise UKCouncilTaxBandDError( + f"{resource}: {record!r} is not a {_SCOTLAND_LEVEL} row of vintage " + f"{SCOTLAND_BAND_D_GEOGRAPHY_VINTAGE!r}." + ) + code = _nation_code(geography.get("id"), _SCOTLAND_LETTERS, resource=resource) + if code in band_d: + raise UKCouncilTaxBandDError(f"{resource} repeats {code!r}.") + where = f"{resource}: the Band D of {code!r}" + text, _ = band_d_pence(row.get("value"), where=where) + band_d[code] = band_d_decimal(text, where=where) + return band_d + + +@dataclass(frozen=True) +class UKCouncilTaxBandD: + """2024-25 Band D council tax by Great Britain local authority code. + + ``band_d`` maps every England, Wales and Scotland code of the April 2023 + roster to its Band D in pounds, as an exact rational; Northern Ireland has + none. The digests are the two packaged resources' sha256 (``"supplied"`` + for a payload a test supplied). + """ + + band_d: Mapping[str, Fraction] + england_wales_sha256: str + scotland_sha256: str + scotland_source_fact_feed: Mapping[str, Any] + + def evidence(self) -> dict[str, object]: + return { + "fiscal_year": UK_BAND_D_FISCAL_YEAR, + "resources": { + UK_BAND_D_ENGLAND_WALES_RESOURCE: self.england_wales_sha256, + SCOTLAND_BAND_D_RESOURCE: self.scotland_sha256, + }, + "source_commit": self.scotland_source_fact_feed.get("source_commit"), + } + + +def load_uk_council_tax_band_d( + *, + england_wales_payload: Mapping[str, Any] | None = None, + scotland_payload: Mapping[str, Any] | None = None, + verify_feed_identity: bool = True, +) -> UKCouncilTaxBandD: + """Merge both Band D resources over the roster's Great Britain codes. + + ``england_wales_payload`` and ``scotland_payload`` replace the packaged + resources (tests); the Scottish one is checked against the committed feed + pin first (unless ``verify_feed_identity`` is false). The merged codes must + be exactly the roster's England, Wales and Scotland codes. + """ + + england_wales_data, england_wales_sha256 = _england_wales_payload( + england_wales_payload + ) + rows, scotland_sha256, feed = _scotland_rows( + scotland_payload, verify_feed_identity=verify_feed_identity + ) + england_wales = _england_wales_band_d(england_wales_data) + scotland = _scotland_band_d(rows) + band_d = {**england_wales, **scotland} + roster = load_uk_local_authority_names_resource()["areas"] + great_britain = { + code for code in roster if not code.startswith(_NORTHERN_IRELAND_LETTER) + } + off_roster = sorted(set(band_d) - great_britain) + if off_roster: + raise UKCouncilTaxBandDError( + f"The Band D codes {off_roster} are not on the local authority roster " + f"{UK_LOCAL_AUTHORITY_VINTAGE!r}." + ) + missing = sorted(great_britain - set(band_d)) + if missing: + raise UKCouncilTaxBandDError( + f"The Band D resources lack the roster code(s) {missing}." + ) + return UKCouncilTaxBandD( + band_d=MappingProxyType(dict(sorted(band_d.items()))), + england_wales_sha256=england_wales_sha256, + scotland_sha256=scotland_sha256, + scotland_source_fact_feed=feed, + ) + + +def check_scotland_band_d_declaration( + parameters: Mapping[str, Any], *, stage: str, period: int +) -> None: + """The runtime arm of a stage's ``verify_vendored_fact_resource`` operation. + + The declaration names the Scottish resource, its artifact role and record + set, and maps the facts to the frame's build period. + """ + + expected = { + "artifact_role": SCOTLAND_BAND_D_ARTIFACT_ROLE, + "resource": SCOTLAND_BAND_D_RESOURCE, + "record_sets": list(SCOTLAND_BAND_D_RECORD_SETS), + "mapped_build_period": UK_BAND_D_BUILD_PERIOD, + } + declared = {key: parameters.get(key) for key in expected} + if declared != expected: + raise UKCouncilTaxBandDError( + f"{stage}'s verify_vendored_fact_resource must declare {expected}, " + f"not {declared}." + ) + if int(period) != UK_BAND_D_BUILD_PERIOD: + raise UKCouncilTaxBandDError( + f"{stage} maps Scotland's Band D to build period " + f"{UK_BAND_D_BUILD_PERIOD}, but the frame's period is {period}." + ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/country_adapter.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/country_adapter.py index fccd45770b..3be2f674ab 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/country_adapter.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/country_adapter.py @@ -8,6 +8,7 @@ from __future__ import annotations import hashlib +import re from collections.abc import Mapping from dataclasses import replace from datetime import date @@ -130,12 +131,14 @@ def uk_atomic_support_register( """The committed identity of each atomic-area support, by system. ``sources.yaml`` pins every support's ``sha256`` and ``byte_size`` (the - provenance register carries the same values). A release candidate must - build on exactly these bytes, whatever digest the operator pinned on - the command line; the dense release pre-flight reads the same register - against the manifest's ``geography.assignment.support_pins``. - ``conditioned`` reads the conditioned supports (microcosm#931) instead, - which only the geography-first move binds. + provenance register carries the same values). By default it reads the + base supports, from which ``tools/build_uk_conditioned_area_supports.py`` + builds the conditioned ones; ``conditioned`` reads the conditioned + supports (microcosm#931), the ones a geography-first spine's areas are + drawn on. A release builds on exactly those bytes, whatever digests were + pinned elsewhere: :func:`uk_spine_geography_binding_failures` holds a + spine's support pins to that register, for the dense build's preparation + and the dense release pre-flight alike. """ from .atomic_area_support import CONDITIONED_SOURCES, SOURCES @@ -163,6 +166,160 @@ def uk_atomic_support_register( return register +_SHA256_HEX = re.compile(r"[0-9a-f]{64}") + +#: The register a spine's support pins were held to, by the name its +#: ``geography`` block records under ``register`` (microcosm#931): a release's +#: are held to sources.yaml's conditioned supports +#: (:func:`uk_atomic_support_register` with ``conditioned=True``), which the +#: spine build does before any stage runs. A synthetic fixture's toy supports +#: are held to no register, and its block says so. +UK_SPINE_SUPPORT_REGISTER = "sources.yaml" +UK_SYNTHETIC_SUPPORT_REGISTER = "synthetic" +_SUPPORT_REGISTERS = (UK_SPINE_SUPPORT_REGISTER, UK_SYNTHETIC_SUPPORT_REGISTER) + + +def uk_spine_geography_binding( + supports: Mapping[str, bytes], *, register: str +) -> dict[str, object]: + """A spine's ``geography`` block: how its areas were drawn (microcosm#931). + + ``supports`` holds the three conditioned supports' bytes, by system, as + the spine's area stages read them, and ``register`` names the register + their pins were held to (:data:`UK_SPINE_SUPPORT_REGISTER`, or + :data:`UK_SYNTHETIC_SUPPORT_REGISTER` for a synthetic fixture's toys). + The block holds, under the roster leaf's keys and in their order + (``geography_first_roster.SIDECAR_GEOGRAPHY_KEYS``): the atomic law + (``assignment``), the geography key's column (``identity``), the SHA-256 + of the conditioned definition the area draw builds on those supports at + its declared seed, as the shared operator digests a definition + (``definition_sha256``), each system's support SHA-256 and size in bytes + (``support_pins``, the size an integer), the definition's stream, its seed + at index 3 (``stream``) and ``register``. The spine build's sidecar writer + and a spine request's dense build both compose it here, so + :func:`uk_spine_geography_binding_failures` reads one shape whichever + route bound the spine. + + Refused: a register other than those two, and every refusal of the + definition (a support missing or not held to canonical bytes), by the + shared operator's texts, which name no value. + """ + + from microcosm.graph.canonical import canonical_json + + from .atomic_area_conditioning import ( + IDENTITY_COLUMN, + UK_SPINE_ATOMIC_AREA_SEED, + uk_conditioned_assignment_definition, + ) + from .atomic_area_support import SYSTEMS + from .geography_first_roster import SIDECAR_GEOGRAPHY_KEYS + + if register not in _SUPPORT_REGISTERS: + raise ValueError( + "A spine's geography block names the register its support pins were " + f"held to: one of {list(_SUPPORT_REGISTERS)}." + ) + definition = uk_conditioned_assignment_definition( + supports, seed=UK_SPINE_ATOMIC_AREA_SEED + ) + values: dict[str, object] = { + "assignment": "atomic", + "identity": IDENTITY_COLUMN, + "definition_sha256": hashlib.sha256(canonical_json(definition)).hexdigest(), + "support_pins": { + system: { + "sha256": hashlib.sha256(supports[system]).hexdigest(), + "size_bytes": len(supports[system]), + } + for system in SYSTEMS + }, + "stream": [*definition["stream"]], + "register": register, + } + return {key: values[key] for key in SIDECAR_GEOGRAPHY_KEYS} + + +def uk_spine_geography_binding_failures( + binding: object, + *, + clone_count: int | None = None, + spec: CountrySpec | None = None, +) -> tuple[str, ...]: + """What a spine's geography binding fails of a release's (microcosm#931). + + ``binding`` is the spine's ``geography`` block as the dense build binds + it: the bound spine's build sidecar block, with the spine's + ``geographic_clone_count`` beside it, which the rowwise manifest records + as ``geography.assignment``. A release's areas are drawn by the atomic + law (``assignment`` ``"atomic"``), keyed on each clone's geography key + (``identity``, the column's name), under a definition its SHA-256 names + (``definition_sha256``), on exactly the conditioned supports + ``sources.yaml`` registers (``support_pins``, each system's ``sha256`` and + ``size_bytes`` as :func:`uk_atomic_support_register` gives them with + ``conditioned=True``, the size an integer) and no system it does not + list, on the spine's area stream at its declared seed (``stream``, the + four entries of ``UK_SPINE_ATOMIC_AREA_DRAW_STREAM``, each of the same + type as there), with the pins held to sources.yaml's register (``register``, + :data:`UK_SPINE_SUPPORT_REGISTER`); with ``clone_count``, the block's K + must be that integer, so a boolean or a float such as ``15.0`` is not it. + A binding that is not a record fails every field. Its fields are those + :func:`uk_spine_geography_binding` composes. + + The dense build's preparation holds a release candidate's spine to it, + and the dense release pre-flight a candidate's manifest. Each failure + names a field and what it must be, never the value it holds, so a + refusal carries names and K only. + """ + + from .atomic_area_conditioning import UK_SPINE_ATOMIC_AREA_DRAW_STREAM + from .atomic_area_support import IDENTITY_COLUMN + + block = binding if isinstance(binding, Mapping) else {} + failures: list[str] = [] + if block.get("assignment") != "atomic": + failures.append("assignment is not 'atomic'") + if block.get("identity") != IDENTITY_COLUMN: + failures.append(f"identity is not {IDENTITY_COLUMN!r}") + digest = block.get("definition_sha256") + if not (isinstance(digest, str) and _SHA256_HEX.fullmatch(digest)): + failures.append("definition_sha256 is not a SHA-256 hex digest") + register = uk_atomic_support_register(spec, conditioned=True) + pins = block.get("support_pins") + pins = pins if isinstance(pins, Mapping) else {} + for system, expected in register.items(): + pin = pins.get(system) + if not ( + isinstance(pin, Mapping) + and pin.get("sha256") == expected["sha256"] + and pin.get("size_bytes") == expected["size_bytes"] + and type(pin.get("size_bytes")) is int + ): + failures.append( + f"support_pins.{system} is not the registered conditioned support" + ) + if set(pins) - set(register): + failures.append("support_pins names a system the register does not") + stream = block.get("stream") + expected_stream = list(UK_SPINE_ATOMIC_AREA_DRAW_STREAM) + if not ( + isinstance(stream, list) + and stream == expected_stream + and all( + type(found) is type(wanted) + for found, wanted in zip(stream, expected_stream, strict=True) + ) + ): + failures.append("stream is not the spine's area stream at its declared seed") + if block.get("register") != UK_SPINE_SUPPORT_REGISTER: + failures.append(f"register is not {UK_SPINE_SUPPORT_REGISTER!r}") + if clone_count is not None: + clones = block.get("geographic_clone_count") + if type(clones) is not int or clones != clone_count: + failures.append(f"geographic_clone_count is not K = {clone_count}") + return tuple(failures) + + def build_uk_country_graph( config: UKFullBuildConfig | None = None, *, @@ -172,13 +329,12 @@ def build_uk_country_graph( gate_review_date: date | None = None, release_candidate: bool = False, skip_holdout: bool = True, - atomic_geography_definition: Mapping | None = None, ) -> UKFullGraph: """Compile the canonical full graph; default target scope is all geographies. - ``atomic_geography_definition`` is the UK assignment declaration over the - admitted support bytes (``uk_atomic_assignment_definition``), required - while the config's ``geography_assignment`` is ``atomic`` (the default). + The dense line runs on the declared spine's version as it stands: the + spine's clones, areas and sampling are the build's, so the config's clone + count must be the spine's (one for the declared spine; microcosm#931). ``gate_review_date`` is the calibrated gates' clock, by default the solve clock ``review_date`` (microcosm#1301). """ @@ -218,7 +374,6 @@ def build_uk_country_graph( config, spine=spine, review_date=review_date.isoformat(), - atomic_geography_definition=atomic_geography_definition, ) graph = append_uk_full_gate_nodes( full.graph, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/data_target_parity.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/data_target_parity.py index f834985a8e..98505a762c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/data_target_parity.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/data_target_parity.py @@ -340,7 +340,7 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: "concern_id": "national_ons_land_values", "status": "ported_national", "classification": "red_line_national_family", - "evidence": "uk-data targets/sources/ons_land_values.py and _land.py; Microcosm target ids ons.land.{land_value,household_land_value,corporate_land_value}.", + "evidence": "uk-data targets/sources/ons_land_values.py and _land.py; Microcosm target ids ons.land.{land_value,household_land_value,corporate_land_value}, of which ons.land.land_value and ons.land.corporate_land_value are measure exclusions tracking microcosm#1521 since the #931 move.", }, { "concern_id": "regional_household_land_values", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/diagnostics.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/diagnostics.py index 24a8985627..1dd368a015 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/diagnostics.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/diagnostics.py @@ -514,8 +514,9 @@ def uk_weight_summary( #: Lineage columns that identify a support-split family: every copy the split #: created shares its root's source household, support channel, support clone -#: index and clone flag. These are the geography identity kernel's inputs -#: (microcosm#932) less the copy flag and index, so on the national spine only +#: index and clone flag. These are the first six lineage columns a household's +#: clone path is read from (``household_lineage.UK_HOUSEHOLD_LINEAGE_COLUMNS``) +#: less the support split's copy flag and index, so on the national spine only #: the split creates rows that agree on all four; on a K-expanded frame the #: geographic clone index (:func:`uk_support_family_geographic_clone_column`) #: joins the key, so each geographic clone folds only its own copies @@ -533,7 +534,8 @@ def uk_weight_summary( def uk_support_family_geographic_clone_column() -> str: - """The household geographic clone-index column ``uk.full.expand`` writes.""" + """The household geographic clone-index column, which the spine's + ``geographic_expand`` stage writes (microcosm#931).""" from microcosm.build.uk_runtime.rowwise_dataset import ladder_clone_index_column diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py index b9ea27b057..6fde51c5e8 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_services.py @@ -29,6 +29,7 @@ ) from microcosm.build.uk_runtime.etb_vat import UK_ETB_FAMILY_ROLE_COUNTS from microcosm.build.uk_runtime.frs_spine import read_pinned_tab +from microcosm.build.uk_runtime.geographic_clones import geographic_clone_count from microcosm.build.uk_runtime.household_income import ( UK_HBAI_NET_INCOME_PREDICTOR, UK_HBAI_NET_INCOME_VARIABLE, @@ -211,7 +212,12 @@ def __call__(self, frame: Frame) -> Frame: outcomes=outcomes, ) clip_result = support_clip_to_donor( - draws, donor, exempt=support_clip_exempt(self.stage) + draws, + donor, + exempt=support_clip_exempt(self.stage), + clone_count=geographic_clone_count( + frame.table("household"), stage=UK_ETB_SERVICES_STAGE_NAME + ), ) draws = clip_result.clipped household = frame.table("household").copy() @@ -519,7 +525,11 @@ def impute_etb_services( def support_clip_to_donor( - draws: pd.DataFrame, donor: pd.DataFrame, *, exempt: set[str] | None = None + draws: pd.DataFrame, + donor: pd.DataFrame, + *, + exempt: set[str] | None = None, + clone_count: int = 1, ) -> UKSupportClipResult: return support_clip_to_donor_with_receipt( draws, @@ -527,6 +537,7 @@ def support_clip_to_donor( columns=UK_ETB_SERVICES_HOUSEHOLD_OUTPUT_COLUMNS[:3], stage=UK_ETB_SERVICES_STAGE_NAME, exempt=exempt, + clone_count=clone_count, ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py index d294a5d81d..87e864717d 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py @@ -19,6 +19,7 @@ uprating_operation, ) from microcosm.build.uk_runtime.frs_spine import read_pinned_tab +from microcosm.build.uk_runtime.geographic_clones import geographic_clone_count from microcosm.build.uk_runtime.household_income import ( UK_HBAI_GROSS_INCOME_VARIABLES, etb_disposable_income, @@ -147,7 +148,13 @@ def __call__(self, frame: Frame) -> Frame: household_ids=frame.table("household")["household_id"].to_numpy(), salt=_qrf_salt(self.stage), ) - clip_result = support_clip_to_donor(imputed, donor) + clip_result = support_clip_to_donor( + imputed, + donor, + clone_count=geographic_clone_count( + frame.table("household"), stage=UK_ETB_VAT_STAGE_NAME + ), + ) imputed = clip_result.clipped household = frame.table("household").copy() household["full_rate_vat_expenditure_rate"] = imputed[ @@ -398,13 +405,14 @@ def impute_etb_vat( def support_clip_to_donor( - draws: pd.DataFrame, donor: pd.DataFrame + draws: pd.DataFrame, donor: pd.DataFrame, *, clone_count: int = 1 ) -> UKSupportClipResult: return support_clip_to_donor_with_receipt( draws, donor, columns=UK_ETB_VAT_OUTPUT_COLUMNS, stage=UK_ETB_VAT_STAGE_NAME, + clone_count=clone_count, ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_brma.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_brma.py index 262cb0ed16..862bb5a15d 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_brma.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_brma.py @@ -1,4 +1,17 @@ -"""UK FRS BRMA stochastic assignment from committed census household counts.""" +"""UK FRS BRMA stochastic assignment from committed census household counts. + +Each benefit unit draws a BRMA from the census count-table cell of its region +and ``LHA_category``, with a uniform keyed on its own id, and each household +picks one of its benefit units' BRMAs with a uniform keyed on its own id +(:func:`draw_household_brmas`). Every uniform is keyed on its own id alone, +so drawing a subset of a frame's households gives each the BRMA the whole +frame's draw gives it. ``spine_geography``'s BRMA stage draws Northern Ireland's +households this way, and its SPI-copy area stage the Northern Ireland copies, +each on its own benefit units (microcosm#931). + +Refusals name columns and declared cells only, never a household's or a +benefit unit's id. +""" from __future__ import annotations @@ -28,6 +41,13 @@ FRS_BRMA_OUTPUT_COLUMNS = ("brma",) UK_BRMA_PREDICTORS = ("LHA_category",) UK_BRMA_DECLARED_SEEDS = {"brma": 0} +#: The committed census counts the draw samples from. +UK_BRMA_COUNT_RESOURCE = "brma_private_rented_households.json" +#: The salts each uniform is keyed on with the seed and an id: each benefit +#: unit's draw from its cell, and each household's pick among its benefit +#: units' BRMAs. +UK_BRMA_BENUNIT_SALT = "brma" +UK_BRMA_HOUSEHOLD_PICK_SALT = "brma:household_pick" @dataclass(frozen=True) @@ -59,9 +79,7 @@ def output_columns() -> tuple[str, ...]: def load_brma_count_resource() -> Mapping[str, Any]: return json.loads( - files("microcosm.build.uk") - .joinpath("brma_private_rented_households.json") - .read_text() + files("microcosm.build.uk").joinpath(UK_BRMA_COUNT_RESOURCE).read_text() ) @@ -74,22 +92,14 @@ def add_frs_brma( person = frame.table("person").copy() benunit = frame.table("benunit").copy() household = frame.table("household").copy() - if len(lha_category) != len(benunit): - raise ValueError("LHA_category materialization must align to benunit rows.") - benunit["LHA_category"] = [_enum_name(value) for value in lha_category] - benunit["region"] = _benunit_regions(person, household, benunit) - benunit["brma"] = assign_brma_by_cell( - benunit, - count_resource=count_resource, - seed=UK_BRMA_DECLARED_SEEDS["brma"], - ) - household["brma"] = collapse_benunit_brma_to_household( + household["brma"] = draw_household_brmas( person, - benunit[["benunit_id", "brma"]], + benunit, household, + lha_category=lha_category, + count_resource=count_resource, seed=UK_BRMA_DECLARED_SEEDS["brma"], ) - benunit = benunit.drop(columns=["LHA_category", "region", "brma"]) result = uk_national_frame( person=person, benunit=benunit, @@ -103,6 +113,50 @@ def add_frs_brma( return result +def draw_household_brmas( + person: pd.DataFrame, + benunit: pd.DataFrame, + household: pd.DataFrame, + *, + lha_category: Sequence[object], + count_resource: Mapping[str, Any], + seed: int = UK_BRMA_DECLARED_SEEDS["brma"], +) -> np.ndarray: + """Each of ``household``'s rows' BRMA, drawn on its own benefit units. + + ``lha_category`` is aligned to ``benunit``'s rows, and ``person`` places + each benefit unit in its household. ``household`` may hold any subset of + the frame's households: the draw reads only the benefit units of its + rows. Each of them samples a BRMA from its region and ``LHA_category`` + cell with the uniform keyed on its own id (:data:`UK_BRMA_BENUNIT_SALT`), + and each household picks one of its benefit units' BRMAs, in id order, + with the uniform keyed on its own id (:data:`UK_BRMA_HOUSEHOLD_PICK_SALT`), + both under ``seed``. So a subset's households take the BRMAs a draw of the + whole frame gives them, and a copy keyed on ids of its own takes a draw of + its own. Returns an object array aligned to ``household``'s rows. + + Refused, naming no id: a ``lha_category`` not aligned to ``benunit``, a + benefit unit split across households, a household with no benefit unit, + and a count-table cell the resource lacks (by region and category). + """ + + if len(lha_category) != len(benunit): + raise ValueError("LHA_category materialization must align to benunit rows.") + owners = benunit["benunit_id"].map(_benunit_household_map(person)) + drawn = owners.isin(household["household_id"]).to_numpy(dtype=bool) + units = benunit.loc[drawn, ["benunit_id"]].copy() + units["LHA_category"] = [ + _enum_name(value) + for value, keep in zip(lha_category, drawn.tolist(), strict=True) + if keep + ] + units["region"] = _benunit_regions(person, household, units) + units["brma"] = assign_brma_by_cell(units, count_resource=count_resource, seed=seed) + return collapse_benunit_brma_to_household( + person, units[["benunit_id", "brma"]], household, seed=seed + ) + + def assign_brma_by_cell( benunit: pd.DataFrame, *, @@ -111,7 +165,7 @@ def assign_brma_by_cell( ) -> np.ndarray: cells = count_resource["cells"] draws = stable_identity_uniforms( - benunit["benunit_id"].to_numpy(), seed=seed, salt="brma" + benunit["benunit_id"].to_numpy(), seed=seed, salt=UK_BRMA_BENUNIT_SALT ) assigned: list[str] = [] for index, row in enumerate(benunit.itertuples(index=False)): @@ -146,7 +200,7 @@ def collapse_benunit_brma_to_household( values["household_id"] = values["benunit_id"].map(mapping) household_ids = household["household_id"].to_numpy() draws = stable_identity_uniforms( - household_ids, seed=seed, salt="brma:household_pick" + household_ids, seed=seed, salt=UK_BRMA_HOUSEHOLD_PICK_SALT ) options_by_household = { household_id: group.sort_values("benunit_id")["brma"].astype(str).to_list() @@ -156,7 +210,10 @@ def collapse_benunit_brma_to_household( for index, household_id in enumerate(household_ids): options = options_by_household.get(household_id) if not options: - raise ValueError(f"household {household_id!r} has no benunit BRMA options.") + raise ValueError( + "Each household picks its BRMA among its benefit units' draws; " + "a household has no benefit unit with a drawn BRMA." + ) pick = min(int(draws[index] * len(options)), len(options) - 1) assigned.append(options[pick]) return np.asarray(assigned, dtype=object) @@ -179,12 +236,9 @@ def _benunit_regions( def _benunit_household_map(person: pd.DataFrame) -> pd.Series: placements = person[["person_benunit_id", "person_household_id"]].drop_duplicates() if placements["person_benunit_id"].duplicated().any(): - duplicated = placements.loc[ - placements["person_benunit_id"].duplicated(keep=False), - "person_benunit_id", - ].unique() raise ValueError( - f"benunit id(s) split across households: {duplicated[:5].tolist()}." + "A benefit unit's members must share one household; the person table " + "splits a benefit unit across households." ) return placements.set_index("person_benunit_id")["person_household_id"] diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_council_tax.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_council_tax.py index 1398fee172..b1d1be93a6 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_council_tax.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_council_tax.py @@ -23,11 +23,16 @@ uk_time_period, validate_uk_national_frame, ) -from microcosm.build.uk_runtime.receipt_counts import support_band +from microcosm.build.uk_runtime.receipt_counts import ( + distinct_count, + record_source_households, + rounded_count, + support_band, +) from microcosm.frame import Frame -FRS_COUNCIL_TAX_OUTPUT_COLUMNS = ("council_tax",) SCOTLAND_GVTREGNO = 12 +NORTHERN_IRELAND_GVTREGNO = 13 #: Raw household cells the derivation reads; the pinned tab carries them all. FRS_COUNCIL_TAX_RAW_COLUMNS = ( "ctannual", @@ -55,6 +60,34 @@ SCOTTISH_CHARGES_IN_RECIPIENT_BILL = SCOTTISH_WATER_CHARGES_REDUCTION_RECIPIENT_SHARE FRS_COUNCIL_TAX_STAGE_NAME = "frs_council_tax" FRS_COUNCIL_TAX_IMPUTATION_KIND = "impute_cell_means" +#: The household column naming the branch each stage-5 bill takes, or +#: ``not_levied`` in Northern Ireland (``derive_council_tax_provenance``): +#: one of the four values below. +FRS_COUNCIL_TAX_BILL_PROVENANCE_COLUMN = "council_tax_bill_provenance" +#: The bill is the observed CTANNUAL, netted of Scotland's water and sewerage +#: charges, with any reported reduction added back: an observed gross bill. A +#: household without a reported reduction whose CTANNUAL is zero and whose +#: CTPAID is not 1 (2, 3 or blank) keeps its zero bill under this value too. +FRS_COUNCIL_TAX_BILL_OBSERVED_GROSS = "observed_gross" +#: CTANNUAL is negative or blank, or zero where someone pays and no reduction +#: is reported: the bill is the cell mean. +FRS_COUNCIL_TAX_BILL_MISSING = "missing" +#: A reduction is reported (CTREB 1) without a positive CTREBAMT: the bill is +#: the larger of the netted CTANNUAL and the cell mean. +FRS_COUNCIL_TAX_BILL_UNKNOWN_REDUCTION = "unknown_reduction" +#: Northern Ireland, which pays domestic rates, not council tax. +FRS_COUNCIL_TAX_BILL_NOT_LEVIED = "not_levied" +FRS_COUNCIL_TAX_BILL_PROVENANCE_VALUES = ( + FRS_COUNCIL_TAX_BILL_OBSERVED_GROSS, + FRS_COUNCIL_TAX_BILL_MISSING, + FRS_COUNCIL_TAX_BILL_UNKNOWN_REDUCTION, + FRS_COUNCIL_TAX_BILL_NOT_LEVIED, +) +#: The graph's string dtype, which the provenance column is stored in. +_PROVENANCE_DTYPE = pd.StringDtype(storage="python") +#: The household columns the stage writes: the bill, then its provenance, +#: which the release boundary drops and council_tax_localise alone reads. +FRS_COUNCIL_TAX_OUTPUT_COLUMNS = ("council_tax", FRS_COUNCIL_TAX_BILL_PROVENANCE_COLUMN) FRS_COUNCIL_TAX_BUILD_PERIOD = 2024 VERIFY_VENDORED_FACT_RESOURCE_KIND = "verify_vendored_fact_resource" #: S2(a) of the November event-data plan (microcosm#1095): the FRS's Scottish @@ -118,6 +151,27 @@ def frs_council_tax_operation_parameters() -> dict[str, Any]: } +def frs_council_tax_provenance_operation_parameters() -> dict[str, Any]: + """The ``derive`` declaration ``derive_council_tax_provenance`` implements.""" + + return { + "kind": "derive", + "output": FRS_COUNCIL_TAX_BILL_PROVENANCE_COLUMN, + "formula": ( + "first match: not_levied where raw gvtregno == 13 (Northern " + "Ireland, which pays domestic rates), whatever branch its bill " + "takes; else the branch derive_council_tax takes: missing where " + "the bill is missing (raw ctannual < 0 or NaN, or ctannual == 0 " + "with ctpaid == 1 and ctreb != 1); observed_gross where ctreb == 1 " + "and ctrebamt > 0 (the reported reduction added back); " + "unknown_reduction where ctreb == 1 (the larger of the netted " + "ctannual and the cell mean); else observed_gross (the netted " + "ctannual, kept at zero where ctannual == 0 and ctpaid != 1, as " + "with ctpaid 2 and 3)" + ), + } + + def frs_council_tax_resource_operation_parameters() -> dict[str, Any]: """The ``verify_vendored_fact_resource`` declaration the Scottish level reads.""" @@ -184,6 +238,7 @@ def frs_council_tax_operations() -> tuple[dict[str, Any], ...]: frs_council_tax_operation_parameters(), frs_council_tax_resource_operation_parameters(), frs_council_tax_levelling_operation_parameters(), + frs_council_tax_provenance_operation_parameters(), ) @@ -200,7 +255,18 @@ def frs_council_tax_resource_artifact() -> dict[str, Any]: def assert_frs_council_tax_stage_parameters(stage: SourceStageSpec) -> None: - """Closed-world drift assert: the manifest declares the implemented rule.""" + """Closed-world drift assert: the manifest declares the implemented rules. + + The stage reads its table, imputes the bill + (:func:`frs_council_tax_operation_parameters`), checks the vendored + Scottish Band D facts and levels Scotland's bills to them + (:func:`frs_council_tax_resource_operation_parameters`, + :func:`frs_council_tax_levelling_operation_parameters`) and derives the + bill's provenance (:func:`frs_council_tax_provenance_operation_parameters`), + in that order (:func:`frs_council_tax_operations`), each declaration + exactly as those functions state it, and declares the Band D facts' + artifact once (:func:`frs_council_tax_resource_artifact`). + """ if stage.stage != FRS_COUNCIL_TAX_STAGE_NAME: raise ValueError(f"frs_council_tax transform received stage {stage.stage!r}.") @@ -210,7 +276,8 @@ def assert_frs_council_tax_stage_parameters(stage: SourceStageSpec) -> None: expected_kinds = ["read_tables", *(str(op["kind"]) for op in expected_operations)] if kinds != expected_kinds: raise ValueError( - f"frs_council_tax stage must declare {expected_kinds}, got {kinds}." + f"frs_council_tax stage must declare {', '.join(expected_kinds)}, in " + f"that order, got {kinds}." ) for operation, expected in zip(operations[1:], expected_operations, strict=True): declared = {"kind": operation.kind, **dict(operation.parameters)} @@ -223,7 +290,7 @@ def assert_frs_council_tax_stage_parameters(stage: SourceStageSpec) -> None: raise ValueError( f"frs_council_tax {operation.kind} declaration drifted from the " f"rule the runtime implements on parameter(s) {drifted} " - "(uk-data#496/#499, microcosm#1095)." + "(uk-data#496/#499, microcosm#1095, microcosm#931)." ) expected_artifact = frs_council_tax_resource_artifact() resources = [ @@ -419,7 +486,14 @@ def level_scottish_council_tax( class UKFRSCouncilTaxStageTransform: - """Whole-stage callable for FRS council-tax imputation.""" + """Whole-stage callable for FRS council-tax imputation. + + Running it writes each household's bill, levelled in Scotland, and the + bill's provenance (:func:`frs_council_tax_with_receipt`), then keeps the + stage's two receipts for :meth:`checkpoint_metadata`: the Scottish + level's, and the provenance's (:func:`council_tax_provenance_evidence`), + counted on the frame it wrote. + """ def __init__( self, @@ -433,12 +507,17 @@ def __init__( #: The Scottish level; read from the vendored resource when not given. self.level = level self.last_receipt: dict[str, Any] | None = None + self.last_evidence: dict[str, Any] | None = None def __call__(self, frame: Frame) -> Frame: result, receipt = frs_council_tax_with_receipt( frame, self.raw_dir, stage=self.stage, level=self.level ) self.last_receipt = receipt + self.last_evidence = council_tax_provenance_evidence( + result, + result.table("household")[FRS_COUNCIL_TAX_BILL_PROVENANCE_COLUMN], + ) return result @staticmethod @@ -446,11 +525,11 @@ def output_columns() -> tuple[str, ...]: return FRS_COUNCIL_TAX_OUTPUT_COLUMNS def checkpoint_metadata(self) -> dict[str, object]: - if self.last_receipt is None: - return {} + if self.last_receipt is None or self.last_evidence is None: + raise RuntimeError("checkpoint metadata requires a completed stage run.") return { "evidence": { - "stage": FRS_COUNCIL_TAX_STAGE_NAME, + **self.last_evidence, "scottish_level": dict(self.last_receipt), } } @@ -463,6 +542,15 @@ def add_frs_council_tax( stage: SourceStageSpec, level: ScottishCouncilTaxLevel | None = None, ) -> Frame: + """Each household's bill and the branch the bill takes, on the frame. + + ``council_tax`` is :func:`derive_council_tax`'s bill, with Scotland's + bills levelled (:func:`level_scottish_council_tax`), and + ``FRS_COUNCIL_TAX_BILL_PROVENANCE_COLUMN`` is + :func:`derive_council_tax_provenance`'s flag, both from one set of masks; + the flag is in the graph's string dtype (``StringDtype("python")``). + """ + return frs_council_tax_with_receipt(frame, raw_dir, stage=stage, level=level)[0] @@ -487,8 +575,8 @@ def frs_council_tax_with_receipt( ) ) household = frame.table("household").copy() - derived = derive_council_tax(household, household_raw) - bills = derived.reindex(household["household_id"]).to_numpy(dtype=float) + bill, provenance = _council_tax_branches(household, household_raw) + bills = bill.reindex(household["household_id"]).to_numpy(dtype=float) gvtregno = pd.to_numeric( household_raw.set_index("household_id")["gvtregno"].reindex( household["household_id"] @@ -502,6 +590,11 @@ def frs_council_tax_with_receipt( level=level, ) household["council_tax"] = levelled + household[FRS_COUNCIL_TAX_BILL_PROVENANCE_COLUMN] = pd.Series( + provenance.reindex(household["household_id"]).to_numpy(dtype=object), + index=household.index, + dtype=_PROVENANCE_DTYPE, + ) result = uk_national_frame( person=frame.table("person"), benunit=frame.table("benunit"), @@ -554,6 +647,19 @@ def derive_council_tax( quantity, ``frs_spine.scottish_water_and_sewerage_weekly``. """ + return _council_tax_branches(household, household_raw)[0] + + +def _council_tax_branches( + household: pd.DataFrame, household_raw: pd.DataFrame +) -> tuple[pd.Series, pd.Series]: + """``derive_council_tax``'s bill, and the branch each bill takes. + + One set of masks chooses both, so the provenance names the branch the + bill took on every row outside Northern Ireland, where it reads + ``not_levied`` whatever the branch. + """ + missing_columns = sorted(set(FRS_COUNCIL_TAX_RAW_COLUMNS) - set(household_raw)) if missing_columns: raise KeyError(f"FRS council tax needs raw household cells {missing_columns}.") @@ -605,7 +711,88 @@ def positive(column: str) -> pd.Series: np.where(recipient, np.maximum(tax_only, imputed), tax_only), ), ) - return pd.Series(gross, index=aligned.index).fillna(0.0).clip(lower=0) + # The bill's own branches in the bill's order, after Northern Ireland: the + # first condition that holds names the row. + provenance = np.select( + [ + (gvtregno == NORTHERN_IRELAND_GVTREGNO).to_numpy(dtype=bool), + missing.to_numpy(dtype=bool), + known_reduction.to_numpy(dtype=bool), + recipient.to_numpy(dtype=bool), + ], + [ + FRS_COUNCIL_TAX_BILL_NOT_LEVIED, + FRS_COUNCIL_TAX_BILL_MISSING, + FRS_COUNCIL_TAX_BILL_OBSERVED_GROSS, + FRS_COUNCIL_TAX_BILL_UNKNOWN_REDUCTION, + ], + default=FRS_COUNCIL_TAX_BILL_OBSERVED_GROSS, + ) + return ( + pd.Series(gross, index=aligned.index).fillna(0.0).clip(lower=0), + pd.Series(provenance, index=aligned.index, dtype=_PROVENANCE_DTYPE), + ) + + +def derive_council_tax_provenance( + household: pd.DataFrame, household_raw: pd.DataFrame +) -> pd.Series: + """The branch each household's ``derive_council_tax`` bill takes. + + One of ``FRS_COUNCIL_TAX_BILL_PROVENANCE_VALUES`` per household, on the + bill's index, in the graph's string dtype (``StringDtype("python")``). + The first that holds: ``not_levied`` in Northern Ireland (raw GVTREGNO + ``NORTHERN_IRELAND_GVTREGNO``); ``missing`` where the bill takes the cell + mean as a missing one; ``observed_gross`` where a reported reduction is + added back; ``unknown_reduction`` for any other household reporting a + reduction; else ``observed_gross``. The masks are the bill's own, so + outside Northern Ireland the value is the branch the bill took. + """ + + return _council_tax_branches(household, household_raw)[1] + + +def council_tax_provenance_evidence( + frame: Frame | Mapping[str, pd.DataFrame], provenance: Any +) -> dict[str, Any]: + """Stage 5's receipt of its bill provenance, by distinct true sources. + + ``provenance`` holds one of ``FRS_COUNCIL_TAX_BILL_PROVENANCE_VALUES`` for + each household row of ``frame`` (a frame, or its tables by entity), in + row order. Each value's carriers are counted in distinct true source + households through ``receipt_counts.record_source_households`` (before + the first copying step each household is its own source; a copy or clone + counts as its source), UK-wide, and recorded as + ``receipt_counts.rounded_count`` gives them, every value present, zero + included. Refusals name the column (and the four values), never a value + read or a count. + """ + + sources = record_source_households( + frame, "household", stage=FRS_COUNCIL_TAX_STAGE_NAME, before_copies=True + ) + values = np.asarray(provenance, dtype=object) + if values.ndim != 1 or len(values) != len(sources): + raise ValueError( + f"{FRS_COUNCIL_TAX_STAGE_NAME}'s bill provenance receipt needs one " + f"{FRS_COUNCIL_TAX_BILL_PROVENANCE_COLUMN} value per household row." + ) + flags = pd.Series(values) + if not flags.isin(FRS_COUNCIL_TAX_BILL_PROVENANCE_VALUES).all(): + raise ValueError( + f"{FRS_COUNCIL_TAX_STAGE_NAME}'s " + f"{FRS_COUNCIL_TAX_BILL_PROVENANCE_COLUMN} holds a value outside " + f"{list(FRS_COUNCIL_TAX_BILL_PROVENANCE_VALUES)}." + ) + return { + "stage": FRS_COUNCIL_TAX_STAGE_NAME, + "bill_provenance_sources": { + value: rounded_count( + distinct_count(sources, (flags == value).to_numpy(dtype=bool)) + ) + for value in FRS_COUNCIL_TAX_BILL_PROVENANCE_VALUES + }, + } def _artifact_by_table(stage: SourceStageSpec) -> dict[str, Mapping[str, Any]]: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_disability.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_disability.py index 873dd4f1ee..86f2507cb0 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_disability.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_disability.py @@ -475,7 +475,11 @@ def _carer_routes(person: pd.DataFrame) -> _CarerRoutes: def carer_link_share( - person: pd.DataFrame, *, weights: np.ndarray, min_hours: float + person: pd.DataFrame, + *, + weights: np.ndarray, + min_hours: float, + support_keys: np.ndarray | None = None, ) -> tuple[float | None, int]: """The share the carer link draws outside carers at, and its households. @@ -483,7 +487,12 @@ def carer_link_share( hours reach ``min_hours`` and who report no Carer's Allowance, the share by ``weights`` who give it to a qualifying member; None where none of them has weight. The second value is the distinct households that base rests - on, for the receipt to band. + on, for the receipt to band. ``support_keys`` are what that count reads, + by default each person's household. On the clone-first spine the FRS + channel holds every geographic clone, so the SPI channel's refresh passes + each person's true source household (``spi_income``), and a survey + household's clones count once, as the disclosure floor counts them + (microcosm#931). """ _assert_care_line(min_hours) @@ -501,7 +510,8 @@ def carer_link_share( if base_weight > 0.0 else None ) - return share, len(set(routes.households[base].tolist())) + keys = routes.households if support_keys is None else np.asarray(support_keys) + return share, len(set(keys[base].tolist())) def derive_carer_link( @@ -519,19 +529,22 @@ def derive_carer_link( the care carriers and the categories this stage derives; ``weights`` are the persons' household weights. ``share`` is the drawn route's share and the households it rests on, by default :func:`carer_link_share` of - ``person``; the SPI channel passes the survey's own (``spi_income``). - ``support_keys`` are what the evidence counts support in, by default each - person's household; the SPI channel passes source households, so a survey - household's copies count once. The evidence gives each route's records in - rounded counts and support bands, withheld where one or two households - carry them, and the share the drawn route takes. + ``person`` over ``support_keys``; the SPI channel passes the survey's own + (``spi_income``). ``support_keys`` are what the evidence counts support + in, by default each person's household; the SPI channel passes source + households, so a survey household's copies count once. The evidence + gives each route's records in rounded counts and support bands, withheld + where one or two households carry them, and the share the drawn route + takes. """ _assert_care_line(min_hours) routes = _carer_routes(person) weights = np.asarray(weights, dtype=float) if share is None: - share = carer_link_share(person, weights=weights, min_hours=min_hours) + share = carer_link_share( + person, weights=weights, min_hours=min_hours, support_keys=support_keys + ) share_value, share_households = share keys = routes.households if support_keys is None else np.asarray(support_keys) linked = routes.carer & routes.member_qualifies diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_person_draws.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_person_draws.py index f445f3a33d..ec5609a37b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_person_draws.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_person_draws.py @@ -50,6 +50,8 @@ ) from microcosm.build.uk_runtime.frs_employment import FRS_EMPSTATI_EMPLOYMENT_STATUS from microcosm.build.uk_runtime.frs_take_up import ( + geographic_clone_indices, + geographic_clone_zero_rows, take_up_signal_outputs, uk_take_up_signal_receipt, ) @@ -186,6 +188,10 @@ def add_frs_person_draws_with_receipt( person_sources=record_source_households( frame, "person", stage="frs_person_draws", before_copies=True ), + person_clone_zero=geographic_clone_zero_rows(frame, "person"), + # Each person's household's clone, which the apprentice walk meets its + # target within; None on a frame without household_clone_index. + person_clone_index=geographic_clone_indices(frame, "person"), ) for column in FRS_PERSON_DRAW_OUTPUT_COLUMNS: person[column] = derived[column].to_numpy() @@ -209,12 +215,18 @@ def derive_frs_person_draws( person_weights: np.ndarray, relief_schedule: Mapping[str, float] | None = None, person_sources: np.ndarray, + person_clone_zero: np.ndarray | None = None, + person_clone_index: np.ndarray | None = None, ) -> pd.DataFrame: """Return the stage's person draws. ``person_weights`` are each person's household weight, which set - ``is_apprentice``'s target; ``person_sources`` are each person's true - source household, which its receipt counts support by. + ``is_apprentice``'s target, met within each geographic clone of + ``person_clone_index`` (every person in one when not given); + ``person_sources`` are each person's true source household, which its + receipt counts support by, among geographic clone 0's people + (``person_clone_zero``; where it is not given, clone 0 of + ``person_clone_index``, else everyone). """ return derive_frs_person_draws_with_receipt( @@ -223,6 +235,8 @@ def derive_frs_person_draws( person_weights=person_weights, relief_schedule=relief_schedule, person_sources=person_sources, + person_clone_zero=person_clone_zero, + person_clone_index=person_clone_index, )[0] @@ -233,6 +247,8 @@ def derive_frs_person_draws_with_receipt( person_weights: np.ndarray, relief_schedule: Mapping[str, float] | None = None, person_sources: np.ndarray, + person_clone_zero: np.ndarray | None = None, + person_clone_index: np.ndarray | None = None, ) -> tuple[pd.DataFrame, Mapping[str, object]]: """The stage's person draws and the receipt of its ``is_apprentice`` draw.""" @@ -266,6 +282,8 @@ def derive_frs_person_draws_with_receipt( person_weights=person_weights, relief_schedule=relief_schedule, person_sources=person_sources, + person_clone_zero=person_clone_zero, + person_clone_index=person_clone_index, ) values[APPRENTICE_OUTPUT] = apprentices.flags return values, {"build_year": contract.build_year, **apprentices.receipt} @@ -286,6 +304,8 @@ def derive_is_apprentice( person_weights: np.ndarray | None = None, relief_schedule: Mapping[str, float] | None = None, person_sources: np.ndarray, + person_clone_zero: np.ndarray | None = None, + person_clone_index: np.ndarray | None = None, ) -> UKApprenticeDraw: """Draw ``is_apprentice`` for employees aged 21 to 24 outside full-time education. @@ -298,6 +318,23 @@ def derive_is_apprentice( eligible weight falls short of the target every eligible employee is flagged and the receipt says the draw was cap-bound. + The target is met within each geographic clone (microcosm#931). + ``person_clone_index`` gives each person's clone (everyone is in one + clone when it is not given), and each clone walks its own employees to + the rate times its own weight aged 21 to 24. Clone 0 holds a one-clone + spine's people at their weights over K, so it takes that spine's flags + at any power-of-two K, which scales its cumulative weights exactly, and + elsewhere except at a near-tie: where the target lies within rounding of + the midpoint between two consecutive prefixes of the walk, or of the + eligible weight, at which the walk flags everyone. One walk over every + clone would interleave the clones' employees and stop once, so clone 0's + flags would not be one clone's. The receipt's masses and realized weight + are sums over the clones. Its target is the rate times the weight aged + 21 to 24 over every clone, and the draw is cap-bound where the eligible + weight over every clone is no more than that target: the figures + stage_health's apprentice_draw gate recomputes from the receipt, and at + one clone the walk's own, bit for bit. + Given a ``relief_schedule`` (the engine's Class 1 rate and its secondary and apprentice upper secondary thresholds, annual), the receipt also records the relief the flags imply: the rate on each flagged employee's @@ -307,7 +344,15 @@ def derive_is_apprentice( The receipt counts support in ``person_sources``, each person's true source household (``receipt_counts.record_source_households``). A caller without the frame's lineage passes ``own_source_households`` itself, so - no table is counted by its rows unawares (microcosm#931). + no table is counted by its rows unawares (microcosm#931). The supports + it prints are counted among geographic clone 0's people, + ``person_clone_zero`` (where it is not given, the people + ``person_clone_index`` puts in clone 0, else everyone): each clone flags + its own employees, so the flagged sources over every clone would grow + with the clone count. The masses are taken over every clone, so each is + withheld where one or two source households carry it in clone 0's people + or over every clone: clone 0 can flag none of the employees one or two + households hold in the other clones. """ if "employment_status" not in person.columns: @@ -329,6 +374,7 @@ def derive_is_apprentice( ) if weights.shape != (len(person),): raise ValueError("person_weights must hold one weight per person.") + clones = _clone_rows(person_clone_index, len(person)) age = pd.to_numeric(person["age"], errors="coerce").to_numpy(dtype=float) in_ages = (age >= APPRENTICE_MIN_AGE) & (age < APPRENTICE_MAX_AGE_EXCLUSIVE) employee = ( @@ -344,18 +390,38 @@ def derive_is_apprentice( draws = _draws(person["person_id"].to_numpy(), APPRENTICE_OUTPUT) population_mass = float(weights[in_ages].sum()) eligible_mass = float(weights[eligible].sum()) + flags = np.zeros(len(person), dtype=bool) + for rows in clones: + clone_eligible = eligible & rows + threshold = _nearest_target_threshold( + float(rate) * float(weights[in_ages & rows].sum()), + draws=draws[clone_eligible], + weights=weights[clone_eligible], + ) + flags |= clone_eligible & (draws < threshold) + # The receipt's target and cap bound are taken over every clone, as + # stage_health's apprentice_draw gate recomputes them from the receipt's + # masses; at one clone they are the walk's own, bit for bit. target = float(rate) * population_mass - threshold = _nearest_target_threshold( - target, draws=draws[eligible], weights=weights[eligible] - ) - flags = eligible & (draws < threshold) realized = float(weights[flags].sum()) sources = np.asarray(person_sources) if sources.shape != (len(person),): raise ValueError("person_sources must hold one source per person.") - population_support = distinct_count(sources, in_ages, weights) - eligible_support = distinct_count(sources, eligible, weights) - flagged_support = distinct_count(sources, flags, weights) + clone_zero = _clone_zero_people(person_clone_zero, person_clone_index, len(person)) + population_support = distinct_count(sources, in_ages & clone_zero, weights) + eligible_support = distinct_count(sources, eligible & clone_zero, weights) + flagged_support = distinct_count(sources, flags & clone_zero, weights) + # The masses are taken over every clone, so a mass is withheld where one + # or two source households carry it in clone 0 or over every clone. + population_withheld = _thin(population_support) or _thin( + distinct_count(sources, in_ages, weights) + ) + eligible_withheld = _thin(eligible_support) or _thin( + distinct_count(sources, eligible, weights) + ) + flagged_withheld = _thin(flagged_support) or _thin( + distinct_count(sources, flags, weights) + ) receipt: dict[str, object] = { "output": APPRENTICE_OUTPUT, "rate_key": APPRENTICE_RATE_KEY, @@ -376,13 +442,13 @@ def derive_is_apprentice( # A weighted total carried by one or two households is withheld, with the # figures that would give it back: the target is the rate times the # population, and the deviation is the flagged weight less the target. - if 0 < population_support < UK_DISCLOSURE_FLOOR: + if population_withheld: receipt.update( population_mass=UK_WITHHELD, target=UK_WITHHELD, realization_deviation=UK_WITHHELD, ) - if 0 < eligible_support < UK_DISCLOSURE_FLOOR: + if eligible_withheld: receipt["eligible_mass"] = UK_WITHHELD if relief_schedule is not None and "employment_income" in person.columns: pay = pd.to_numeric(person["employment_income"], errors="coerce") @@ -399,13 +465,65 @@ def derive_is_apprentice( receipt["implied_relief"] = float( relief_schedule["rate"] * (weights[flags] * relieved[flags]).sum() ) - if 0 < flagged_support < UK_DISCLOSURE_FLOOR: + if flagged_withheld: receipt.update(realized_mass=UK_WITHHELD, realization_deviation=UK_WITHHELD) if "implied_relief" in receipt: receipt["implied_relief"] = UK_WITHHELD return UKApprenticeDraw(flags=flags, receipt=receipt) +def _thin(support: int) -> bool: + """Whether one or two source households carry a figure: it is withheld.""" + + return 0 < support < UK_DISCLOSURE_FLOOR + + +def _clone_rows(clone_index: np.ndarray | None, rows: int) -> tuple[np.ndarray, ...]: + """Each geographic clone's rows of ``rows`` people, in sorted clone order. + + A ``clone_index`` that is None or holds one clone gives one mask of every + row, so a one-clone frame's walk runs on its rows as a frame without the + index does. The index must hold one integer per person; the refusals name + no count, clone or row. + """ + + if clone_index is None: + return (np.ones(rows, dtype=bool),) + labels = np.asarray(clone_index) + if labels.shape != (rows,): + raise ValueError("person_clone_index must hold one clone per person.") + if labels.dtype.kind not in "iu": + raise TypeError("person_clone_index must hold an integer clone per person.") + clones = np.unique(labels) + if clones.size <= 1: + return (np.ones(rows, dtype=bool),) + return tuple(labels == clone for clone in clones) + + +def _clone_zero_people( + clone_zero: np.ndarray | None, clone_index: np.ndarray | None, rows: int +) -> np.ndarray: + """The people the receipt counts support among: ``clone_zero``, or where + it is not given the people ``clone_index`` puts in clone 0, or everyone. + Both given, they must agree; the refusals name no count or row.""" + + if clone_zero is None: + if clone_index is None: + return np.ones(rows, dtype=bool) + return np.asarray(clone_index) == 0 + flags = np.asarray(clone_zero, dtype=bool) + if flags.shape != (rows,): + raise ValueError("person_clone_zero must hold one flag per person.") + if clone_index is not None and not np.array_equal( + flags, np.asarray(clone_index) == 0 + ): + raise ValueError( + "person_clone_zero must flag exactly the people whose " + "person_clone_index is zero." + ) + return flags + + def _nearest_target_threshold( need: float, *, draws: np.ndarray, weights: np.ndarray ) -> float: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py index 11a503d900..8ae1cb27dd 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_spine.py @@ -13,13 +13,19 @@ from microcosm.build.source_manifest import SourceStageSpec from microcosm.build.stochastic_assignment import stable_identity_uniforms from microcosm.build.trace import sha256_file +from microcosm.build.uk_runtime import copy_id_table +from microcosm.build.uk_runtime.annuity_trust_column import ( + ANNUITY_AND_TRUST_INCOME_COLUMN, +) from microcosm.build.uk_runtime.birth_month import ( UK_BIRTH_MONTH_COLUMN, uk_months_since_last_birthday, ) from microcosm.build.uk_runtime.council_tax_constants import ( UK_COUNCIL_TAX_BANDS, + UK_COUNCIL_TAX_BANDS_BY_REGION, UK_COUNCIL_TAX_PLACEHOLDER_BAND, + UK_NATION_BY_REGION, ) from microcosm.build.uk_runtime.national_frame import ( uk_national_frame, @@ -40,9 +46,12 @@ from microcosm.frame import Frame, WeightKind __all__ = [ + "FRS_AREA_CONDITIONING_COLUMNS", "FRS_CARE_HELPER_BIT", "FRS_CARE_LINE_HOURS", "FRS_CARE_RECIPIENTS_COLUMN", + "FRS_COUNCIL_TAX_BAND_OBSERVED_COLUMN", + "FRS_IMD_DECILE_COLUMN", "FRS_SPINE_TABLES", "FRS_UC_CARER_ELEMENT_COLUMN", "OUTPUT_COLUMNS", @@ -153,6 +162,41 @@ code: band for code, band in enumerate(UK_COUNCIL_TAX_BANDS, start=1) } +#: The two household carriers the clone-first spine's area assignment +#: conditions on (microcosm#931): each household's IMD decile within its own +#: nation, and whether its council_tax_band is a band the FRS gives it that +#: its region uses. The root writes them typed (an int64 decile, 0 where none +#: is observed, since the root refuses NaN; a bool flag); neither is an engine +#: input, so they leave at the release boundary +#: (``national_frame.UK_RELEASE_EXPORT_DROPPED_COLUMNS``), and they stay out +#: of every receipt and of the spine sidecar's nonzero shares. The decile's +#: name is spelled here, not imported from ``atomic_area_support``, whose +#: conditioned supports carry the areas' deciles under the same name, so an +#: edit to that module does not re-key the root; a test holds the spellings +#: equal. +FRS_IMD_DECILE_COLUMN = "imd_decile" +FRS_COUNCIL_TAX_BAND_OBSERVED_COLUMN = "council_tax_band_observed" +FRS_AREA_CONDITIONING_COLUMNS = ( + FRS_IMD_DECILE_COLUMN, + FRS_COUNCIL_TAX_BAND_OBSERVED_COLUMN, +) +#: Each nation's FRS household IMD decile variable (HOUSEHOL IMDE, IMDW, IMDS +#: and IMDN), by the nation names of ``council_tax_constants``: the +#: deprivation tenth of the address within that nation's own index, 1 the +#: most deprived. A household reads its own nation's variable only. +FRS_IMD_DECILE_COLUMN_BY_NATION: Mapping[str, str] = MappingProxyType( + { + "ENGLAND": "imde", + "WALES": "imdw", + "SCOTLAND": "imds", + "NORTHERN_IRELAND": "imdn", + } +) +#: The decile codes; anything else, a blank included, is no observed decile. +FRS_IMD_DECILES = tuple(range(1, 11)) +#: The decile a household with no observed decile carries. +FRS_IMD_DECILE_UNOBSERVED = 0 + MARITAL_MAP = { 1: "MARRIED", 2: "SINGLE", @@ -436,11 +480,11 @@ def benefits_by_recipient( #: (ITTOIA 2005 s. 717). SPI Table 3.7 counts both as other income (note 6), #: so the part they make of private_pension_income is stored apart, in #: ANNUITY_AND_TRUST_INCOME_COLUMN, a calibration-only descriptor that the -#: Table 3.7 other-income measure adds (microcosm#1428 review, finding 1). +#: Table 3.7 other-income measure adds (microcosm#1428 review, finding 1). The +#: descriptor's name lives in annuity_trust_column, which holds nothing else, +#: so the dense measures and the national problem, which read it, key on it +#: without keying on this module (microcosm#931, ruled 9 Nov (late)). FRS_ANNUITY_AND_TRUST_PENSION_TYPES = (4, 5) -#: The descriptor: each person's annual annuity and trust payments, already -#: inside private_pension_income. No engine input reads it. -ANNUITY_AND_TRUST_INCOME_COLUMN = "annuity_and_trust_income" #: Each person's company car from the FRS job tab, the four carriers the #: company_cars stage reads to set policyengine-uk's company car inputs #: (PolicyEngine/uk-autumn-budget-dashboard-2026#316); see @@ -554,11 +598,13 @@ def benefits_by_recipient( "liable_for_share_of_household_rent", "household_id", "region", + FRS_IMD_DECILE_COLUMN, "tenure_type", "accommodation_type", "num_bedrooms", "council_tax_reported", "council_tax_band", + FRS_COUNCIL_TAX_BAND_OBSERVED_COLUMN, "council_tax_rebate", "council_tax_single_adult_raw", "water_and_sewerage_charges", @@ -677,9 +723,32 @@ def build_uk_frs_spine_frame( normalized = {name: _normalize_ids(table) for name, table in tables.items()} frame = _assemble_frame(normalized, evidence=evidence) validate_uk_national_frame(frame) + _require_raw_ids_below_copy_fields(frame) return frame +def _require_raw_ids_below_copy_fields(frame: Frame) -> None: + """Refuse raw ids that reach the first copy-id field (microcosm#931). + + Every copy layer adds its ordinal times a declared radix to its copies' + ids, the lowest radix being the reserved ``vintage`` field's, so a raw + id (person_id is sernum x 1000 + p) must sit below it. + """ + + person = frame.table("person") + for entity, column in ( + ("person", "person_id"), + ("person", "person_household_id"), + ("person", "person_benunit_id"), + ("benunit", "benunit_id"), + ("household", "household_id"), + ): + table = person if entity == "person" else frame.table(entity) + copy_id_table.require_below( + table[column], "vintage", what=f"raw FRS {column} values" + ) + + def _artifact_by_table(stage: SourceStageSpec) -> dict[str, Mapping[str, Any]]: by_table: dict[str, Mapping[str, Any]] = {} for artifact in stage.artifacts: @@ -1826,6 +1895,7 @@ def _add_household_columns( pe_household["region"] = _map_dictionary_codes( household, "gvtregno", REGION_MAP, blank="UNKNOWN" ) + pe_household[FRS_IMD_DECILE_COLUMN] = frs_imd_decile(household) pe_household["tenure_type"] = _map_dictionary_codes( household, "ptentyp2", TENURE_MAP, blank="RENT_PRIVATELY" ) @@ -1837,6 +1907,9 @@ def _add_household_columns( pe_household["num_bedrooms"] = _number(household, "bedroom6") pe_household["council_tax_reported"] = _positive(household, "ctannual") pe_household["council_tax_band"] = frs_council_tax_band(household) + pe_household[FRS_COUNCIL_TAX_BAND_OBSERVED_COLUMN] = frs_council_tax_band_observed( + household + ) pe_household["council_tax_rebate"] = ( _positive(household, "ctrebamt") * WEEKS_IN_YEAR ) @@ -1936,6 +2009,70 @@ def frs_council_tax_band(household: pd.DataFrame) -> pd.Series: return band.mask(region == "NORTHERN_IRELAND", UK_COUNCIL_TAX_PLACEHOLDER_BAND) +def frs_council_tax_band_observed(household: pd.DataFrame) -> pd.Series: + """Whether each household's ``council_tax_band`` is a band the FRS gives it + that its region uses. + + True where ``ctband`` codes a band, as ``frs_council_tax_band`` maps the + codes 1-9, that the household's region uses + (``UK_COUNCIL_TAX_BANDS_BY_REGION``: A-H in England and Scotland, A-I in + Wales). False in Northern Ireland, which pays domestic rates, for any + other code, which takes the placeholder band, for a band the region does + not use (band I outside Wales), and for a household of no known region + (microcosm#931). The band itself is unchanged; the flag tells the + survey's band from the placeholder and from a band its region does not + hold. + """ + + # The band each code maps to, by the band's own mapping; an unmapped code + # takes the empty string, which no region uses. + bands = _map_codes(household, "ctband", COUNCIL_TAX_BAND_MAP, "") + region = _map_dictionary_codes(household, "gvtregno", REGION_MAP, blank="UNKNOWN") + region_bands = region.map(lambda name: UK_COUNCIL_TAX_BANDS_BY_REGION.get(name, ())) + in_region = np.fromiter( + ( + band in allowed + for band, allowed in zip(bands.tolist(), region_bands.tolist(), strict=True) + ), + dtype=bool, + count=len(household), + ) + observed = in_region & (region != "NORTHERN_IRELAND").to_numpy(dtype=bool) + return pd.Series(observed, index=household.index, dtype=bool) + + +def frs_imd_decile(household: pd.DataFrame) -> pd.Series: + """Each household's IMD decile within its own nation: 1-10, else 0. + + The FRS gives the decile of each address on its own nation's index, one + variable per nation (``FRS_IMD_DECILE_COLUMN_BY_NATION``); a household + reads its own nation's variable, its nation coming from its region. A + code outside 1-10, a blank, and a household of no known region carry 0, + no observed decile, since the root refuses NaN (microcosm#931). A tab + without any of the four variables refuses, naming them, rather than read + an absent variable as zeros and so give nobody a decile. + """ + + missing = [ + column.upper() + for column in FRS_IMD_DECILE_COLUMN_BY_NATION.values() + if column not in household.columns + ] + if missing: + raise KeyError( + f"The FRS household tab has no {missing} column(s), so the IMD " + "decile cannot be set." + ) + region = _map_dictionary_codes(household, "gvtregno", REGION_MAP, blank="UNKNOWN") + nation = region.map(UK_NATION_BY_REGION) + decile = np.full(len(household), FRS_IMD_DECILE_UNOBSERVED, dtype=np.int64) + for name, column in FRS_IMD_DECILE_COLUMN_BY_NATION.items(): + codes = _raw_number(household, column) + rows = ((nation == name) & codes.isin(FRS_IMD_DECILES)).to_numpy(dtype=bool) + decile[rows] = codes.to_numpy(dtype=np.float64)[rows].astype(np.int64) + return pd.Series(decile, index=household.index, dtype="int64") + + def _map_codes( frame: pd.DataFrame, column: str, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_take_up.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_take_up.py index b62cbd78f7..31a5d648c4 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_take_up.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/frs_take_up.py @@ -19,13 +19,16 @@ stable_identity_uniforms, ) from microcosm.build.uk_runtime.dfe_spring_term import dfe_age_at_31_december +from microcosm.build.uk_runtime.geographic_clones import ladder_clone_index_column from microcosm.build.uk_runtime.national_frame import ( + UK_NATIONAL_SCHEMA, uk_household_weight_kind, uk_national_frame, uk_time_period, validate_uk_national_frame, ) from microcosm.build.uk_runtime.receipt_counts import ( + UK_DISCLOSURE_FLOOR, distinct_count, extreme_count, record_source_households, @@ -403,7 +406,13 @@ def add_frs_take_up( anchors = aggregate_person_reported_to_benunit(person, benunit) uc_age_eligible = uc_age_eligible_benunits(person, benunit, population_policy) derived = derive_frs_take_up( - benunit, anchors=anchors, contract=contract, uc_age_eligible=uc_age_eligible + benunit, + anchors=anchors, + contract=contract, + uc_age_eligible=uc_age_eligible, + # Each unit's household's clone, read through the person table's + # memberships; None on a frame without household_clone_index. + clone_index=geographic_clone_indices(frame, "benunit"), ) for column in FRS_TAKE_UP_OUTPUT_COLUMNS: benunit[column] = derived[column].to_numpy() @@ -477,16 +486,34 @@ def derive_frs_take_up( anchors: pd.DataFrame, contract: UKTakeUpContract, uc_age_eligible: np.ndarray, + clone_index: np.ndarray | None = None, ) -> pd.DataFrame: + """The stage's benefit-unit outputs, each drawn on ``benunit_id`` under + the output's own salt. + + The Child Benefit, Pension Credit and Universal Credit flags fill their + anchored residuals within each geographic clone (microcosm#931): + ``clone_index``, aligned with ``benunit``, gives each unit's clone and is + the anchored draws' ``strata``. Clone 0's units have a one-clone spine's + ids, anchors and population, so they take its flags exactly at any clone + count; one count over every clone would round the target once over all + of them and move clone 0's threshold. ``None``, for a frame without the + clone index, counts over every unit, as clone indices that are all 0 do. + """ + ids = benunit["benunit_id"].to_numpy() values = pd.DataFrame(index=benunit.index) population = np.asarray(uc_age_eligible, dtype=bool) if population.shape != ids.shape: raise ValueError("uc_age_eligible must align with the benefit-unit table") + strata = None if clone_index is None else np.asarray(clone_index) + if strata is not None and strata.shape != ids.shape: + raise ValueError("clone_index must align with the benefit-unit table") values["would_claim_child_benefit"] = assign_binary_with_anchored_residual( _draws(ids, "would_claim_child_benefit"), contract.rate("child_benefit"), anchors["child_benefit_reported_anchor"].to_numpy(dtype=bool), + strata=strata, ) values["child_benefit_opts_out"] = assign_binary_from_rate( _draws(ids, "child_benefit_opts_out"), @@ -496,12 +523,14 @@ def derive_frs_take_up( _draws(ids, "would_claim_pc"), contract.rate("pension_credit"), anchors["pension_credit_reported_anchor"].to_numpy(dtype=bool), + strata=strata, ) values["would_claim_uc"] = assign_binary_with_anchored_residual( _draws(ids, "would_claim_uc"), contract.rate("universal_credit"), anchors["universal_credit_reported_anchor"].to_numpy(dtype=bool), population=population, + strata=strata, ) childcare = derive_childcare_take_up(ids, contract=contract) for output, _ in UK_CHILDCARE_TAKE_UP_FLAGS: @@ -730,6 +759,84 @@ def uk_take_up_signal_gate( ) +def geographic_clone_zero_rows(frame: Frame, entity: str) -> np.ndarray: + """Whether each ``entity`` row belongs to geographic clone 0 (microcosm#931). + + A row's clone is its household's ``household_clone_index``: a person's + household, or the household a benefit unit's members share. A frame + without the index is one clone, every row clone 0's. A count a receipt + takes in clone 0's rows does not grow with the clone count, and on a + one-clone spine it is the count over every row. + """ + + household = frame.table("household") + column = ladder_clone_index_column("household") + if column not in household.columns: + return np.ones(len(frame.table(entity)), dtype=bool) + return np.asarray( + _household_values_on_rows(frame, entity, household[column].to_numpy() == 0), + dtype=bool, + ) + + +def geographic_clone_indices(frame: Frame, entity: str) -> np.ndarray | None: + """Each ``entity`` row's geographic clone, or ``None`` (microcosm#931). + + A row's clone is its household's ``household_clone_index``, as + :func:`geographic_clone_zero_rows` reads it: a person's household's, or + that of the household a benefit unit's members share, through the person + table's memberships, which a stage's frame always carries. + ``frs_take_up`` and ``frs_person_draws`` consume the household column + alone, so their draws count within each clone without reading another + table's index. A frame without the column (a unit fixture) is one clone + and gives ``None``. + """ + + household = frame.table("household") + column = ladder_clone_index_column("household") + if column not in household.columns: + return None + return _household_values_on_rows(frame, entity, household[column].to_numpy()) + + +def _household_values_on_rows( + frame: Frame, entity: str, values: np.ndarray +) -> np.ndarray: + """``values``, one per household row, on each ``entity`` row: a person's + household's, or that of the household a benefit unit's members share. + The refusals name no id, count or value.""" + + household = frame.table("household") + by_household = pd.Series( + values, index=household[UK_NATIONAL_SCHEMA.id_column("household")].to_numpy() + ) + if entity == "household": + return by_household.to_numpy() + person = frame.table(UK_NATIONAL_SCHEMA.person_entity) + on_person = person[UK_NATIONAL_SCHEMA.membership_column("household")].map( + by_household + ) + if on_person.isna().any(): + raise ValueError( + "Every person needs a household on the frame to read its geographic clone." + ) + if entity == UK_NATIONAL_SCHEMA.person_entity: + return on_person.to_numpy() + by_unit = ( + pd.Series( + on_person.to_numpy(), + index=person[UK_NATIONAL_SCHEMA.membership_column(entity)].to_numpy(), + ) + .groupby(level=0) + .first() + ) + table = frame.table(entity) + on_unit = table[UK_NATIONAL_SCHEMA.id_column(entity)].map(by_unit) + if on_unit.isna().any(): + raise ValueError(f"A {entity} without members has no geographic clone to read.") + return on_unit.to_numpy() + + @dataclass(frozen=True) class _TakeUpSignal: """One flag's weighted share over its draw's population.""" @@ -748,14 +855,21 @@ def _measure_take_up_signal( output: str, key: str, contract: UKTakeUpContract, + clone_zero: np.ndarray | None = None, ) -> _TakeUpSignal | str: - """One flag's share at the frame's weights, or why it cannot be measured.""" + """One flag's share at the frame's weights, or why it cannot be measured. + + The flag's distinct values are counted in geographic clone 0's rows, + ``clone_zero``, as one clone counts them (microcosm#931); without it, in + every row, as the terminal gate counts them. + """ table = frame.table(entity) if output not in table.columns: return "missing stochastic column." values = np.asarray(table[output], dtype=bool) - unique_count = int(pd.Series(values).nunique(dropna=False)) + counted = values if clone_zero is None else values[clone_zero] + unique_count = int(pd.Series(counted).nunique(dropna=False)) weights = np.asarray(frame.resolve_weights(entity).values, dtype=np.float64) population = np.ones(values.shape, dtype=bool) if key == UK_UNDER_TWO_USAGE_SIGNAL_KEY: @@ -818,29 +932,55 @@ def uk_take_up_signal_receipt( 2026-10-29, microcosm#1095). The receipt is aggregate-only (UKDS disclosure control): a share or a target is given only where three or more source households carry the flag's population, whose support is a band. + + The population's support and the flag's distinct values are counted in + geographic clone 0's rows (:func:`geographic_clone_zero_rows`; + microcosm#931): each clone draws its own flags, and the under-2 usage + population with them, so a count over every clone would grow with the + clone count. The share and the target are measured over every clone, so + each is withheld where one or two source households carry the population + in clone 0's rows or over every clone: clone 0 can carry none of a + population that one or two households carry in the other clones. """ sources: dict[str, np.ndarray] = {} + clone_zero: dict[str, np.ndarray] = {} flags: list[dict[str, object]] = [] for entity, output, key in outputs: row: dict[str, object] = {"output": f"{entity}.{output}", "rate_key": key} if key == UK_UNDER_TWO_USAGE_SIGNAL_KEY: row["under_two_usage"] = True - measured = _measure_take_up_signal(frame, entity, output, key, contract) + if entity not in clone_zero: + clone_zero[entity] = geographic_clone_zero_rows(frame, entity) + measured = _measure_take_up_signal( + frame, entity, output, key, contract, clone_zero[entity] + ) if isinstance(measured, str): row["unmeasured"] = measured flags.append(row) continue if entity not in sources: - # Before the SPI support channel each household is its own source, - # so its id serves until ``source_household_id`` exists. + # On the clone-first spine geographic_expand writes + # source_household_id before this stage runs, so the shared reader + # counts a household's geographic clones once; a frame without the + # column or copy lineage (unit fixtures) counts each household as + # its own source (microcosm#931). sources[entity] = record_source_households( frame, entity, stage="The take-up signal receipt", before_copies=True ) - support = distinct_count(sources[entity], measured.population, measured.weights) + support = distinct_count( + sources[entity], + measured.population & clone_zero[entity], + measured.weights, + ) + carriers = distinct_count( + sources[entity], measured.population, measured.weights + ) + # The figures are measured over every clone: withheld on either count. + withheld_on = carriers if 0 < carriers < UK_DISCLOSURE_FLOOR else support row["population_support"] = support_band(support) - row["target"] = small_cell_share(measured.target, support) - row["weighted_share"] = small_cell_share(measured.share, support) + row["target"] = small_cell_share(measured.target, withheld_on) + row["weighted_share"] = small_cell_share(measured.share, withheld_on) row["distinct_values"] = measured.unique_count if measured.nested_violations is not None: row["extended_without_universal_units"] = extreme_count( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py index 5e32ba43ff..f1786b6fc9 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py @@ -14,7 +14,21 @@ :mod:`microcosm.build.uk_runtime.national_role`) materialised from the graph's stored artifacts. Both lines are built here through the graph. The role supplies every unset solve default and refuses the other role's flags -through :mod:`microcosm.build.uk_runtime.rowwise_cli`. +through :mod:`microcosm.build.uk_runtime.rowwise_cli`. The dense line builds +only on a geography-first spine: :func:`prepare_full_build` refuses any other +before it pins an input or composes a graph +(:mod:`microcosm.build.uk_runtime.spine_posture`, microcosm#931), and takes the +spine's geographic clone count, sampling and areas as its own: its targets, +measures and pool run on the spine's version as it stands, and the run's +recorded clone count and rung are the spine's. Its engine blocks are one or +the spine's K, and a release candidate needs an unsampled spine of the +doctrine's K, built as a release candidate, whose geography binding is a +release's (:func:`~microcosm.build.uk_runtime.country_adapter.uk_spine_geography_binding_failures`). +The national line builds only on a geography-first spine of one geographic +clone, and only once the degenerate-surface register's area-basis exclusion +is signed for the national line and in force on the build's review date: +:func:`prepare_national_build` refuses otherwise before it pins the input or +does any engine work (microcosm#931). ``tools/build_uk_rowwise_candidate.py`` and ``tools/build_uk_full.py`` are stubs over :func:`main`, so every UK line is built by this one command. @@ -35,7 +49,9 @@ from __future__ import annotations import argparse +import contextlib import hashlib +import io import json import sys import tempfile @@ -44,7 +60,6 @@ from collections.abc import Callable, Collection, Mapping from dataclasses import asdict, dataclass, replace from datetime import UTC, date, datetime -from importlib.resources import files from pathlib import Path from typing import TYPE_CHECKING, Protocol @@ -73,7 +88,6 @@ run_graph, ) from microcosm.graph.canonical import canonical_json -from microcosm.graph.codecs import load_raw_bytes from ..logbook_adoption import ( AttemptState, @@ -100,19 +114,7 @@ handover_record, refresh_sha256sums_entry, ) -from . import country_adapter, national_role -from .atomic_area_support import ( - IDENTITY_COLUMN, - uk_atomic_assignment_definition, -) -from .atomic_area_support import ( - SOURCES as ATOMIC_SUPPORT_SOURCES, -) -from .atomic_area_support import ( - SYSTEMS as ATOMIC_SUPPORT_SYSTEMS, -) -from .brma_area_layer import BRMA_AREA_LAYER_SOURCE -from .brma_area_layer import METHOD_VERSION as BRMA_AREA_LAYER_METHOD +from . import national_role from .calibration_run import ( _write_json, load_bound_spine_checkpoint, @@ -121,6 +123,7 @@ spine_provenance_from_sidecar, ) from .chronicle_feed import load_uk_chronicle_feed +from .country_adapter import uk_spine_geography_binding_failures from .dataset_size import ( UK_SIZE_L2_ANCHORS, UK_SIZE_L2_BASES, @@ -158,13 +161,6 @@ solved_national_registry, uk_national_graph, ) -from .graph_population import ( - BRMA_LOCALISATION_MODES, - GEOGRAPHY_ASSIGNMENTS, - PRIVATE_RENT_LOCALISATION_MODES, - PROPERTY_LOCALISATION_MODES, - RFDR_LOCALISATION_MODES, -) from .graph_targets import TARGET_SELECTION_TYPE, registry_from_payload from .graph_terminal import ( FULL_DIAGNOSTICS_CSV_TYPE, @@ -196,21 +192,6 @@ ) from .national_sampling import UK_SAMPLE_RUNG_TOKENS from .orrery_contract import persist_uk_run_evidence, save_uk_graph_schema -from .private_rent_localisation import COUNTS_SOURCE as PRIVATE_RENTER_COUNTS_SOURCE -from .private_rent_localisation import METHOD_VERSION as PRIVATE_RENT_METHOD -from .private_rent_localisation import QUANTILES_SOURCE as RENT_QUANTILES_SOURCE -from .property_localisation import ( - BAND_MIX_SOURCE, - QUANTILES_SOURCE, - SCOTLAND_BAND_MIX_RESOURCE, - SCOTLAND_BAND_MIX_SOURCE, - SUPPORT_STRATUM_LOWER, -) -from .property_localisation import ( - METHOD_VERSION as PROPERTY_LOCALISATION_METHOD, -) -from .rfdr_area_layer import METHOD_VERSION as RFDR_AREA_LAYER_METHOD -from .rfdr_area_layer import RFDR_AREA_LAYER_SOURCE from .rowwise_cli import ( MANIFEST_FILENAME, REPOSITORY, @@ -257,6 +238,13 @@ ) from .size_checkpoint import uk_size_checkpoint_identity from .solve_scope import UK_GATE_ONLY_RESOURCES, UK_POST_SOLVE_RESOURCES +from .spine_posture import ( + require_geography_first_spine, + require_national_exclusion_signed, + require_national_spine, + spec_roster, + stage_roster, +) from .staging import UK_STAGED_DATASET_REPOSITORY, UK_STAGING_REPOSITORY from .weight_year import uk_dataset_year_frame @@ -285,11 +273,6 @@ #: local battery report takes the posture's ``*.local_gates.json`` name. FULL_GATE_REPORT_FILENAME = "uk.full.gates.calibrated.gate_report.json" -#: CLI labels for the three UK atomic-area support systems, in SYSTEMS order. -_SUPPORT_ARGUMENTS = dict( - zip(ATOMIC_SUPPORT_SYSTEMS, ("ew", "scotland", "ni"), strict=True) -) - def _run_graph_with_progress(compiled, *, retain_populations, execution=None, **kwargs): """Run a graph and emit one lightweight update per completed node. @@ -485,120 +468,6 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: help="Full-UK OA geography ladder NPZ (required by the dense role).", ) parser.add_argument("--ladder-sha256", type=sha256_argument) - parser.add_argument( - "--geography-assignment", - choices=GEOGRAPHY_ASSIGNMENTS, - default="atomic", - help=( - "Dense role: atomic (default) is the identity-keyed single-stage " - "draw on the three UK atomic-area supports; legacy is the sequential " - "ladder draw, for measurement builds only." - ), - ) - for system, label in _SUPPORT_ARGUMENTS.items(): - parser.add_argument( - f"--atomic-support-{label}", - type=Path, - dest=f"atomic_support_{label}", - help=f"Atomic-area support NPZ for {system} (dense role, required with atomic).", - ) - parser.add_argument( - f"--atomic-support-sha256-{label}", - type=sha256_argument, - dest=f"atomic_support_sha256_{label}", - ) - parser.add_argument( - "--property-localisation", - choices=PROPERTY_LOCALISATION_MODES, - default="price_paid", - help=( - "Dense role: price_paid (default) reads each owner clone's home value " - "off its assigned constituency's uprated Land Registry sale distribution " - "and its council-tax band off its LSOA's VOA mix (microcosm#1240), or a " - "Scottish owner clone's off its council area's CTAXBASE mix " - "(microcosm#1095 B12); none " - "keeps the spine's values and bands on every clone, for measurement " - "builds only." - ), - ) - for name, label in _PROPERTY_RESOURCE_ARGUMENTS.items(): - parser.add_argument( - f"--{label}", - type=Path, - dest=label.replace("-", "_"), - default=None, - help=( - f"Resource bytes for the graph source {name!r} (default: the " - "committed UK package resource)." - ), - ) - parser.add_argument( - f"--{label}-sha256", - type=sha256_argument, - dest=f"{label.replace('-', '_')}_sha256", - ) - parser.add_argument( - "--brma-localisation", - choices=BRMA_LOCALISATION_MODES, - default="area", - help=( - "Dense role: area (default) reads each placed Great Britain clone's BRMA " - "off its assigned LSOA or output area (microcosm#1262); none keeps the " - "spine's regional draw on every clone, for measurement builds only." - ), - ) - parser.add_argument( - "--brma-area-layer", - type=Path, - default=None, - help=( - "Resource bytes for the graph source 'uk_brma_area_layer' (default: the " - "committed UK package resource)." - ), - ) - parser.add_argument("--brma-area-layer-sha256", type=sha256_argument) - parser.add_argument( - "--rfdr-localisation", - choices=RFDR_LOCALISATION_MODES, - default=None, - help=( - "Dense role: area (the default with the atomic assignment) sets each " - "placed household's Rural Fuel Duty Relief flag off its assigned output " - "area (microcosm#1418); none (the default with the legacy assignment) " - "leaves the engine's default, for measurement builds only." - ), - ) - parser.add_argument( - "--rfdr-area-layer", - type=Path, - default=None, - help=( - "Resource bytes for the graph source 'uk_rfdr_area_layer' (default: the " - "committed UK package resource)." - ), - ) - parser.add_argument("--rfdr-area-layer-sha256", type=sha256_argument) - parser.add_argument( - "--private-rent-localisation", - choices=PRIVATE_RENT_LOCALISATION_MODES, - default=None, - help=( - "Dense role: brma_list (the default with area BRMAs) re-draws each Great " - "Britain private renter's rent from its assigned BRMA's list of rents at " - "its FRS rank and region level (microcosm#1262); none (the default " - "without them) keeps the spine's rents, for measurement builds only." - ), - ) - parser.add_argument( - "--brma-rent-quantiles", - type=Path, - default=None, - help=( - "Resource bytes for the graph source 'uk_brma_rent_quantiles' (default: " - "the committed UK package resource)." - ), - ) - parser.add_argument("--brma-rent-quantiles-sha256", type=sha256_argument) parser.add_argument( "--ledger-facts", type=Path, @@ -640,7 +509,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: type=_target_geographies, default=None, metavar="all|country,...", - help="Default all: calibrate all applicable geographies together. country is an explicit filter in this same build; brma (Scotland's BRMA rent rows) needs area BRMAs.", + help="Default all: calibrate all applicable geographies together. country is an explicit filter in this same build; brma (Scotland's BRMA rent rows) binds on the BRMAs the spine reads off its areas.", ) parser.add_argument( "--households-only", @@ -650,12 +519,19 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument( "--n-clones", type=int, - help="Geographic pool copies K; defaults to the role's doctrine clone count.", + help=( + "Dense role: an assertion of the bound spine's geographic clone " + "count K, which the build takes from the spine; refused where it " + "differs from the spine's, and with --candidate-clone-counts." + ), ) parser.add_argument( "--candidate-clone-counts", type=candidate_clone_counts_argument, - help="Dry-run only comma-separated candidate clone counts.", + help=( + "Dry-run only, with --spine-request: comma-separated candidate " + "clone counts, each planned as the spine request's --n-clones." + ), ) parser.add_argument( "--dataset-households", @@ -666,10 +542,15 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: "--sample-fraction", type=float, default=1.0, - help="Optional pool sampling before cloning. Cannot resample an already sampled source spine.", + help=( + "Refused other than 1.0: the dense line takes its sampling from " + "the spine it binds; sample the spine at its own build." + ), ) parser.add_argument( - "--sample-seed", type=int, help="Dense role only; defaults to the spine seed." + "--sample-seed", + type=int, + help="Refused: the dense line takes its sampling seed from its spine.", ) parser.add_argument("--seed", type=int, help="Defaults to the role's seed.") parser.add_argument("--selection-seed", type=int) @@ -769,14 +650,17 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: ), help="Defaults to the role's doctrine; any other admitted rule is a receipted override.", ) - parser.add_argument("--engine-blocks", type=int, default=1) - parser.add_argument("--source-year", type=int) - parser.add_argument("--calibration-year", type=int) - parser.add_argument("--source-lineage-modulus", type=int) parser.add_argument( - "--expected-constituency-vintage", - help="Dense role only: constituency vintage required from the ladder.", + "--engine-blocks", + type=int, + default=1, + help=( + "Dense role: engine resolution blocks, 1 (the default) or one per " + "geographic clone, the spine's K; any other count is refused." + ), ) + parser.add_argument("--source-year", type=int) + parser.add_argument("--calibration-year", type=int) parser.add_argument( "--rotated-holdout", action="store_true", @@ -798,7 +682,17 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: "block. Refused with --release-candidate." ), ) - parser.add_argument("--release-candidate", action="store_true") + parser.add_argument( + "--release-candidate", + action="store_true", + help=( + "Dense role: certify a release candidate. The spine must be " + "unsampled, at the doctrine's " + f"{UK_ROWWISE_DENSE_POSTURE.clone_count} geographic clones, built " + "as a release candidate, with a release's geography binding " + "(microcosm#931)." + ), + ) parser.add_argument( "--review-date", type=date.fromisoformat, @@ -973,274 +867,6 @@ def _input_record(path: Path, pin: dict, *, pinned: bool) -> dict: } -#: Graph source name to command-line label of the property-localisation -#: resources (microcosm#1240; the Scottish band mix, microcosm#1095 B12). Each -#: defaults to the committed package file. -_PROPERTY_RESOURCE_ARGUMENTS = { - QUANTILES_SOURCE: "price-paid-quantiles", - BAND_MIX_SOURCE: "voa-band-mix", - SCOTLAND_BAND_MIX_SOURCE: "scotgov-band-mix", -} -_PROPERTY_RESOURCE_FILES = { - QUANTILES_SOURCE: "high_value_home_value_knots.json", - BAND_MIX_SOURCE: "voa_council_tax_band_mix_lsoa.json", - SCOTLAND_BAND_MIX_SOURCE: SCOTLAND_BAND_MIX_RESOURCE, -} - - -def _packaged_property_resource(name: str) -> Path: - return Path( - str(files("microcosm.build.uk").joinpath(_PROPERTY_RESOURCE_FILES[name])) - ) - - -def _prepare_property_localisation( - args: argparse.Namespace, pins: dict, sources: dict[str, Path] -) -> dict: - """Pin the knots and band-mix resources and record the localisation binding. - - Each resource enters ``pins`` under its graph source name, so the Logbook - input-pins digest, the candidate identity digest and the size-checkpoint - identity all bind it. A release candidate must build on the committed - package resources, whatever path the operator named: the committed bytes - are the reviewed ones. ``none`` binds nothing here. - """ - mode = getattr(args, "property_localisation", "none") - if mode == "none": - return {"mode": "none"} - resource_pins = {} - for name, label in _PROPERTY_RESOURCE_ARGUMENTS.items(): - attribute = label.replace("-", "_") - path = getattr(args, attribute, None) or _packaged_property_resource(name) - pin = _pin(path, getattr(args, f"{attribute}_sha256", None)) - if getattr(args, "release_candidate", False): - committed = _pin(_packaged_property_resource(name)) - if pin != committed: - raise ValueError( - f"--release-candidate requires the committed {name!r} resource: " - f"sha256 {pin['sha256']} is not the package's {committed['sha256']}." - ) - pins[name] = pin - resource_pins[name] = pin - sources[name] = path - return { - "mode": mode, - "method": PROPERTY_LOCALISATION_METHOD, - "support_stratum_lower": SUPPORT_STRATUM_LOWER, - "resource_pins": resource_pins, - } - - -def _prepare_brma_localisation( - args: argparse.Namespace, pins: dict, sources: dict[str, Path] -) -> dict: - """Pin the area-to-BRMA layer and record the BRMA binding (microcosm#1262). - - The layer enters ``pins`` under its graph source name, as the property - resources do, and a release candidate must build on the committed bytes. - ``none`` binds nothing here. - """ - mode = getattr(args, "brma_localisation", "none") - if mode == "none": - if getattr(args, "brma_area_layer", None) or getattr( - args, "brma_area_layer_sha256", None - ): - raise ValueError( - "--brma-area-layer and --brma-area-layer-sha256 need " - "--brma-localisation area: without it nothing reads the layer." - ) - return {"mode": "none"} - committed_path = Path( - str(files("microcosm.build.uk").joinpath("brma_area_layer.json")) - ) - path = getattr(args, "brma_area_layer", None) or committed_path - pin = _pin(path, getattr(args, "brma_area_layer_sha256", None)) - if getattr(args, "release_candidate", False): - committed = _pin(committed_path) - if pin != committed: - raise ValueError( - "--release-candidate requires the committed BRMA area layer: " - f"sha256 {pin['sha256']} is not the package's {committed['sha256']}." - ) - pins[BRMA_AREA_LAYER_SOURCE] = pin - sources[BRMA_AREA_LAYER_SOURCE] = path - return { - "mode": mode, - "method": BRMA_AREA_LAYER_METHOD, - "resource_pins": {BRMA_AREA_LAYER_SOURCE: pin}, - } - - -def _prepare_rfdr_localisation( - args: argparse.Namespace, pins: dict, sources: dict[str, Path] -) -> dict: - """Pin the Rural Fuel Duty Relief area layer and record its binding. - - As with the BRMA layer, the layer enters ``pins`` under its graph source - name and a release candidate must build on the committed bytes - (microcosm#1418). ``none`` binds nothing here. - """ - mode = getattr(args, "rfdr_localisation", None) - if mode is None: - # Unset, the mode follows the geography assignment, as the build config - # resolves it (the independent review of microcosm#1420). - mode = ( - "area" - if getattr(args, "geography_assignment", "atomic") == "atomic" - else "none" - ) - args.rfdr_localisation = mode - if mode == "none": - if getattr(args, "rfdr_area_layer", None) or getattr( - args, "rfdr_area_layer_sha256", None - ): - raise ValueError( - "--rfdr-area-layer and --rfdr-area-layer-sha256 need " - "--rfdr-localisation area: without it nothing reads the layer." - ) - return {"mode": "none"} - committed_path = Path( - str(files("microcosm.build.uk").joinpath("rfdr_area_layer.json")) - ) - path = getattr(args, "rfdr_area_layer", None) or committed_path - pin = _pin(path, getattr(args, "rfdr_area_layer_sha256", None)) - if getattr(args, "release_candidate", False): - committed = _pin(committed_path) - if pin != committed: - raise ValueError( - "--release-candidate requires the committed RFDR area layer: " - f"sha256 {pin['sha256']} is not the package's {committed['sha256']}." - ) - pins[RFDR_AREA_LAYER_SOURCE] = pin - sources[RFDR_AREA_LAYER_SOURCE] = path - return { - "mode": mode, - "method": RFDR_AREA_LAYER_METHOD, - "resource_pins": {RFDR_AREA_LAYER_SOURCE: pin}, - } - - -#: Graph source name to committed package file of the private-rent node's -#: resources (microcosm#1262). Only the quantiles take an override. -_PRIVATE_RENT_RESOURCE_FILES = { - RENT_QUANTILES_SOURCE: "brma_rent_quantiles.json", - PRIVATE_RENTER_COUNTS_SOURCE: "brma_private_rented_households.json", -} - - -def _prepare_private_rent_localisation( - args: argparse.Namespace, pins: dict, sources: dict[str, Path] -) -> dict: - """Pin the rent quantiles and census counts and record the rent binding. - - As with the property resources, each enters ``pins`` under its graph source - name and a release candidate must build on the committed bytes. ``none`` - binds nothing here. - """ - mode = getattr(args, "private_rent_localisation", None) - if mode is None: - # Unset, the mode follows the BRMA mode, as the build config resolves it. - mode = ( - "brma_list" - if getattr(args, "brma_localisation", "none") == "area" - else "none" - ) - args.private_rent_localisation = mode - if mode == "none": - if getattr(args, "brma_rent_quantiles", None) or getattr( - args, "brma_rent_quantiles_sha256", None - ): - raise ValueError( - "--brma-rent-quantiles and --brma-rent-quantiles-sha256 need " - "--private-rent-localisation brma_list: without it nothing " - "reads the quantiles." - ) - return {"mode": "none"} - resource_pins = {} - for name, filename in _PRIVATE_RENT_RESOURCE_FILES.items(): - committed_path = Path(str(files("microcosm.build.uk").joinpath(filename))) - override = ( - getattr(args, "brma_rent_quantiles", None) - if name == RENT_QUANTILES_SOURCE - else None - ) - expected = ( - getattr(args, "brma_rent_quantiles_sha256", None) - if name == RENT_QUANTILES_SOURCE - else None - ) - path = override or committed_path - pin = _pin(path, expected) - if getattr(args, "release_candidate", False): - committed = _pin(committed_path) - if pin != committed: - raise ValueError( - f"--release-candidate requires the committed {name!r} resource: " - f"sha256 {pin['sha256']} is not the package's {committed['sha256']}." - ) - pins[name] = pin - resource_pins[name] = pin - sources[name] = path - return { - "mode": mode, - "method": PRIVATE_RENT_METHOD, - "resource_pins": resource_pins, - } - - -def _prepare_geography( - args: argparse.Namespace, pins: dict, sources: dict[str, Path] -) -> tuple[dict | None, dict]: - """Pin the three supports and declare the identity-keyed assignment. - - The definition is the UK adapter's declaration over the admitted support - bytes at the build seed; its sha256 (and every keyed draw) moves iff a - support artifact or the seed moves. Each support enters ``pins`` as a - flat pin role named by its graph source, so the Logbook input-pins - digest, the candidate identity digest and the size-checkpoint identity - all bind it. Legacy builds bind nothing here. - """ - if args.geography_assignment != "atomic": - return None, {"assignment": "legacy"} - # A release candidate builds on the registered supports (sources.yaml, - # the provenance register): the operator's pin proves the bytes are the - # ones requested, the register proves they are the ones reviewed. - register = ( - country_adapter.uk_atomic_support_register() - if getattr(args, "release_candidate", False) - else None - ) - support_pins = {} - payloads = {} - for system, label in _SUPPORT_ARGUMENTS.items(): - path = getattr(args, f"atomic_support_{label}") - if path is None: - raise ValueError( - "Atomic geography assignment requires --atomic-support-ew, " - "--atomic-support-scotland and --atomic-support-ni." - ) - pin = _pin(path, getattr(args, f"atomic_support_sha256_{label}")) - if register is not None and pin != register[system]: - raise ValueError( - f"--release-candidate requires the registered atomic-area support " - f"for --atomic-support-{label}: sha256 {pin['sha256']} " - f"({pin['size_bytes']} bytes) is not the sources.yaml row " - f"{register[system]['sha256']} ({register[system]['size_bytes']} bytes)." - ) - pins[ATOMIC_SUPPORT_SOURCES[system]] = pin - support_pins[system] = pin - payloads[system] = load_raw_bytes(path) - sources[ATOMIC_SUPPORT_SOURCES[system]] = path - definition = uk_atomic_assignment_definition(payloads, seed=args.seed) - return definition, { - "assignment": "atomic", - "definition_sha256": hashlib.sha256(canonical_json(definition)).hexdigest(), - "support_pins": support_pins, - "identity": IDENTITY_COLUMN, - "stream": list(definition["stream"]), - } - - @dataclass(frozen=True) class PreparedUKFullBuild: full: UKFullGraph @@ -1305,6 +931,291 @@ def _require_gate_kernel_completed(manifest, node_id: str) -> None: ) +def _read_spine_sidecar(path: Path) -> dict: + """A bound spine's build sidecar as a JSON object. + + The refusals are ``load_bound_spine_checkpoint``'s, word for word, so a + sidecar that cannot be read is refused alike whichever reader meets it + first. + """ + if not path.is_file(): + raise ValueError(f"input H5 build sidecar absent: {path}") + try: + sidecar = json.loads(path.read_bytes()) + except (json.JSONDecodeError, UnicodeError) as exc: + raise ValueError(f"input H5 build sidecar is invalid JSON: {path}") from exc + if not isinstance(sidecar, dict): + raise ValueError(f"input H5 build sidecar must be a JSON object: {path}") + return sidecar + + +def _spine_request_roster() -> tuple[tuple[str, ...], dict[str, tuple[str, ...]]]: + """The stage names and operation kinds a spine request runs. + + Every request runs the packaged UK spec's whole roster: no spine argument + selects stages, and the spine build derives its roster from that spec, + refusing a spine graph that disagrees with it. So the roster is read off + the spec, without parsing the request or composing its spine. + """ + from microcosm.build.country_spec import load_country_spec + + return spec_roster(load_country_spec("uk")) + + +def _require_geography_first_spine(args: argparse.Namespace) -> None: + """Refuse a spine that does not assign areas, before any input is pinned. + + The dense line builds only on a geography-first spine (microcosm#931; + :mod:`~microcosm.build.uk_runtime.spine_posture`). A bound spine + (``--input-h5``) is read off its build sidecar: the stages it ran, their + operation kinds and its ``geography`` block. A spine request's stage names + and operation kinds are read off the packaged spec it runs + (:func:`_spine_request_roster`); its spine runs inside the dense build's + own graph, so it has no block to read. The check only refuses: it reads + the sidecar before the spine is pinned, the bound checkpoint is still + authenticated after it, and :func:`prepare_full_build` checks the roster + it then binds again, the authenticated sidecar's or the composed spine's. + """ + if args.input_h5 is not None: + sidecar = _read_spine_sidecar( + Path(args.input_sidecar or args.input_h5.with_suffix(".build.json")) + ) + require_geography_first_spine( + sidecar.get("stages"), sidecar.get("operations"), sidecar.get("geography") + ) + else: + stages, operations = _spine_request_roster() + require_geography_first_spine(stages, operations, from_sidecar=False) + + +def _spine_request_arguments(args: argparse.Namespace) -> list[str]: + """The spine request's command arguments, as the spine build parses them. + + ``--spine-h5`` is an output control of the checkpoint command: where the + request names none, one under ``--out`` is added. Preparation uses it only + for path configuration; the full graph writes its own H5. + """ + raw_arguments = json.loads(args.spine_request.read_text()) + if not isinstance(raw_arguments, list) or not all( + isinstance(x, str) for x in raw_arguments + ): + raise ValueError("The spine request must be a JSON array of command arguments.") + if "--spine-h5" not in raw_arguments and not any( + x.startswith("--spine-h5=") for x in raw_arguments + ): + raw_arguments += ["--spine-h5", str(args.out / "spine.h5")] + return raw_arguments + + +def _spine_rung_token(args: argparse.Namespace) -> str: + """The rung token a dense attempt's build id carries: its spine's. + + The id is minted before preparation reads the spine, so the rung is read + ahead, unauthenticated and for the label only: off the bound spine's build + sidecar (``--input-h5``) or the spine request's ``--sample-fraction``. + Where this early read cannot place the spine, the label is f100, and + preparation then reports the real error; it also refuses an attempt whose + label is not its bound spine's rung (microcosm#931). + """ + fraction: object = 1.0 + try: + if args.input_h5 is not None: + sidecar = _read_spine_sidecar( + Path(args.input_sidecar or args.input_h5.with_suffix(".build.json")) + ) + sampling = sidecar.get("sampling") or {} + if isinstance(sampling, Mapping): + fraction = float(sampling.get("fraction", 1.0)) + elif args.spine_request is not None: + from .spine_build import parse_uk_spine_args + + # The spine's own parser, quietly: a request it refuses exits + # through argparse, and preparation reports that refusal. + with contextlib.redirect_stderr(io.StringIO()): + fraction = float( + parse_uk_spine_args(_spine_request_arguments(args)).sample_fraction + ) + except (OSError, TypeError, ValueError, SystemExit): + fraction = 1.0 + return UK_SAMPLE_RUNG_TOKENS.get(fraction, UK_SAMPLE_RUNG_TOKENS[1.0]) + + +def _spine_sampling(fraction: float, seed: object) -> dict[str, object]: + """A spine's sampling as the dense build binds it: its rung, and the seed + it drew with (none at f100, where the root draws nothing). + + A fraction that is not a scale-ladder rung is refused here, before the + run records it, naming the rungs only (microcosm#931). + """ + fraction = float(fraction) + if fraction not in UK_SAMPLE_RUNG_TOKENS: + raise ValueError( + "The spine's sampling fraction is not a scale-ladder rung (one of " + + ", ".join(sorted(UK_SAMPLE_RUNG_TOKENS.values())) + + "): the dense line's rung is its spine's (microcosm#931)." + ) + sampled = fraction != 1.0 + return { + "fraction": fraction, + "seed": int(seed) if sampled and seed is not None else None, + "sampled": sampled, + "rung_token": UK_SAMPLE_RUNG_TOKENS[fraction], + } + + +def _bound_spine_binding(sidecar: Mapping[str, object]) -> dict[str, object]: + """What a bound spine's authenticated sidecar binds the dense build to. + + Its geographic clone count K (one where a sidecar predates the record, and + held to the frame's by the checkpoint authentication), its sampling (its + root's receipt, none at f100), its release posture and its geography block + with K beside it (microcosm#931). + """ + clones = int(sidecar.get("geographic_clone_count", 1)) + sampling = sidecar.get("sampling") or {} + return { + "geographic_clone_count": clones, + "sampling": _spine_sampling( + sampling.get("fraction", 1.0), sampling.get("seed") + ), + "release_posture": sidecar.get("release_posture"), + "geography": { + **dict(sidecar.get("geography") or {}), + "geographic_clone_count": clones, + }, + } + + +def _requested_spine_binding( + raw: argparse.Namespace, geography: Mapping[str, object] | None +) -> dict[str, object]: + """What a spine request binds the dense build to: the request's + ``--n-clones``, ``--sample-fraction`` and ``--sample-seed``, the release + posture the spine build's sidecar writer gives the same request, and the + ``geography`` block its preparation composed, through the helper the + sidecar writer calls (``country_adapter.uk_spine_geography_binding``), + with K beside it (microcosm#931). The in-graph spine writes no sidecar, + so its preparation's block stands in for one: a spine whose roster runs + no area draw composes none, and its binding carries K alone.""" + clones = int(raw.n_clones) + return { + "geographic_clone_count": clones, + "sampling": _spine_sampling(raw.sample_fraction, raw.sample_seed), + "release_posture": ( + "non_release_smoke" + if raw.smoke + else "release_candidate" + if raw.release_candidate + else "development" + ), + "geography": {**dict(geography or {}), "geographic_clone_count": clones}, + } + + +def _without_spine_option(arguments: list[str], flag: str) -> list[str]: + """A spine request's arguments without ``flag`` and its value.""" + kept: list[str] = [] + skip = False + for item in arguments: + if skip: + skip = False + elif item == flag: + skip = True + elif not item.startswith(f"{flag}="): + kept.append(item) + return kept + + +def _bind_spine_arguments( + args: argparse.Namespace, + binding: Mapping[str, object], + *, + requested: bool = False, +) -> None: + """Make the run's clone count and sampling the bound spine's (microcosm#931). + + The dense line takes K, the sampling rung and its seed from the spine it + binds, so the recorded run parameters, the identity digest, the size + checkpoint's identity and the Logbook rung are the spine's. An explicit + ``--n-clones`` is an assertion of the spine's K: one that differs is + refused, naming both K, and on a spine request (``requested``) pointing + at the request's own ``--n-clones``. ``--engine-blocks`` must be one or + the spine's K, and is refused otherwise, naming K. + """ + clones = int(binding["geographic_clone_count"]) + explicit = getattr(args, "_explicit_arguments", frozenset()) + if "n_clones" in explicit and args.n_clones != clones: + if requested: + raise ValueError( + f"--n-clones {args.n_clones} differs from the spine request's " + f"--n-clones K = {clones}: the dense line takes K from its " + "spine; set the clone count in the spine request (microcosm#931)." + ) + raise ValueError( + f"--n-clones {args.n_clones} differs from the bound spine's " + f"geographic clone count K = {clones}: the dense line takes K " + "from its spine (microcosm#931)." + ) + if args.engine_blocks not in {1, clones}: + raise ValueError( + "--engine-blocks is neither one nor the spine's geographic clone " + f"count K = {clones}: the engine resolves the pool in one block or " + "one block per clone (microcosm#931)." + ) + sampling = binding["sampling"] + args.n_clones = clones + args.sample_fraction = float(sampling["fraction"]) + if sampling["seed"] is not None: + args.sample_seed = int(sampling["seed"]) + + +def _require_release_candidate_spine( + binding: Mapping[str, object], *, clone_count: int +) -> None: + """Refuse a release candidate on a spine a release cannot build on. + + A dense release builds on a spine of the doctrine's ``clone_count`` + geographic clones, which the spine build made as a release candidate (its + ``release_posture``), whose areas are the release line's: its geography + binding passes + :func:`~microcosm.build.uk_runtime.country_adapter.uk_spine_geography_binding_failures` + (microcosm#931). A sampled spine is refused before this, where its rung is + read. A spine request's binding carries the block its preparation + composed, through the helper the sidecar writer calls, so both routes are + held to the same check; a spine that runs no area draw, a bound spine + built before its area stages among them, records no block, and a release + candidate on one is refused here, as recording no geography binding. The + refusal names every failure, with K, never a value. + """ + clones = int(binding["geographic_clone_count"]) + failures: list[str] = [] + if clones != clone_count: + failures.append( + f"its geographic clone count is K = {clones}, not the doctrine's " + f"K = {clone_count}" + ) + if binding.get("release_posture") != "release_candidate": + failures.append("it was not built as a release candidate") + geography = binding.get("geography") or {} + if set(geography) <= {"geographic_clone_count"}: + failures.append( + "it records no geography binding (a spine that runs no area draw " + "writes none)" + ) + else: + failures.extend( + f"its geography binding's {failure}" + for failure in uk_spine_geography_binding_failures( + geography, clone_count=clones + ) + ) + if failures: + raise ValueError( + "--release-candidate builds only on a release-candidate spine " + "(microcosm#931): " + "; ".join(failures) + "." + ) + + def prepare_full_build( args: argparse.Namespace, *, telemetry=None, attempt: dict | None = None ) -> PreparedUKFullBuild: @@ -1320,6 +1231,9 @@ def prepare_full_build( # The engine freeze, before any input is read (microcosm#1095). assert_installed_engine_is_pinned() + # The spine's posture, before any input is pinned or any graph composed: + # the dense line builds only on a spine that assigns areas (microcosm#931). + _require_geography_first_spine(args) posture = posture_of(args) release = load_uk_frs_release() pins = {"ladder": _pin(args.ladder, args.ladder_sha256)} @@ -1347,11 +1261,17 @@ def prepare_full_build( gate_report_path=gates_path, input_sha256=pins["dataset"]["sha256"], ) + # The authenticated sidecar is the one the build binds, so its roster + # is checked again: one changed since the early read is refused too. + require_geography_first_spine( + sidecar.get("stages"), sidecar.get("operations"), sidecar.get("geography") + ) spine = bound_spine_graph(frame) endpoint = "uk.full.spine_checkpoint" - weight_kind = frame.weights_for("household").kind.value time_period = str(frame.metadata["time_period"]) - source_fraction = float((sidecar.get("sampling") or {}).get("fraction", 1.0)) + # The checkpoint authentication held the sidecar's clone count to the + # frame's, so either is the spine's K. + spine_binding = _bound_spine_binding(sidecar) stages = tuple(sidecar["stages"]) _rules_engine() # the uk extra must be installed; the identity is provenance engine_identity = hashlib.sha256( @@ -1367,29 +1287,30 @@ def prepare_full_build( "spine_provenance", endpoint, "spine_provenance", SPINE_PROVENANCE_TYPE ) else: - raw_arguments = json.loads(args.spine_request.read_text()) - if not isinstance(raw_arguments, list) or not all( - isinstance(x, str) for x in raw_arguments - ): - raise ValueError( - "The spine request must be a JSON array of command arguments." - ) - # --spine-h5 is an output control of the checkpoint command. Preparation - # uses it only for path configuration; the full graph writes its own H5. - if "--spine-h5" not in raw_arguments and not any( - x.startswith("--spine-h5=") for x in raw_arguments - ): - raw_arguments += ["--spine-h5", str(args.out / "spine.h5")] + raw_arguments = _spine_request_arguments(args) + # A dry run over candidate clone counts plans each one as the spine + # request's own K (microcosm#931). + candidate = getattr(args, "_candidate_clone_count", None) + if candidate is not None: + raw_arguments = [ + *_without_spine_option(raw_arguments, "--n-clones"), + "--n-clones", + str(int(candidate)), + ] raw = parse_uk_spine_args(raw_arguments) if args.release_candidate: raw.release_candidate = True prepared = prepare_uk_spine_execution(raw) + # The composed spine is the one the build runs, so its own roster is + # checked again: the early check read the packaged spec's. + require_geography_first_spine( + *stage_roster(prepared.stages), from_sidecar=False + ) spine, kernels = prepared.graph, prepared.kernels sources.update(prepared.sources) endpoint = uk_spine_endpoint(spine).population - weight_kind = "importance" time_period = prepared.frs_release.time_period - source_fraction = raw.sample_fraction + spine_binding = _requested_spine_binding(raw, prepared.geography) stages = prepared.stage_names engine_identity = prepared.engine_identity inputs["dataset"] = { @@ -1399,13 +1320,21 @@ def prepare_full_build( "pin_verified": False, "spine_request": str(args.spine_request.resolve()), } - # The atomic-area supports are admitted after the spine so a spine - # refusal (sidecar, digest) is reported before a support one. - definition, geography_binding = _prepare_geography(args, pins, sources) - property_binding = _prepare_property_localisation(args, pins, sources) - brma_binding = _prepare_brma_localisation(args, pins, sources) - private_rent_binding = _prepare_private_rent_localisation(args, pins, sources) - rfdr_binding = _prepare_rfdr_localisation(args, pins, sources) + # The attempt's build id was minted with the spine's rung, read ahead of + # this; one that is not the bound spine's is refused, naming both rung + # tokens (microcosm#931). + labelled = getattr(args, "_build_id_rung", None) + rung = spine_binding["sampling"]["rung_token"] + if labelled is not None and labelled != rung: + raise ValueError( + f"The attempt's build id carries rung {labelled}, but the spine it " + f"binds is at rung {rung}: a dense build id takes its spine's rung " + "(microcosm#931)." + ) + # The spine's clone count, sampling and geography binding are the build's + # (microcosm#931): they enter the recorded run parameters, and so the + # identity digest, before anything records them. + _bind_spine_arguments(args, spine_binding, requested=args.input_h5 is None) stage( telemetry, "input_pinning", @@ -1423,20 +1352,10 @@ def prepare_full_build( target_families=("census_households/constituency",) if args.households_only else None, - n_clones=args.n_clones, - sample_fraction=args.sample_fraction, - source_sample_fraction=source_fraction, - sample_seed=args.sample_seed, + n_clones=int(spine_binding["geographic_clone_count"]), + source_sample_fraction=float(spine_binding["sampling"]["fraction"]), seed=args.seed, engine_blocks=args.engine_blocks, - constituency_vintage=args.expected_constituency_vintage, - source_lineage_modulus=args.source_lineage_modulus, - geography_assignment=args.geography_assignment, - source_vintage=release.vintage, - property_localisation=args.property_localisation, - brma_localisation=args.brma_localisation, - private_rent_localisation=args.private_rent_localisation, - rfdr_localisation=args.rfdr_localisation, calibration=UKGraphCalibrationConfig( epochs=args.epochs, learning_rate=args.learning_rate, @@ -1464,10 +1383,19 @@ def prepare_full_build( state.identity_digest = candidate_identity_digest( pins=pins, args=args, source_year=config.source_year ) + # The spine's rung (its sidecar's sampling, or the request's + # --sample-fraction): the dense line samples nothing more (microcosm#931). if args.release_candidate and config.effective_sample_fraction != 1.0: raise ValueError( "Sampled builds cannot request release-candidate certification." ) + # The release candidate's K pin, posture and geography binding: the + # doctrine's K, a release-candidate spine and the release line's areas + # (microcosm#931). + if args.release_candidate: + _require_release_candidate_spine( + spine_binding, clone_count=int(posture.clone_count) + ) feed = load_uk_chronicle_feed() for supplied, committed in ( (args.ledger_facts_sha256, feed.facts_sha256), @@ -1509,11 +1437,9 @@ def prepare_full_build( config, spine=spine, spine_population=endpoint, - spine_weight_kind=weight_kind, optional_target_sources=tuple(optional), review_date=args.review_date.isoformat(), checkpoint_identity=checkpoint_identity, - atomic_geography_definition=definition, ) if args.resume_size_checkpoint: from .size_checkpoint import ( @@ -1567,11 +1493,13 @@ def prepare_full_build( }, "paired_ladder_sha256": pins["ladder"]["sha256"], }, - "geography": geography_binding, - "property_localisation": property_binding, - "brma_localisation": brma_binding, - "private_rent_localisation": private_rent_binding, - "rfdr_localisation": rfdr_binding, + # The spine the build binds and its geography binding (microcosm#931): + # the rowwise manifest reads its sampling and areas from here. + "spine": { + key: spine_binding[key] + for key in ("geographic_clone_count", "sampling", "release_posture") + }, + "geography": dict(spine_binding["geography"]), } graph = add_uk_export_preparation( graph, @@ -1841,10 +1769,6 @@ def _argument_sources(args: argparse.Namespace) -> dict[str, Path]: ("native_scorecard", args.native_scorecard), ("matched_size_scorecard", args.matched_size_scorecard), ("resume_size_checkpoint", args.resume_size_checkpoint), - *( - (f"atomic_support_{label}", getattr(args, f"atomic_support_{label}")) - for label in _SUPPORT_ARGUMENTS.values() - ), ) if path is not None } @@ -2216,6 +2140,8 @@ def _close_attempt( disposition=classified.disposition, predecessor=attempt["predecessor"], spool_dir=output / "logbook-spool", + # The spine's rung: preparation set the run's fraction to the spine's + # (microcosm#931). rung=UK_SAMPLE_RUNG_TOKENS[args.sample_fraction], ) print(f"Wrote Logbook row: {spool_path}", file=sys.stderr) @@ -2945,7 +2871,20 @@ def refuse_occupied_national_output(args: argparse.Namespace) -> None: def prepare_national_build( args: argparse.Namespace, *, telemetry=None, attempt: dict | None = None ) -> PreparedUKNationalBuild: - """Bind the national request to its graph: the checkpoint, the doctrine, the register.""" + """Bind the national request to its graph: the checkpoint, the doctrine, the register. + + The national line builds only once the degenerate-surface register's + area-basis exclusion is signed for it and in force on ``--review-date``, + the build's own review date (the release cut reads the register again on + the day it certifies), and only on a geography-first spine of one + geographic clone (microcosm#931). Both are + refused before the input is pinned or any engine work: the exclusion + before any input is read, and the spine by its build sidecar, read ahead + of the checkpoint's authentication and checked again on the sidecar the + authentication returns. The register decides: an entry signed for the + national line, in force on the review date, lifts the first refusal with + no code change. + """ from .hmrc_uprating import assert_installed_engine_is_pinned from .spine_build import _rules_engine, _rules_engine_provenance @@ -2953,20 +2892,28 @@ def prepare_national_build( assert_installed_engine_is_pinned() posture = posture_of(args) national_role.require_bound_input(args) + require_national_exclusion_signed(args.review_date) release = load_uk_frs_release() input_h5 = national_role.require_file(args.input_h5, label="--input-h5") + sidecar_path = args.input_sidecar or input_h5.with_suffix(".build.json") + gates_path = args.input_spine_gates or input_h5.with_suffix(".spine_gates.json") + # The spine's clone count and roster, read off its build sidecar before + # the input is pinned (_read_spine_sidecar refuses an absent, unreadable + # or non-object sidecar as the authentication does). + require_national_spine(_read_spine_sidecar(Path(sidecar_path))) pins = {"dataset": _pin(input_h5, args.input_sha256)} inputs = {"dataset": _input_record(input_h5, pins["dataset"], pinned=True)} frame, _ = load_uk_national_frame(input_h5) require_uk_producer_isa_income_contract(frame, label="--input-h5") - sidecar_path = args.input_sidecar or input_h5.with_suffix(".build.json") - gates_path = args.input_spine_gates or input_h5.with_suffix(".spine_gates.json") - load_bound_spine_checkpoint( + sidecar = load_bound_spine_checkpoint( sidecar_path, frame, gate_report_path=gates_path, input_sha256=pins["dataset"]["sha256"], ) + # The authenticated sidecar is the one the build binds, so it is checked + # again: one changed since the early read is refused too. + require_national_spine(sidecar) spine = bound_spine_graph(frame) _rules_engine() # the uk extra must be installed; the identity is provenance engine_identity = hashlib.sha256( @@ -3452,13 +3399,16 @@ def _execute_national_build( # 7. The H5, then the graph's readback of it. stage(telemetry, "candidate_h5_creation", "started") # The release boundary (microcosm#1063 c9): the reviewed export exclusions - # leave the candidate here, and the graph readback expects the same. The + # leave the candidate here, with the national line's geographic clone + # indices (microcosm#931), and the graph readback expects the same. The # weights are written at the dataset's year, so policyengine-uk's own # projection returns the solved weights at the solve year (microcosm#796 # A19); the solve, its gates and its evidence keep the solved weights. solve_year = int(registry_document["calibration_year"]) write_uk_national_frame( - uk_dataset_year_frame(uk_release_export_frame(frame), solve_year=solve_year), + uk_dataset_year_frame( + uk_release_export_frame(frame, line="national"), solve_year=solve_year + ), paths["dataset"], ) if state is not None: @@ -3908,24 +3858,33 @@ def _record_failure( predecessor=attempt["predecessor"], base_dir=args.out.resolve(), spool_dir=args.out.resolve() / "logbook-spool", - rung=UK_SAMPLE_RUNG_TOKENS[args.sample_fraction], + # The attempt's rung, its build id's: the spine's, read ahead of + # preparation and held there to the bound spine's, so a refusal + # before the spine is bound records it too (microcosm#931). + rung=getattr(args, "_build_id_rung", None) + or UK_SAMPLE_RUNG_TOKENS[args.sample_fraction], pipeline=pipeline, disposition=disposition, ) def _dry_run(args: argparse.Namespace) -> int: - """Plan without solving or writing; loop over candidate clone counts.""" + """Plan without solving or writing. + + With ``--candidate-clone-counts`` (a spine request only), the spine + request is planned once per candidate, each as its own ``--n-clones``: + the dense line's K is its spine's (microcosm#931). + """ counts = args.candidate_clone_counts if counts is None: return execute_full_build(prepare_full_build(args), args) inventories = [] for count in counts: planned = argparse.Namespace(**vars(args)) - planned.n_clones = int(count) + planned._candidate_clone_count = int(count) prepared = prepare_full_build(planned) inventories.append( - {"n_clones": int(count), **prepared.full.operation_inventory()} + {"n_clones": int(planned.n_clones), **prepared.full.operation_inventory()} ) print(json.dumps(inventories, indent=2)) return 0 @@ -3952,10 +3911,11 @@ def _main(argv: list[str] | None = None) -> int: raise ValueError("--candidate-clone-counts is valid only with --dry-run.") started_at = time.perf_counter() started_ts = datetime.now(UTC) + # The id carries the spine's rung, read ahead of preparation, which + # refuses an attempt whose label is not its bound spine's (microcosm#931). + args._build_id_rung = _spine_rung_token(args) build_id = new_candidate_build_id( - seed=args.seed, - timestamp=started_ts, - rung=UK_SAMPLE_RUNG_TOKENS[args.sample_fraction], + seed=args.seed, timestamp=started_ts, rung=args._build_id_rung ) return _run_with_telemetry( args, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_certification.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_certification.py index c336198395..d1143d6c13 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_certification.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_certification.py @@ -401,9 +401,27 @@ class UKFullCertificationKernel(KernelBase): def implementation_hash(self): from microcosm.data import contract - from . import full_gates, graph_terminal + from .. import artifact_files, gate_battery, gates, trace + from . import full_gates, gate_scopes, graph_evidence, graph_terminal - return source_hash(sys.modules[__name__], contract, full_gates, graph_terminal) + # The readiness also computes through the comparison files' records + # (artifact_files, trace's file digest), the phase reports it decodes + # (gate_battery, gates), the spine gate manifest and its scope + # (graph_evidence, gate_scopes) and its report's canonical encoding + # (microcosm#931). + return source_hash( + sys.modules[__name__], + contract, + full_gates, + graph_terminal, + artifact_files, + trace, + gate_battery, + gates, + graph_evidence, + gate_scopes, + canonical_json, + ) def run(self, context: KernelContext) -> KernelResult: sources = {} diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_gates.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_gates.py index 6d3d201960..9d58ec3e65 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_gates.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_gates.py @@ -319,22 +319,21 @@ def classify_full_gate_outcomes( The combined driver exports a diagnostic candidate after failures of its five local fit/support/weight checks, but stops on a non-passing geography - ladder or local BRMA domain check. Below f100 those five checks are - recorded without enforcement. + ladder check. Below f100 those five checks are recorded without + enforcement. The local BRMA domain recheck retired when the dense line + took its areas from the spine, whose transferred gate checks the column + (``uk_brma_enum_domain``, microcosm#931). All newly included national/source checks keep the shared battery's own BLOCKS_ARTIFACT policy, including its declared missing-evidence rules. """ from microcosm.build.gate_battery import GatePhaseReport, GateStatus - from microcosm.build.uk_runtime.calibration_run import UK_LOCAL_GATE_SCOPE + from microcosm.build.uk_runtime.gate_scopes import UK_LOCAL_GATE_SCOPE if not isinstance(report, GatePhaseReport): raise TypeError("Classify a validated GatePhaseReport, not unbound JSON.") if not 0.0 < sample_fraction <= 1.0: raise ValueError("sample_fraction must be in (0, 1].") - structural_local = { - "uk_local_geography_ladder_post_calibration", - "uk_local_brma_enum_domain", - } + structural_local = {"uk_local_geography_ladder_post_calibration"} local_export_exception = set(UK_LOCAL_GATE_SCOPE) - structural_local blocking = { outcome.entry.id diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_measure.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_measure.py index 7497b27ca6..77f39b4b70 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_measure.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_measure.py @@ -7,6 +7,7 @@ from __future__ import annotations import gc +import math from collections.abc import Mapping from pathlib import Path from typing import Any @@ -25,28 +26,62 @@ ladder_clone_index_column, materialize_uk_ledger_targets, ) +from microcosm.build.uk_runtime.geographic_clones import geographic_clone_count from microcosm.build.uk_runtime.measure_simulation import UKMeasureResolver from microcosm.build.uk_runtime.national_frame import uk_time_period from microcosm.calibrate.matrix import CalibrationProblem, build_constraint_matrix from microcosm.frame import Frame, MassChangeRecord +from microcosm.frame.adapters.engine_population import ( + MEASURE_PASS, + PoolShare, + PoolShareError, + factor_spread, + pool_shares, +) +from microcosm.frame.adapters.policyengine_uk_population import UK_ENGINE_POPULATION + +#: The engine variables of the national rows that read policyengine-uk's share +#: formulas (``ons.land.corporate_land_value`` and ``ons.land.land_value``), +#: matched against the run's measure inputs. Both rows have been held out of +#: the solve since the #931 move (calibration_measure_exclusions.json), so a +#: run on the packaged register resolves neither variable. +UK_BLOCK_SENSITIVE_MEASURE_COLUMNS = ("corporate_land_value", "land_value") -UK_BLOCK_SENSITIVE_MEASURE_COLUMNS = ( - "ons/corporate_land_value", - "ons/land_value", - "slc/student_loan_repayment/england", +#: What ``engine_population_representation.exact`` certifies for per-clone +#: engine blocks (María's ruling of 2026-11-06, microcosm#931). A receipt +#: without it predates the correction, when exact meant identical copies. +UK_ENGINE_BLOCK_EXACT_BASIS = "pool_share_injection_with_population_guard" +UK_ENGINE_BLOCK_EXACT_SCOPE = ( + "per-clone engine blocks stand for the pool: each block's household weights " + "are scaled to the pool's mass, and in every simulation the block builds " + "shareholding, corporate_land_value and consumption_shareholding are set at " + "each period it computes to the block's own value times its pool-mass " + "factor times its allocation-key total over the pool's, the engine carrying " + "each key and the weights by one factor per year (checked per block); " + "months_since_last_birthday and attends_private_school are stored inputs, " + "every other policyengine-uk whole-population formula that ran was zero " + "under the parameters checked, and nothing the block computed fell in the " + "Universal Credit rebalancing years (2026 on), where policyengine-uk draws " + "new-claimant status by benefit-unit position; the result equals one " + "engine over the pool to float32 rounding, not bitwise, at the measure " + "step's starting weights, and the annual-income simulation carries other " + "spells by the engine's own growth parameters, the same in every block" ) +#: How far the blocks' parameter-driven factors (the engine's growth of each +#: key and of the weights, and the annual-income factor) may spread, relative +#: to their mean: float32 rounding, far inside it. +_BLOCK_FACTOR_AGREEMENT = 1e-6 #: policyengine-uk's weight-share formulas (a national total or a share #: allocated by ``x * w / sum(x * w)``) and the frame columns their allocation #: key ``x`` reads (microcosm#1115 review). Scaling a block's engine weights by -#: pool mass / block mass makes such a formula exact only when every block -#: carries the same ``sum(x * w)``; identical (source household, weight) -#: multisets guarantee that only while ``x`` is copied into every clone, which -#: holds for these keys (spine wealth and consumption inputs) and would not for -#: anything drawn per clone. The representation check therefore records each -#: block's ``sum(x * w)`` for every key here and requires them to agree; a -#: formula of this shape that reads a per-clone input must be added here so -#: the check sees it, never assumed exact. +#: pool mass / block mass leaves such a formula exact only while every block +#: carries the same ``sum(x * w)``, which clones that differ do not. The +#: representation therefore records each block's ``sum(x * w)`` for every key +#: here, and each block's resolver sets the formula to its own value times +#: ``f_b * S_b / S`` (microcosm#931). The columns are the adds the UK +#: declaration names for each key (``UK_ENGINE_POPULATION.key_adds``), which a +#: test holds them to, and which the engine's own adds are held to. UK_WEIGHT_SHARE_FORMULA_INPUTS: Mapping[str, tuple[str, ...]] = { # corporate_land_value and shareholding allocate by corporate_sector_wealth. "corporate_land_value": ("corporate_wealth", "private_pension_wealth"), @@ -98,12 +133,52 @@ def _without_scratch_paths(value, scratch_dir: Path): return value -def _engine_block_frames(frame, blocks: int) -> list[tuple[int | None, Frame]]: - """The engine blocks: the whole clone, or one scratch frame per clone index.""" +def _require_engine_clone_count(frame, clones: int) -> None: + """Refuse a frame that does not hold the ``clones`` geographic clones its + node binds (microcosm#931). + + The engine resolves the pool its node binds, the spine's version as it + stands with all K of its geographic clones; a frame of another clone + count (a part of the pool that misses a clone, say) stands for another + population, at one block as at K; the check reads the clone count alone, + so a part that keeps every clone index is not refused here. Reads only + the household clone index, through + :func:`~.geographic_clones.geographic_clone_count`, whose own refusals + name the column; the refusal here names the two clone counts, never a + row count, an id or a value. + """ + + held = geographic_clone_count(frame.table("household"), stage="Engine resolution") + if held != clones: + raise ValueError( + "Engine resolution binds the geographic clone count " + f"K = {clones}, but its frame's is K = {held}: a frame of another " + "clone count is not the pool its node binds (microcosm#931)." + ) + + +def _engine_block_frames( + frame, blocks: int, *, clones: int | None = None +) -> list[tuple[int | None, Frame]]: + """The engine blocks: the whole pool, or one scratch frame per clone index. + + With ``clones``, the spine's geographic clone count K that the calling + node binds (microcosm#931), the frame must hold exactly K geographic + clones at every block count, one included, and the blocks must be one or + K; without it (callers outside the build graph) any frame resolves in one + block, and K blocks need K realized clone indices. + """ household = frame.table("household") if blocks < 1: raise ValueError("engine resolution blocks must be positive.") + if clones is not None: + _require_engine_clone_count(frame, clones) + if blocks not in {1, clones}: + raise ValueError( + "Engine resolution blocks must be one or one per geographic " + f"clone, K = {clones} (microcosm#931)." + ) if blocks == 1: return [(None, frame)] clone_column = ladder_clone_index_column("household") @@ -171,17 +246,19 @@ def _engine_block_frames(frame, blocks: int) -> list[tuple[int | None, Frame]]: def _engine_population_representation(frame, block_frames) -> dict[str, Any]: - """How each engine block stands for the pool, and whether exactly. + """How each engine block stands for the pool, before the blocks run. A weight-share formula (a national total allocated by ``x*w / sum(x*w)``, policyengine-uk's corporate land value among them) sees only its block's denominator, so each block would reproduce the whole national total: the ``K`` times artefact behind the #736 erratum. Every block's engine weights - are therefore scaled by ``pool mass / block mass``. For blocks that are - identical copies of one another (the clone expansion's contract) every - weighted sum then equals the pool's and the formula is exact; the checks - record whether the blocks are such copies. A pool that is not leaves the - scaling approximate, and the measures receipt keeps its caveat. + are therefore scaled by ``pool mass / block mass``, and each block's + resolver corrects the share formulas to the pool's denominator from the + allocation-key totals recorded here (:func:`_block_pool_shares`). Whether + that made the blocks exact is known only once they ran: ``exact`` is + provisional until :func:`_finalize_representation`. The checks of whether + the blocks are identical copies stay as diagnostics; a block that lacks a + key column gets no correction, and the representation stays inexact. """ if len(block_frames) == 1 and block_frames[0][0] is None: @@ -209,11 +286,14 @@ def _engine_population_representation(frame, block_frames) -> dict[str, Any]: "represent the pool." ) factors[str(clone_index)] = pool_mass / block_mass - # Blocks are compared as multisets of (source household, weight): the - # clone expansion copies every spine household into every clone, so - # identical copies have identical multisets whatever geography each - # clone drew. Without a source identity the comparison is weights - # only, which cannot establish copies, so the result is never exact. + # A diagnostic (microcosm#931): blocks are compared as multisets of + # (source household, weight). The clone expansion copies every spine + # household into every clone, so identical copies have identical + # multisets whatever geography each clone drew. Without a source + # identity the comparison is weights only, which cannot establish + # copies, so the identical-copies diagnostic never holds. Whether the + # blocks stand for the pool is decided once they ran + # (_finalize_representation). key = next((c for c in _SOURCE_IDENTITY_COLUMNS if c in household), None) if key is None: identity_present = False @@ -275,29 +355,25 @@ def _engine_population_representation(frame, block_frames) -> dict[str, Any]: } return { "mode": "block_weights_scaled_to_pool", - "exact": bool( - identity_present - and identical - and equal_households - and equal_persons - and inputs_match - ), + # Provisional: _finalize_representation decides it from what every + # block's resolver recorded. + "exact": False, + "exact_basis": UK_ENGINE_BLOCK_EXACT_BASIS, "blocks": blocks, - # What ``exact`` certifies, and what it does not: on a slice of the - # K=25 pool, per-clone and single-block resolution matched on every - # household column and constituency metric but differed on a small - # share of person records (state pension type, pension credit, SLC - # repayment), so engine draws that depend on a record's position in - # the simulation are outside it (microcosm#1124). A dense release - # states this in its engine_measurement limitation; re-measuring the - # released file on one engine is the incumbent-surface evaluation's, - # an opt-in diagnostic since microcosm#1301. - "exact_scope": ( - "weight-share formulas see the pool's denominator in every block: " - "identical clone copies, pool-scaled block weights and equal " - "allocation-key sums; engine draws that depend on a record's " - "position in the simulation are not covered" - ), + # What ``exact`` certifies, and what it does not. A slice of the K=25 + # pool once differed between per-clone and single-block resolution on + # a few person records (microcosm#1124), through draws a record's + # position or a block's own identifiers decided. The identifiers are + # the release's since microcosm#1194; the one draw policyengine-uk + # still makes by position, which Universal Credit claimants count as + # new claimants in the rebalancing years from 2026 + # (scenarios/uc_reform.py, REBALANCING_YEARS), is outside it, and the + # guard refuses any block work in those years rather than certify it. + # A dense release states this in its engine_measurement limitation; + # re-measuring the released file on one engine is the + # incumbent-surface evaluation's, an opt-in diagnostic since + # microcosm#1301. + "exact_scope": UK_ENGINE_BLOCK_EXACT_SCOPE, "factor_by_block": factors, "checks": { "identity_key": identity_key, @@ -314,6 +390,227 @@ def _engine_population_representation(frame, block_frames) -> dict[str, Any]: } +def _block_pool_shares( + representation: Mapping[str, Any], +) -> dict[str, dict[str, PoolShare]] | None: + """Each block's :class:`PoolShare` for every share formula, or None. + + From the representation's own frame totals: ``S_b``, each block's + ``sum(x * w)`` at unscaled weights, the pool's ``S`` as their sum, and the + block's pool-mass factor ``f_b``, so ``c_b = f_b * S_b / S``. None for a + single block, and when a block lacks a key column or a total is not + finite: no block is then corrected, and the representation stays inexact. + """ + + if representation.get("mode") != "block_weights_scaled_to_pool": + return None + factors = representation["factor_by_block"] + inputs = representation["checks"]["weight_share_inputs"] + formulas = dict(UK_ENGINE_POPULATION.corrected) + if set(inputs) != set(formulas): + return None + for formula in formulas: + entry = inputs[formula] + if entry["missing_columns"] or set(entry["sum_by_block"]) != set(factors): + return None + try: + return pool_shares( + { + block: { + formula: float(inputs[formula]["sum_by_block"][block]) + for formula in formulas + } + for block in factors + }, + {block: float(factor) for block, factor in factors.items()}, + key_variables=formulas, + ) + except PoolShareError: + return None + + +def _finite_or_none(value: float) -> float | None: + return float(value) if math.isfinite(value) else None + + +def _finalize_representation( + representation: Mapping[str, Any], + resolver_receipts: list[Mapping[str, Any]], + *, + shares_built: bool, +) -> dict[str, Any]: + """Whether the per-clone engine blocks stood for the pool, once they ran. + + Read from each block's resolver receipt (``engine_pool_shares`` and + ``engine_population_reach``). ``exact`` holds only if every block's + resolver validated the declaration, passed checks (a) and (b), applied + the correction in every simulation it built, refused nothing, and still + held the block's keys and weights in each at the end; the blocks computed + at the same periods; their growth factors and annual-income factors + agree within ``_BLOCK_FACTOR_AGREEMENT``; and every block's verdict under + the measure pass is clean, so nothing reached a refused rank or median, + nothing a stored input stands for was derived, and nothing ran in the + rebalancing years. A single block is returned as it is. + """ + + finalized = dict(representation) + if representation.get("mode") != "block_weights_scaled_to_pool": + return finalized + blocks = [str(block) for block in representation["factor_by_block"]] + receipts = list(resolver_receipts) + by_block = dict(zip(blocks, receipts, strict=False)) + complete = shares_built and len(receipts) == len(blocks) + shares = { + block: receipt.get("engine_pool_shares") for block, receipt in by_block.items() + } + shares = { + block: entry for block, entry in shares.items() if isinstance(entry, Mapping) + } + complete = complete and len(shares) == len(blocks) + + def every(field: str) -> bool: + return complete and all(entry.get(field) is True for entry in shares.values()) + + periods = {tuple(entry.get("periods") or ()) for entry in shares.values()} + growth = { + block: dict(entry.get("growth_factors") or {}) + for block, entry in shares.items() + } + growth_periods = sorted( + {period for factors in growth.values() for period in factors} + ) + growth_complete = all( + set(factors) == set(growth_periods) for factors in growth.values() + ) + growth_spread = 0.0 + for period in growth_periods: + spread = factor_spread( + { + block: factors[period] + for block, factors in growth.items() + if period in factors + } + ) + growth_spread = max(growth_spread, *spread.values(), 0.0) + growth_agree = growth_complete and growth_spread <= _BLOCK_FACTOR_AGREEMENT + annual = { + block: dict( + (receipt.get("annual_income_measures") or {}).get( + "other_spells_engine_factors" + ) + or {} + ) + for block, receipt in by_block.items() + } + annual_spread = max(factor_spread(annual).values(), default=0.0) + annual_agree = annual_spread <= _BLOCK_FACTOR_AGREEMENT + + verdicts = { + block: receipt.get("engine_population_reach") + for block, receipt in by_block.items() + } + verdicts = { + block: verdict + for block, verdict in verdicts.items() + if isinstance(verdict, Mapping) + } + clean = ( + complete + and len(verdicts) == len(blocks) + and all( + verdict.get("clean") is True and verdict.get("pass") == MEASURE_PASS.name + for verdict in verdicts.values() + ) + ) + digests = {verdict.get("declaration_sha256") for verdict in verdicts.values()} + engines = { + tuple(sorted(dict(verdict.get("engine") or {}).items())) + for verdict in verdicts.values() + } + classes: dict[str, set[str]] = {} + refused: set[str] = set() + window_violations: dict[tuple[str, str, str], dict[str, Any]] = {} + for verdict in verdicts.values(): + for name, members in dict(verdict.get("classes") or {}).items(): + classes.setdefault(str(name), set()).update(map(str, members)) + refused.update(map(str, verdict.get("refused") or ())) + for violation in dict(verdict.get("window") or {}).get("violations") or (): + violation = dict(violation) + key = tuple( + str(violation.get(field)) for field in ("kind", "name", "period") + ) + window_violations[key] = violation + refusals = { + block: list(entry["refusals"]) + for block, entry in shares.items() + if entry.get("refusals") + } + exact = bool( + complete + and every("declaration_valid") + and every("applied") + and every("key_sums_verified") + and every("uniform_growth_verified") + and every("siblings_match") + and not refusals + and len(periods) == 1 + and growth_agree + and annual_agree + and clean + and len(digests) == 1 + and not window_violations + ) + finalized["exact"] = exact + finalized["share_correction"] = { + "formulas": dict(UK_ENGINE_POPULATION.corrected), + "keys_complete": bool(shares_built), + "periods": sorted({period for entry in periods for period in entry}), + "corrections_by_block": { + block: dict(entry.get("corrections") or {}) + for block, entry in shares.items() + }, + "applied_in_every_block": every("applied"), + "key_sums_verified": every("key_sums_verified"), + "largest_key_sum_gap": max( + ( + float(entry["largest_key_sum_gap"]) + for entry in shares.values() + if entry.get("largest_key_sum_gap") is not None + ), + default=None, + ), + "uniform_growth_verified": every("uniform_growth_verified"), + "growth_factors_by_block": growth, + "growth_factors_agree_across_blocks": growth_agree, + "max_growth_factor_spread": _finite_or_none(growth_spread), + "siblings_match": every("siblings_match"), + "refusals_by_block": refusals, + } + finalized["population_reach"] = { + "pass": MEASURE_PASS.name, + "declaration_valid": every("declaration_valid"), + "declaration_sha256": next(iter(digests)) if len(digests) == 1 else None, + "engine": dict(next(iter(engines))) if len(engines) == 1 else None, + "clean_in_every_block": clean, + "classes": {name: sorted(members) for name, members in sorted(classes.items())}, + "refused": sorted(refused), + "window": { + "first_year": UK_ENGINE_POPULATION.window.first_year, + "violations": [window_violations[key] for key in sorted(window_violations)], + }, + "violations_by_block": { + block: list(verdict.get("violations") or ()) + for block, verdict in verdicts.items() + if verdict.get("clean") is not True + }, + } + finalized["annual_income_projection_factors"] = { + "agree_across_blocks": annual_agree, + "max_rel_spread": _finite_or_none(annual_spread), + } + return finalized + + def _run_engine_blocks( block_frames, national_registry, @@ -324,6 +621,7 @@ def _run_engine_blocks( local_grains: tuple[str, ...], on_block, engine_weight_scales: Mapping[str, float] | None = None, + engine_pool_shares: Mapping[str, Mapping[str, PoolShare]] | None = None, identifier_universe=None, ) -> tuple[ dict[str, list[pd.DataFrame]], @@ -352,6 +650,12 @@ def _run_engine_blocks( resolver_kwargs["engine_weight_scale"] = float( engine_weight_scales[str(clone_index)] ) + block_shares = None + if engine_pool_shares is not None and clone_index is not None: + # The block's share formulas take the pool's denominator in every + # simulation its resolver builds (microcosm#931). + block_shares = engine_pool_shares[str(clone_index)] + resolver_kwargs["engine_pool_shares"] = block_shares if identifier_universe is not None and clone_index is not None: # A block's rows take the identifiers they have in the release, # which numbers the whole frame (microcosm#1194). @@ -385,6 +689,12 @@ def _run_engine_blocks( input_period=int(uk_time_period(block_frame)), ) ) + if block_shares is not None: + # Everything the block computes is computed: judge it while its + # engine is alive, so the receipt below carries the verdict. + close = getattr(resolver, "close_engine_block", None) + if callable(close): + close() resolver_receipts.append( _without_scratch_paths(dict(resolver.receipt()), scratch_dir) ) @@ -479,37 +789,45 @@ def _measures_receipt( receipt["engine_population_representation"] = dict(representation) exact = bool(representation and representation.get("exact")) receipt["deviation"] = "per_clone_block_engine_resolution" + # The engine variables the measures read, matched by name: the + # measure inputs are keyed (entity, engine variable). + measured = {variable for _, variable in national_input_keys} present = sorted( column for column in UK_BLOCK_SENSITIVE_MEASURE_COLUMNS - if column in {variable for _, variable in national_input_keys} + if column in measured ) receipt["block_sensitivity"] = { "known_population_normalised_measures": list( UK_BLOCK_SENSITIVE_MEASURE_COLUMNS ), "present_in_this_run": present, + "corrected_formulas": dict(UK_ENGINE_POPULATION.corrected), "weight_share_formulas": { formula: list(columns) for formula, columns in UK_WEIGHT_SHARE_FORMULA_INPUTS.items() }, "mitigation": ( "each block's engine household weights are scaled by pool mass / " - "block mass, so weight-share formulas see the pool's denominator; " - "exact only while the blocks are identical copies and every " - "allocation key named in weight_share_formulas carries the same " - "sum(x * w) in every block" + "block mass, and every simulation the block builds sets the " + "corrected formulas, at each period it computes, to the block's " + "own value times its pool-mass factor times its allocation-key " + "total over the pool's, so they see the pool's denominator; " + "exact only when every block applied the correction with its " + "checks holding and every whole-population formula it ran is one " + "the declaration allows (engine_population_representation)" ), "exact": exact, "caveat": ( None if exact else ( - "the engine blocks are not identical copies of one another, so " - "the scaling is approximate and per-block engine resolution " - "may mis-measure population-normalised formulas; rows on these " - "measures are not evidence for adjudication from this run. " - "Resolve in a single block before ruling on them." + "the engine blocks are not certified to stand for the pool " + "(see engine_population_representation), so per-block engine " + "resolution may mis-measure population-normalised formulas; " + "rows on these measures are not evidence for adjudication " + "from this run. Resolve in a single block before ruling on " + "them." ) ), } @@ -533,6 +851,7 @@ def resolve_uk_full_measures( band_edge_registry=None, resolver_factory=UKMeasureResolver, blocks: int = 1, + clones: int | None = None, local_grains: tuple[str, ...] = ("constituency", "la"), ) -> tuple[Any, Any, UKRowwiseNationalRows, dict[str, pd.DataFrame], dict[str, Any]]: """Resolve national inputs and local metrics on the cloned frame. @@ -541,12 +860,16 @@ def resolve_uk_full_measures( reviewed escape hatch ``blocks=K`` resolves each clone index separately, then rejoins every entity-level prepared column by its stable entity id so the full-frame target materialization and single solve retain frame order. - The dense role's measures node uses :func:`resolve_uk_full_national_problem` - instead, which never materializes the whole pool. + ``clones`` is the geographic clone count K the caller binds; with it the + frame must hold K clones at every block count (:func:`_engine_block_frames`, + microcosm#931). The national line passes one. The dense role's measures + node uses :func:`resolve_uk_full_national_problem` instead, which never + materializes the whole pool. """ - block_frames = _engine_block_frames(frame, blocks) + block_frames = _engine_block_frames(frame, blocks, clones=clones) representation = _engine_population_representation(frame, block_frames) + block_shares = _block_pool_shares(representation) measure_parts: dict[tuple[str, str], list[pd.Series]] = {} def collect(block_frame, block_inputs): @@ -567,10 +890,14 @@ def collect(block_frame, block_inputs): scratch_dir=scratch_dir, resolver_factory=resolver_factory, engine_weight_scales=representation.get("factor_by_block"), + engine_pool_shares=block_shares, identifier_universe=frame, local_grains=local_grains, on_block=collect, ) + representation = _finalize_representation( + representation, resolver_receipts, shares_built=block_shares is not None + ) measure_inputs: dict[tuple[str, str], np.ndarray] = {} for (entity, variable), parts in measure_parts.items(): @@ -743,6 +1070,7 @@ def resolve_uk_full_national_problem( band_edge_registry=None, resolver_factory=UKMeasureResolver, blocks: int = 1, + clones: int | None = None, local_grains: tuple[str, ...] = ("constituency", "la"), ) -> tuple[ CalibrationProblem | None, @@ -763,10 +1091,14 @@ def resolve_uk_full_national_problem( stitched into the pool's household order. Materialization is row-wise (band edges come from the compiled register), so the stitched problem is the pool's problem. ``None`` when the registry selects no national target. + The measures node passes ``clones``, the spine's K, so the frame must hold + K geographic clones at every block count (:func:`_engine_block_frames`, + microcosm#931). """ - block_frames = _engine_block_frames(frame, blocks) + block_frames = _engine_block_frames(frame, blocks, clones=clones) representation = _engine_population_representation(frame, block_frames) + block_shares = _block_pool_shares(representation) targets = national_registry.to_target_set() band_edges = national_registry if band_edge_registry is None else band_edge_registry block_problems: list[tuple[CalibrationProblem, np.ndarray]] = [] @@ -795,11 +1127,15 @@ def materialize(block_frame, block_inputs): scratch_dir=scratch_dir, resolver_factory=resolver_factory, engine_weight_scales=representation.get("factor_by_block"), + engine_pool_shares=block_shares, identifier_universe=frame, local_grains=local_grains, on_block=materialize, ) ) + representation = _finalize_representation( + representation, resolver_receipts, shares_built=block_shares is not None + ) if any(report != reports[0] for report in reports[1:]): raise RuntimeError( "per-clone national target materialization differs across blocks." diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/gate_phase_codes.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/gate_phase_codes.py new file mode 100644 index 0000000000..0e3f2dc77a --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/gate_phase_codes.py @@ -0,0 +1,401 @@ +"""Each UK gate battery phase's own code, and the check that refuses a stale one. + +A gate battery's phases run one kernel's code, and that kernel's hash is the +code they share. What only one phase's gates compute through is that phase's +own code, a source hash its node carries as its ``_code`` parameter, +so no other phase's own key carries it (microcosm#931 ruling 6): the spine +gates' assembled and transferred phases (:data:`UK_SPINE_GATE_PHASE_CODE`) +and the full build's preflight and calibrated gates +(:data:`UK_FULL_GATE_PHASE_CODE`). + +The four functions and two tables lived in ``graph_evidence`` and +``graph_terminal``, which define the gate kernels and which every spine +stage kernel, the dense population kernels and the export kernels hash too, +so an edit to a phase's list of modules re-keyed every node whose kernel +hashes either file, and every node after one. They moved here unchanged in +what they hash, with the one check both batteries call (microcosm#931, +ruled 9 Nov (late)). Only the graph construction, which sets each node's +parameter, and that check read this module. The check is listed +refusal-only in test_uk_graph, and no kernel hashes this module: a phase's +own code reaches its key through the parameter, which node_key hashes, and +both gate kernels' runs call :func:`require_current_phase_code` before they +evaluate anything, so a node whose parameter differs from the code this +process computes is refused whenever its kernel runs; the spine gates' +refusal was ruled on 9 Nov (late) too. A record stored under the node's key +is reused without running the kernel, so the check does not guard a cache +hit. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from functools import cache +from types import MappingProxyType + +from microcosm.graph import source_hash + + +@cache +def uk_spine_assembled_only_code_hash() -> str: + """The source hash of the code only the assembled spine phase computes + through: the ONS household type domain its enum gate checks + (``frs_relationships``), which no transferred-phase gate reads. + + So the transferred phase's own key leaves it out, though an edit to it + still re-keys the transferred gate, through the ``frs_relationships`` + stage, whose module it is, and every stage after it. Hashed once per + process, as for the calibrated gates' own code + (:func:`uk_terminal_only_code_hash`). + """ + from . import frs_relationships + + return source_hash(frs_relationships) + + +@cache +def uk_spine_transferred_only_code_hash() -> str: + """The source hash of the code only the transferred spine phase computes + through, which the assembled gate's key leaves out. + + That is what the stage-health checks only the transferred phase declares + compute through: the energy rake's published level and NEED margins + (``energy_pricing``); the bus travel, bus pricing, bus support pricing, + consumption basis and gambling checks' published facts and pricing + (``nts_bus_travel``, ``bus_use_incidence``, ``bus_fare_pricing``, + ``bus_support_pricing``, ``etb_services``, ``lcfs_consumption``, + ``donor_uprating``, ``gambling_participation``, the shared ledger's + calendar-year weights, and the vendored-fact reader with the chronicle + feed and its schema epochs); the annual income spells check + (``frs_annual_income``); the apprentice draw's take-up contract + (``take_up_contract``); the latent-attribute checks' binomial band + (``stochastic_assignment``); the band donor check's copy-id fields, + through which it reads a capped seating's seats from the frame + (``copy_id_table``; microcosm#931); the copy area check's agreement of + each copy with its parent (``copy_agreement``; microcosm#931); and the + council-tax check's reductions held within each household's bill + (``bounded_amounts``; microcosm#931). Its enum gate also checks the CGT + asset type domain (``cgt_asset_type``), and its geography ladder gate + (``uk_geography_ladder``, microcosm#931) computes through the ladder's + evaluator (``geography_ladder``), the local-authority resolution it + checks each row against (``local_authority_input``), the FRS region codes + (``rowwise_geography``), the household weights it reads + (``terminal_gates``) and the engine tables of the gate surface it is + handed (``microcosm.frame.materialize``). + + The assembled gate's own key, on which every later spine stage's chains, + therefore carries none of them (the re-review of microcosm#1420, E; + microcosm#931 ruling 6). An edit to one still re-keys the assembled gate + wherever a stage before it hashes the module. Stages at or before + ``area_brma``, the assembled phase's last since it took ``frs_brma``'s + place, hash ten of them: ``copy_id_table``, which the root stage binds; + ``stochastic_assignment`` and ``take_up_contract``; + ``frs_annual_income``, the module of ``frs_annual_income_spells``; the + vendored-fact reader, the chronicle feed and its schema epochs, which + ``frs_hmrc_spine_leaves`` binds; ``microcosm.frame.materialize``, which + ``frs_legacy_proxies`` binds; and ``rowwise_geography`` and + ``local_authority_input``, which the area stages before ``area_brma`` + bind (``atomic_area_assign`` and ``local_authority``; microcosm#931). + So an edit to any of the ten re-keys every spine node from the first + stage that hashes it onwards, the assembled gate included. No stage + before that gate hashes the other fifteen, so an edit to one of those + re-keys the stages that hash it, those after them and the transferred + gate, never the assembled gate. Hashed once per process, as the assembled + phase's own code is. + """ + import microcosm.frame.materialize as frame_materialize + + from .. import ( + bounded_amounts, + chronicle_epoch, + copy_agreement, + stochastic_assignment, + ) + from .. import ledger_targets as shared_ledger_targets + from . import ( + bus_fare_pricing, + bus_support_pricing, + bus_use_incidence, + cgt_asset_type, + chronicle_feed, + copy_id_table, + donor_uprating, + energy_pricing, + etb_services, + frs_annual_income, + gambling_participation, + geography_ladder, + lcfs_consumption, + ledger_fact_vendoring, + local_authority_input, + nts_bus_travel, + rowwise_geography, + take_up_contract, + terminal_gates, + ) + + return source_hash( + energy_pricing, + lcfs_consumption, + bus_fare_pricing, + bus_support_pricing, + bus_use_incidence, + nts_bus_travel, + etb_services, + gambling_participation, + donor_uprating, + frs_annual_income, + take_up_contract, + cgt_asset_type, + stochastic_assignment, + chronicle_feed, + chronicle_epoch, + ledger_fact_vendoring, + shared_ledger_targets, + copy_id_table, + copy_agreement, + bounded_amounts, + geography_ladder, + local_authority_input, + rowwise_geography, + terminal_gates, + frame_materialize, + ) + + +#: Each spine gate phase's own code: the hash of what only that phase's gates +#: compute through, which its node carries as its ``_code`` parameter +#: and the spine gate kernel checks before it evaluates anything +#: (:func:`require_current_phase_code`), so no other phase's own key carries +#: it. An edit there still re-keys the other phase wherever a node upstream of +#: it hashes the module, as each function's docstring says. The code both +#: phases compute through is the gate kernel's own hash (microcosm#931 ruling +#: 6); test_uk_graph derives each phase's share from the gates it declares. +UK_SPINE_GATE_PHASE_CODE: Mapping[str, Callable[[], str]] = MappingProxyType( + { + "assembled": uk_spine_assembled_only_code_hash, + "transferred": uk_spine_transferred_only_code_hash, + } +) + + +@cache +def uk_terminal_only_code_hash() -> str: + """The source hash of the code only the calibrated gates compute through. + + The source preflight runs the gate kernel's code and the dense solve keys + on the preflight, so the code only the terminal phase computes through + keys the calibrated node through its ``terminal_code`` parameter, not the + kernel's hash (microcosm#1306 review; microcosm#931 ruling 6), and the + preflight's own key, and with it the solve's, carries none of it. An edit + to one of these modules re-keys the calibrated gates and what follows + them, and re-solves too wherever a node upstream of the solve hashes the + module, as a spine stage, a target or problem kernel, a population kernel + or the dense solve itself does for most of them. Nine never re-solve, on + any line: ``high_value_home_gates``, ``microcosm.diagnostics``' catalogs + and schema, ``calibrate.hierarchy_labels``, the target loss attribution, + ``input_mass``, ``cgt_projection``, ``source_runtime`` and + ``cut_reference``. ``uk_runtime.diagnostics`` re-solves only the national + line, whose problem kernel hashes it. ``calibration_run`` and + ``release_certification`` re-solve only on a bound spine, whose kernel + hashes both, and nine that upstream of the solve only the spine's stage + and gate nodes hash re-solve only in the in-graph build + (``donor_uprating``, ``etb_services``, ``etb_vat``, ``frs_take_up``, + ``hmrc_capital_gains``, ``household_lineage``, ``source_lineage``, + ``take_up_contract``, ``uc_capital_coherence``). That is the terminal + branch of the full gate kernel's run, and what the terminal gates compute + through beyond the preflight's: + + - the high-value homes evidence, on the calibrated pool alone + (``high_value_home_gates``, with the tenure categories behind its + owner mask, ``tenure_constants``; the disclosure floor it reads from + ``receipt_counts`` is in the kernel's hash, since both phases' coverage + gate counts true sources through it, microcosm#1095). + ``high_value_homes`` itself is in the kernel's + hash, since the preflight's target-surface evidence reads its metric + names, and the target kernels hash it too, so an edit to it re-solves: + a disclosure fix between two builds that must reuse the solve belongs + in ``high_value_home_gates`` alone. ``calibrate.geography_constants``, + which the preflight's ledger parity reads, is in the kernel's hash too; + - the calibration diagnostics the calibrated node publishes as schema 9: + ``uk_runtime.diagnostics``, with its consumption drift + (``domestic_energy_targets``, ``energy_pricing``, ``etb_vat``, + ``lcfs_consumption``) and label choices (``local_authority_input``, + ``calibrate.hierarchy_labels``); ``microcosm.calibrate``'s + diagnostics, target loss attribution and solve; and + ``microcosm.diagnostics``' catalogs and schema; + - the solved problem and result it decodes (``microcosm.calibrate``'s + artifacts, matrix and monetary binding), the frame it rebuilds + (``graph_population``), the area-support summary (``local_rowwise``), + the support floors, admin anchors and CGT projection it builds the + gate context with (``support_floor``, ``etb_services``, + ``cgt_projection``, ``donor_uprating``, and the runner's admin totals + and projection artifact, ``calibration_run``), the input-mass reference + (``weighted_integrity``), whose candidate carriers it counts in true + source households through the shared lineage reader on the UK's copy + lineage (``source_lineage``, ``household_lineage``) and whose signed + exclusions it keys by the cut's reference releases (``cut_reference``; + microcosm#1095), and the fit-weight records it rehydrates + (``release_certification``); + - the terminal gates' evaluators: the input-mass parity + (``input_mass``), the CGT projection fence's conditioning facts + (``hmrc_capital_gains``, read through the vendored-fact reader, the + chronicle feed and its schema epochs), the take-up signal + (``frs_take_up``, ``take_up_contract``, ``dfe_spring_term``), the + column implication's property shares (``uc_capital_coherence``), the + nonnegative columns (``source_runtime``) and the geography ladder + (``rowwise_geography``). + + Hashed once per process, as the code was imported: an edit to the files + while a build runs cannot fail the calibrated gates after the solve. + """ + import microcosm.calibrate._target_loss_attribution as target_loss_attribution + import microcosm.calibrate.artifacts as calibrate_artifacts + import microcosm.calibrate.diagnostics as calibrate_diagnostics + import microcosm.calibrate.hierarchy_labels as hierarchy_labels + import microcosm.calibrate.matrix as calibrate_matrix + import microcosm.calibrate.monetary_binding as monetary_binding + import microcosm.calibrate.solve as calibrate_solve + import microcosm.diagnostics.catalogs as diagnostics_catalogs + import microcosm.diagnostics.schema as diagnostics_schema + + from .. import chronicle_epoch, input_mass, source_lineage + from . import ( + calibration_run, + cgt_projection, + chronicle_feed, + cut_reference, + dfe_spring_term, + diagnostics, + domestic_energy_targets, + donor_uprating, + energy_pricing, + etb_services, + etb_vat, + frs_take_up, + graph_population, + high_value_home_gates, + hmrc_capital_gains, + household_lineage, + lcfs_consumption, + ledger_fact_vendoring, + local_authority_input, + local_rowwise, + release_certification, + rowwise_geography, + source_runtime, + support_floor, + take_up_contract, + tenure_constants, + uc_capital_coherence, + weighted_integrity, + ) + + return source_hash( + high_value_home_gates, + diagnostics_catalogs, + diagnostics_schema, + hierarchy_labels, + diagnostics, + calibrate_diagnostics, + target_loss_attribution, + calibrate_artifacts, + calibrate_matrix, + monetary_binding, + calibrate_solve, + graph_population, + local_rowwise, + support_floor, + rowwise_geography, + release_certification, + weighted_integrity, + source_lineage, + household_lineage, + cut_reference, + input_mass, + calibration_run, + etb_services, + etb_vat, + domestic_energy_targets, + energy_pricing, + lcfs_consumption, + cgt_projection, + donor_uprating, + hmrc_capital_gains, + frs_take_up, + take_up_contract, + dfe_spring_term, + uc_capital_coherence, + tenure_constants, + source_runtime, + local_authority_input, + ledger_fact_vendoring, + chronicle_feed, + chronicle_epoch, + ) + + +@cache +def uk_preflight_only_code_hash() -> str: + """The source hash of the code only the source preflight computes + through: the shared ledger's registry parity report, behind the ledger + compile parity gates only the preflight declares + (``microcosm.build.ledger_targets``). + + The preflight's node carries it as its ``preflight_code`` parameter, so + the calibrated gates' own key leaves it out (microcosm#931 ruling 6). An + edit to it re-keys the preflight, the solve after it and the calibrated + gates through them, and every other node that hashes it: the full build's + target compilation, selection, measures and problem, the national line's + targets, problem and gates, and in the in-graph build the + ``etb_services`` and ``lcfs_consumption`` stages and the transferred + spine gate, whose own code it is too. Hashed once per process, as the + calibrated gates' own code is. + """ + from .. import ledger_targets as shared_ledger_targets + + return source_hash(shared_ledger_targets) + + +#: Each full gate phase's own code: the hash of what only that phase computes +#: through, which its node carries as its ``_code`` parameter and the +#: kernel checks before it evaluates anything +#: (:func:`require_current_phase_code`). The code both phases compute through +#: is the gate kernels' shared hash (microcosm#931 ruling 6); test_uk_graph +#: derives each phase's share from the gates it declares and the branch of the +#: kernel's run it takes. +UK_FULL_GATE_PHASE_CODE: Mapping[str, Callable[[], str]] = MappingProxyType( + { + "preflight": uk_preflight_only_code_hash, + "terminal": uk_terminal_only_code_hash, + } +) + + +def require_current_phase_code( + params: Mapping[str, object], phase: str, *, battery: str +) -> None: + """Refuse a gate node unless its ``_code`` parameter is its phase's + own code as this process computes it (microcosm#931, ruled 9 Nov (late)). + + ``battery`` is ``"spine"`` or ``"full"``, the battery whose table holds the + phase's code (:data:`UK_SPINE_GATE_PHASE_CODE`, + :data:`UK_FULL_GATE_PHASE_CODE`); a phase its table leaves out computes + through nothing of its own, and its node must carry no parameter. The + graph construction sets the parameter, and node_key hashes it. A node + with a stale or missing parameter, one declared in another process for + instance, is refused whenever its kernel runs, before the kernel + evaluates anything, so no gate evaluates its phase under a key that names + other phase code. A record stored under the node's key is reused without + running the kernel, so the check does not guard a cache hit. The check + reads only its arguments and the phase's code, and writes nothing, so it + cannot change what its caller computes: test_uk_graph lists it as + refusal-only, and no gate kernel's hash binds this module. + """ + tables = {"spine": UK_SPINE_GATE_PHASE_CODE, "full": UK_FULL_GATE_PHASE_CODE} + if battery not in tables: + raise ValueError(f"No UK gate battery {battery!r} carries phase codes.") + code = tables[battery].get(phase) + if params.get(f"{phase}_code") != (None if code is None else code()): + raise ValueError( + f"UK {battery} gate {phase}-only code differs from its declared binding." + ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/gate_scopes.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/gate_scopes.py new file mode 100644 index 0000000000..635cc86cdd --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/gate_scopes.py @@ -0,0 +1,155 @@ +"""The UK spine and local gate scopes, and the scoped gate manifest. + +All three lived in ``calibration_run``, the national calibration runner. The +full build's gate battery computes through them: both its phases build the +spine gate manifest (``graph_evidence.uk_spine_gate_manifest``) and classify +the local gates (``full_gates.classify_full_gate_outcomes``), and the dense +solve keys on the source preflight. So the battery's shared hash binds this +small module instead of all of ``calibration_run`` (microcosm#931), as the +target kernels hash ``ledger_surface`` (microcosm#1492). That keeps the +runner out of the full gate kernels' own hashes and the preflight's own +code. The calibrated gates' own code still binds the runner, whose admin +totals and CGT projection artifact their gate context is built with, so in +the in-graph build an edit elsewhere in the runner re-keys only the +calibrated gates and what follows them. On a bound spine the checkpoint, +whose kernel hashes the runner, still re-keys the preflight and the solve +keyed on it. ``calibration_run`` re-exports all three for its other callers, +beside the calibration, national and local diagnostic scopes it keeps. +""" + +from __future__ import annotations + +from microcosm.build.country_spec import GatesManifest, load_country_spec + +UK_LOCAL_GATE_SCOPE = ( + "uk_local_geography_ladder_post_calibration", + "uk_local_area_support", + "uk_local_target_fit", + "uk_local_per_family_fit", + "uk_local_weight_ratio", + "uk_local_weight_ess", +) + +UK_SPINE_GATE_SCOPE = ( + "uk_stage_was_wealth_support", + "uk_stage_was_wealth_coherence", + "uk_stage_was_lisa_support", + "uk_stage_was_lisa_anchor", + "uk_stage_high_value_home_support_mass", + "uk_stage_property_components", + "uk_stage_nts_bus_travel_support", + "uk_stage_nts_bus_travel_facts", + "uk_stage_uc_deduction_attributes", + "uk_stage_lcfs_consumption_support", + "uk_stage_lcfs_consumption_energy_rake", + "uk_stage_lcfs_consumption_bus_pricing", + "uk_stage_lcfs_consumption_road_fuel_incidence", + "uk_stage_lcfs_consumption_road_fuel_level", + "uk_stage_lcfs_consumption_recomposed_totals", + "uk_stage_lcfs_consumption_excise_quantities", + "uk_stage_lcfs_consumption_gambling_participation", + "uk_stage_lcfs_consumption_basis", + "uk_stage_etb_vat_support", + "uk_stage_etb_services_support", + "uk_stage_etb_services_support_pricing", + "uk_stage_frs_hmrc_spine_leaves_signal", + "uk_stage_frs_annual_income_spells", + "uk_stage_spi_support_channel_mass", + "uk_stage_hmrc_spi_income_spine_identity", + "uk_stage_hmrc_spi_income_spine_hourly_pay", + "uk_stage_spi_benefit_coherence", + "uk_stage_uc_take_up", + "uk_stage_pension_credit_take_up", + "uk_stage_child_benefit_take_up", + "uk_stage_cgt_incidence_clone_mass", + "uk_stage_cgt_support_split_mass", + "uk_stage_cgt_residential_split_mass", + "uk_stage_spi_income_band_donors_support", + "uk_stage_hmrc_cgt_gains_spine_summary", + "uk_stage_hmrc_cgt_asset_type_spine_summary", + "uk_stage_cgt_incidence_anchor_composition", + "uk_stage_salary_sacrifice_realization", + "uk_stage_student_loans_realization", + "uk_stage_age_tail_targets", + "uk_stage_frs_relationships_composition", + "uk_stage_area_home_values_ranking", + # The spine geography gates (microcosm#931): the area draw's mapping + # integrity at the assembled boundary, which closes after area_brma; the + # SPI-copy area stage's, and each later copy's agreement with its parent + # on the areas, at the transferred boundary, after spi_copy_areas. + "uk_atomic_area_mapping_integrity", + "uk_spi_copy_area_mapping_integrity", + "uk_copy_area_agreement", + # Each household's council-tax reductions within its bill at the + # transferred boundary, outside Northern Ireland (microcosm#931): + # council_tax_localise is the last spine stage that writes the bill and + # the reductions, and the copies made after it inherit them; the copy + # area check above also holds those copies to their parents on the bill + # and the rebate. + "uk_council_tax_reduction_within_bill", + "uk_stage_property_purchases_fit", + "uk_stage_uc_legacy_closure_take_up_contract", + "uk_stage_frs_person_draws_apprentices", + # The take-up flags at their drawing stages, before calibration (María's + # ruling of 2026-10-29, microcosm#1095). + "uk_stage_frs_take_up_signal", + "uk_stage_frs_person_draws_take_up_signal", + "uk_stage_frs_household_draws_take_up_signal", + "uk_stage_childcare_copy_take_up_signal", + "uk_stage_isa_subscriptions_invariants", + "uk_stage_isa_subscriptions", + "uk_stage_company_cars_invariants", + "uk_stage_spi_employment_shell_fit", + "uk_stage_aps_occupation_fit", + # Weight-independent like the BRMA enum below, and its column exists from + # frs_relationships onward (#791). + "uk_ons_household_type_enum_domain", + # Weight-independent; the column exists only from hmrc_cgt_asset_type_spine + # onward, so it is checked at the transferred boundary, and its domain is + # declared by the stage module (microcosm#725). + "uk_capital_gains_asset_type_enum_domain", + # Weight-independent; spi_housing_shell is the last spine stage that writes + # the band, so the transferred boundary checks every row's nation + # (microcosm#1278). + "uk_council_tax_band_domain", + # Weight-independent; spi_copy_areas is the last spine stage that writes + # brma and local_authority (the SPI copies and income band donors draw + # their own, microcosm#931), so the transferred boundary checks every + # row's, as for the band above. The BRMA gate replaces the local battery's + # retired recheck (uk_local_brma_enum_domain). + "uk_brma_enum_domain", + "uk_local_authority_enum_domain", + # The geography ladder on the transferred frame, the SPI copies' own + # ladders and the national line's among them (C-31, microcosm#931). + "uk_geography_ladder", +) + + +def uk_scoped_gate_manifest( + scope: tuple[str, ...] | frozenset[str], + *, + phases: tuple[str, ...], + policy_suffix: str, + source: GatesManifest | None = None, +) -> GatesManifest: + """Filter the declared UK gate spec to one battery's scope. + + The one scope-filtering implementation every scoped producer shares + (spine build, calibration seam, release cut). ``source`` lets a caller + that already holds a loaded spec — or a hermetic test that stubs one — + supply it; the default is the committed package spec. + """ + + if source is None: + source = load_country_spec("uk").gates + entries = tuple(entry for entry in source.gates if entry.id in scope) + missing = sorted(set(scope) - {entry.id for entry in entries}) + if missing: + raise RuntimeError(f"UK gate scope names undeclared gate id(s): {missing}.") + return GatesManifest( + country=source.country, + version=source.version, + policy=f"{source.policy}; {policy_suffix}", + phases=phases, + gates=entries, + ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_clones.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_clones.py new file mode 100644 index 0000000000..fc2a2c6df1 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_clones.py @@ -0,0 +1,131 @@ +"""The UK geographic clones' column names, and a frame's clone count. + +Each geographic clone of a household carries its index, 0 to ``K - 1``, on +the household, its benefit units and its people, under a per-entity name +(:func:`ladder_clone_index_column`); the written artifact keeps the legacy +schema's one shared name (:data:`ARTIFACT_CLONE_INDEX_COLUMN`). The UK copy +lineage reads the household's index as its ``geographic_support`` branch +(``household_lineage.UK_COPY_LINEAGE``), and :func:`geographic_clone_count` +reads ``K`` from it. The spine's ``geographic_expand`` stage writes the +indices and every household's source columns +(:data:`UK_GEOGRAPHIC_SOURCE_COLUMNS`) and mints the clones' ids in the +copy-id table's field :data:`GEOGRAPHIC_COPY_ID_FIELD`, and its +``geographic_identity`` stage each clone's geography key +(:data:`GEOGRAPHY_HOUSEHOLD_KEY_COLUMN`), which the dense line's +``atomic_area_support`` names ``IDENTITY_COLUMN``. + +The two index names lived in ``rowwise_dataset`` and the count in +``cgt_support``, which re-export them. This module holds the names and the +count alone and imports nothing first-party, as ``annuity_trust_column`` +holds its column's name. The stages that count by lineage read the +household's name through ``household_lineage``, and their nodes hash this +module with the counting modules (``graph_kernels._COUNTING_MODULES``). The +CGT support split, the support-clip stages and property components read +``K`` through the count, and the WAS stages, which count nothing by lineage, +hash this module as a helper. The bound spine's checkpoint, which holds its +sidecar's ``K`` to its frame's count, the national gates, whose +support-family fold reads the names, and the full build's export kernels, +which read the names, hash it as well. So an edit here re-keys all of them, +without binding the dense line's ``rowwise_dataset`` or +``atomic_area_support`` into the spine (microcosm#931). +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +__all__ = [ + "ARTIFACT_CLONE_INDEX_COLUMN", + "GEOGRAPHIC_COPY_ID_FIELD", + "GEOGRAPHY_HOUSEHOLD_KEY_COLUMN", + "SOURCE_HOUSEHOLD_KEY_COLUMN", + "SOURCE_YEAR_COLUMN", + "UK_GEOGRAPHIC_SOURCE_COLUMNS", + "geographic_clone_count", + "ladder_clone_index_column", +] + +#: Clone-index column name in the written artifact (one name per table — +#: the legacy single-year schema, unchanged by the Frame carrier). +ARTIFACT_CLONE_INDEX_COLUMN = "clone_index" +#: The household's survey year, which ``geographic_expand`` writes on every +#: household. +SOURCE_YEAR_COLUMN = "source_year" +#: ``f"{source_year}:{source_household_id}"``, which ``geographic_expand`` +#: writes on every household. +SOURCE_HOUSEHOLD_KEY_COLUMN = "source_household_key" +#: The households' source columns ``geographic_expand`` writes on every +#: household: the source id (each household's clone-0 id), its year and its +#: key. The source id is ``microcosm.frame.concepts.SOURCE_HOUSEHOLD_ID_COLUMN``, +#: spelled here so that this module imports nothing first-party; a test holds +#: the two spellings equal. +UK_GEOGRAPHIC_SOURCE_COLUMNS = ( + "source_household_id", + SOURCE_YEAR_COLUMN, + SOURCE_HOUSEHOLD_KEY_COLUMN, +) +#: The copy-id table's field the clones' ids are minted in: clone ``k`` of a +#: row takes the row's id at ordinal ``k`` there, and every later copy keeps +#: its clone's ordinal (``copy_id_table``; spelled here so that this module +#: imports nothing first-party). +GEOGRAPHIC_COPY_ID_FIELD = "geographic_support" +#: Each clone's geography key, which ``geographic_identity`` writes on every +#: household of the spine, the dense line's pool among them +#: (``atomic_area_support.IDENTITY_COLUMN`` is this name). +GEOGRAPHY_HOUSEHOLD_KEY_COLUMN = "geography_household_key" + + +def ladder_clone_index_column(entity: str) -> str: + """The in-memory, per-entity clone-index column on the ladder frame. + + Frame enforces globally unique column names across entity tables, so the + artifact's shared ``clone_index`` name cannot appear on all three tables + in memory; the writer renames these back at the export boundary. + """ + + return f"{entity}_{ARTIFACT_CLONE_INDEX_COLUMN}" + + +def geographic_clone_count(household: pd.DataFrame, *, stage: str | None = None) -> int: + """The frame's geographic clone count ``K``: one more than its largest index. + + The clone-first spine's ``geographic_expand`` stage (microcosm#931) gives + every household the index of its geographic clone in + ``household_clone_index``, 0 to ``K - 1``; a household table without that + column is one clone. Refuses a non-integer index column, a household + without an index, a negative index and indices that are not exactly 0 to + ``K - 1`` (an index below the largest that no household carries), since + a count above the clones the frame holds would widen whatever the caller + scales by it. Each refusal is a fixed sentence naming the column, the + calling ``stage`` where one is given and, for a column that is not + integer, its dtype: never a count, an id or a value. + """ + + column = ladder_clone_index_column("household") + if column not in household.columns: + return 1 + values = household[column] + rule = ( + f"{stage or 'A spine'} counts K geographic clones as the {column!r} " + "values 0 to K - 1 (microcosm#931)." + ) + if pd.api.types.is_bool_dtype(values.dtype) or not ( + pd.api.types.is_integer_dtype(values.dtype) + ): + raise ValueError( + f"{column!r} has dtype {values.dtype}, not an integer one. {rule}" + ) + if bool(values.isna().any()): + raise ValueError(f"A household carries no {column!r}. {rule}") + indices = values.to_numpy(dtype=np.int64) + if not indices.size: + return 1 + if bool((indices < 0).any()): + raise ValueError(f"A household carries a negative {column!r}. {rule}") + clone_count = int(indices.max()) + 1 + if int(np.unique(indices).size) != clone_count: + raise ValueError( + f"An index from 0 to the largest is carried by no household. {rule}" + ) + return clone_count diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_expand.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_expand.py new file mode 100644 index 0000000000..36776643d2 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_expand.py @@ -0,0 +1,463 @@ +"""The UK spine's geographic clones: K copies of every FRS household (microcosm#931). + +The clone-first spine gives every FRS household ``K`` geographic clones before +any other copy step runs. Each clone carries an equal share of its household's +design weight, so a later stage can give each clone an area of its own while +together they stand for the survey household once. This module is that step, +G1 of the clone-first plan: :class:`UKGeographicExpandStageTransform` and the +function it runs, :func:`expand_uk_geographic_clones`. + +The clones come from the shared, country-neutral core +(:func:`microcosm.build.clone_expansion.expand_clones`; shared rather than +UK-local, María's ruling of 2026-11-10) under the UK's declaration, +:data:`UK_GEOGRAPHIC_CLONES`: + +- **Ids.** Clone ``k`` of a row takes ``id + k x radix`` on all five id + columns, in the copy-id table's ``geographic_support`` field + (:func:`mint_geographic_clone_ids`). So a clone's id is a function of its + source id and its clone alone, and clone 0 keeps the frame's own ids. + Incoming ids the field has no room for are refused first + (:func:`require_room_for_geographic_clones`). +- **Clone indices.** Each table's ``{entity}_clone_index``, 0 to ``K - 1``, + with exactly ``K`` values on every table. +- **Weights.** Design weights only. Each clone takes ``w / K`` of its + household's weight, the same float in every clone, and the mass log gains + one record whose reason names no clone count. +- **Source id.** ``source_household_id``, the clone-0 id. + +The UK wrapper adds what the core does not do: + +- It refuses a frame that already carries a clone index, a source column or + any of the UK copy lineage's columns, since the step runs once, before any + other copy step. It also refuses weights on any entity but the household: + the core splits every weighted entity, and the UK weights households only. + And it refuses id and membership columns that are not int64, the dtype the + copy-id table mints, at every clone count, so every clone count takes the + same frames. +- Before the core runs, it writes each household's ``source_year`` (the + frame's time period) and ``source_household_key`` + (``f"{source_year}:{source_household_id}"``, as Python strings in an object + column) on the frame's own rows, so every clone carries its household's: + the values the SPI support channel wrote before this stage took the + columns over, as it still does on a household table without the clone + index. +- It validates the clones as a UK national frame. + +Every refusal gives counts, entities, columns, dtypes and declared constants +only, never an id, a weight or another value of a row. The evidence gives +``K``, the copy-id field and its radix, the distinct source households +rounded through ``receipt_counts``, and the weight split and kind; no row +count, id or weight. + +The spine's ``geographic_expand`` stage runs this step, after +``frs_annual_income_spells`` and before ``geographic_identity``. Its +declared operation, ``expand_geographic_clones``, must be the reviewed one +(:func:`geographic_expand_operation_parameters`), which names no clone +count: ``K`` is a parameter of the stage's graph node, so it enters the keys +of that node and every later one, never the stage's contract. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +import numpy as np +import pandas as pd + +from microcosm.build.clone_expansion import ( + CloneDeclaration, + CloneIdRule, + expand_clones, +) +from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.uk_runtime import copy_id_table +from microcosm.build.uk_runtime.cgt_structure import _assert_closed_world_operations +from microcosm.build.uk_runtime.geographic_clones import ( + GEOGRAPHIC_COPY_ID_FIELD, + SOURCE_HOUSEHOLD_KEY_COLUMN, + SOURCE_YEAR_COLUMN, + UK_GEOGRAPHIC_SOURCE_COLUMNS, + ladder_clone_index_column, +) +from microcosm.build.uk_runtime.geography_first_roster import GEOGRAPHIC_EXPAND_STAGE +from microcosm.build.uk_runtime.household_lineage import UK_COPY_LINEAGE +from microcosm.build.uk_runtime.national_frame import ( + UK_NATIONAL_SCHEMA, + uk_time_period, + validate_uk_national_frame, +) +from microcosm.build.uk_runtime.receipt_counts import ( + distinct_carriers, + record_source_households, + rounded_count, +) +from microcosm.frame import Frame, WeightKind +from microcosm.frame.concepts import SOURCE_HOUSEHOLD_ID_COLUMN + +__all__ = [ + "GEOGRAPHIC_CLONE_LIMIT", + "GEOGRAPHIC_COPY_ID_FIELD", + "GEOGRAPHIC_EXPAND_CONSERVATION", + "GEOGRAPHIC_EXPAND_ID_REMAPPING", + "GEOGRAPHIC_EXPAND_MASS_CHANGE_REASON", + "GEOGRAPHIC_EXPAND_OPERATION_KIND", + "GEOGRAPHIC_EXPAND_ORDER", + "GEOGRAPHIC_EXPAND_STAGE_NAME", + "GEOGRAPHIC_EXPAND_WEIGHT_SPLIT", + "SOURCE_HOUSEHOLD_KEY_COLUMN", + "SOURCE_YEAR_COLUMN", + "UK_GEOGRAPHIC_CLONES", + "UK_GEOGRAPHIC_EXPAND_OUTPUT_COLUMNS", + "UK_GEOGRAPHIC_SOURCE_COLUMNS", + "UKGeographicExpandResult", + "UKGeographicExpandStageTransform", + "expand_uk_geographic_clones", + "geographic_expand_operation_parameters", + "mint_geographic_clone_ids", + "require_room_for_geographic_clones", +] + +GEOGRAPHIC_EXPAND_STAGE_NAME = GEOGRAPHIC_EXPAND_STAGE +# The copy-id table's field the clones' ids are minted in +# (GEOGRAPHIC_COPY_ID_FIELD) is named in the geographic_clones leaf, which +# size_evaluation reads it from, and re-exported here. +#: The most clones the field numbers: clone ordinals run below it. Exported +#: so the spine and its graph can read the limit from here. +GEOGRAPHIC_CLONE_LIMIT = copy_id_table.ordinal_bound(GEOGRAPHIC_COPY_ID_FIELD) +# The source columns the step writes (UK_GEOGRAPHIC_SOURCE_COLUMNS, with +# SOURCE_YEAR_COLUMN and SOURCE_HOUSEHOLD_KEY_COLUMN) are named in the +# geographic_clones leaf, which the stages that read them hash without this +# module, and re-exported here. +#: Every column the step adds: each table's clone index, then the household's +#: source columns. +UK_GEOGRAPHIC_EXPAND_OUTPUT_COLUMNS = ( + *(ladder_clone_index_column(entity) for entity in UK_NATIONAL_SCHEMA.entities), + *UK_GEOGRAPHIC_SOURCE_COLUMNS, +) +#: The reason of the step's mass record. It names no clone count, so it is the +#: same at every K. +GEOGRAPHIC_EXPAND_MASS_CHANGE_REASON = ( + "Geographic clones split each household's design weight equally among its " + "clones, so household mass is conserved." +) +#: How the clones share a household's weight: each takes an equal share. +GEOGRAPHIC_EXPAND_WEIGHT_SPLIT = "equal" +#: The operation the stage declares, and the declaration's words for the +#: clones' order and for the household total, which the 1/K split keeps to +#: rounding (microcosm#931). +GEOGRAPHIC_EXPAND_OPERATION_KIND = "expand_geographic_clones" +GEOGRAPHIC_EXPAND_ORDER = "clone_major" +GEOGRAPHIC_EXPAND_CONSERVATION = "household_total_to_rounding" +#: How the declaration numbers the clones' ids: the copy-id table's field and +#: its radix, ``10**9``. +GEOGRAPHIC_EXPAND_ID_REMAPPING = ( + f"copy-id table field {GEOGRAPHIC_COPY_ID_FIELD}; clone k takes every entity " + "id + k x the field's radix " + f"(10**{len(str(copy_id_table.radix(GEOGRAPHIC_COPY_ID_FIELD))) - 1})" +) + + +def mint_geographic_clone_ids(ids: np.ndarray, k: int) -> np.ndarray: + """Clone ``k``'s ids: each id plus ``k`` times the ``geographic_support`` + field's radix, through the copy-id table, which refuses an ordinal outside + the field, an id already in it and an int64 overflow, by count.""" + + return copy_id_table.mint(ids, GEOGRAPHIC_COPY_ID_FIELD, k) + + +def require_room_for_geographic_clones(ids: np.ndarray, what: str) -> None: + """Refuse incoming ids at or above the ``geographic_support`` field's + radix, printing only how many, through the copy-id table.""" + + copy_id_table.require_below(ids, GEOGRAPHIC_COPY_ID_FIELD, what=what) + + +#: The UK's declaration for the shared clone expansion core: ids minted in the +#: ``geographic_support`` field, a clone index on every table, design weights, +#: the step's mass record reason and the household's clone-0 id. +UK_GEOGRAPHIC_CLONES = CloneDeclaration( + ids=CloneIdRule( + bound=GEOGRAPHIC_CLONE_LIMIT, + mint=mint_geographic_clone_ids, + require_below=require_room_for_geographic_clones, + ), + clone_index_columns={ + entity: ladder_clone_index_column(entity) + for entity in UK_NATIONAL_SCHEMA.entities + }, + weight_kinds=frozenset({WeightKind.DESIGN}), + reason=GEOGRAPHIC_EXPAND_MASS_CHANGE_REASON, + source_id_columns={"household": SOURCE_HOUSEHOLD_ID_COLUMN}, +) +#: The columns a frame entering the step must lack: what the step writes, and +#: every copy step's columns, since the step runs before any other copy step. +_RESERVED_COLUMNS = tuple( + dict.fromkeys((*UK_GEOGRAPHIC_EXPAND_OUTPUT_COLUMNS, *UK_COPY_LINEAGE.copy_columns)) +) + + +def geographic_expand_operation_parameters() -> tuple[ + tuple[str, dict[str, object]], ... +]: + """The reviewed operation the stage must declare verbatim: what + :data:`UK_GEOGRAPHIC_CLONES` and the UK wrapper do, and no clone count.""" + + return ( + ( + GEOGRAPHIC_EXPAND_OPERATION_KIND, + { + "entity": "household", + "clone_index_columns": dict(UK_GEOGRAPHIC_CLONES.clone_index_columns), + "ordinal_bound": GEOGRAPHIC_CLONE_LIMIT, + "id_remapping": GEOGRAPHIC_EXPAND_ID_REMAPPING, + "order": GEOGRAPHIC_EXPAND_ORDER, + "weight_split": GEOGRAPHIC_EXPAND_WEIGHT_SPLIT, + "weight_kind_out": WeightKind.DESIGN.value, + "source_columns": list(UK_GEOGRAPHIC_SOURCE_COLUMNS), + "conservation": GEOGRAPHIC_EXPAND_CONSERVATION, + "declared_factor": 1.0, + "reason": GEOGRAPHIC_EXPAND_MASS_CHANGE_REASON, + }, + ), + ) + + +@dataclass(frozen=True) +class UKGeographicExpandResult: + """The clones and what the step's evidence reads of them. + + ``source_households`` is the exact count of distinct source households; + the evidence gives it rounded, and neither field prints in the repr. + """ + + frame: Frame = field(repr=False) + clone_count: int + source_households: int = field(repr=False) + + def evidence(self) -> dict[str, object]: + return { + "stage": GEOGRAPHIC_EXPAND_STAGE_NAME, + "clone_count": int(self.clone_count), + "copy_id_field": GEOGRAPHIC_COPY_ID_FIELD, + "copy_id_radix": copy_id_table.radix(GEOGRAPHIC_COPY_ID_FIELD), + "source_households": rounded_count(self.source_households), + "weight_split": GEOGRAPHIC_EXPAND_WEIGHT_SPLIT, + "weight_kind": WeightKind.DESIGN.value, + } + + +def _clone_count(clones: object) -> int: + """``clones`` as an int, refusing anything outside [1, the limit].""" + + if ( + isinstance(clones, bool) + or not isinstance(clones, int | np.integer) + or not 1 <= clones <= GEOGRAPHIC_CLONE_LIMIT + ): + raise ValueError( + "The geographic expansion takes a clone count from 1 to " + f"{GEOGRAPHIC_CLONE_LIMIT}, the {GEOGRAPHIC_COPY_ID_FIELD!r} copy-id " + "field's declared bound (microcosm#931)." + ) + return int(clones) + + +def _assert_geographic_expand_stage(stage: SourceStageSpec) -> None: + """The stage a transform runs as: the step's name, household grain, + exactly the step's columns as outputs with no rewrites, the reviewed + operation (:func:`geographic_expand_operation_parameters`) and no + artifact.""" + + if not isinstance(stage, SourceStageSpec): + raise TypeError("The geographic expansion runs as a SourceStageSpec stage.") + if stage.stage != GEOGRAPHIC_EXPAND_STAGE_NAME: + raise ValueError( + f"Expected stage {GEOGRAPHIC_EXPAND_STAGE_NAME!r}, got {stage.stage!r}." + ) + if stage.grain != "household": + raise ValueError( + "The geographic expansion clones households; grain must be " + f"'household', got {stage.grain!r}." + ) + if tuple(stage.outputs) != UK_GEOGRAPHIC_EXPAND_OUTPUT_COLUMNS or tuple( + stage.rewrites + ): + raise ValueError( + "The geographic expansion writes exactly the clone indices and the " + "source columns; the stage must declare outputs " + f"{UK_GEOGRAPHIC_EXPAND_OUTPUT_COLUMNS} and no rewrites, got " + f"{tuple(stage.outputs)} / {tuple(stage.rewrites)}." + ) + _assert_closed_world_operations(stage, geographic_expand_operation_parameters()) + if tuple(stage.artifacts): + raise ValueError( + "The geographic expansion reads no artifact; the stage must declare " + f"none, got {len(stage.artifacts)}." + ) + + +def _refuse_other_weighted_entities(frame: Frame) -> None: + weighted = tuple(frame.weighted_entities) + if weighted != ("household",): + raise ValueError( + "The geographic expansion splits household weights alone, as the UK " + f"weights households only; the frame weights {list(weighted)}." + ) + + +def _refuse_narrow_ids(frame: Frame) -> None: + """Refuse id and membership columns that are not int64. + + The copy-id table mints int64 ids, so the core refuses narrower ids from + two clones on; refusing them at one clone too keeps every clone count + taking the same frames. + """ + + schema = frame.schema + person = frame.table(schema.person_entity) + columns = [(frame.table(e), schema.entity_id_column(e)) for e in frame.entities] + columns += [(person, schema.membership_column(g)) for g in schema.group_entities] + narrow = [ + f"{column!r} holds {table[column].dtype}" + for table, column in columns + if table[column].dtype != np.dtype("int64") + ] + if narrow: + raise ValueError( + "The geographic expansion takes int64 id and membership columns, " + f"the dtype the copy-id table mints; {', '.join(narrow)}." + ) + + +def _refuse_carried_lineage(frame: Frame) -> None: + carried = set().union(*(frame.table(entity).columns for entity in frame.entities)) + found = [column for column in _RESERVED_COLUMNS if column in carried] + if found: + raise ValueError( + "The geographic expansion clones a frame once, before any other " + f"copy step; the frame already carries {found}." + ) + + +def _source_year(frame: Frame) -> int: + """The frame's time period as a year: the source year the SPI support + channel wrote before this stage took the column over.""" + + try: + return int(uk_time_period(frame)) + except ValueError as error: + raise ValueError( + "The geographic expansion writes each household's source_year from " + "the frame's time period, which must name a year." + ) from error + + +def _with_source_year_and_key(frame: Frame, source_year: int) -> Frame: + """``frame`` with each household's ``source_year`` and + ``source_household_key``, before any clone exists. + + The key is written as Python strings in an object column whatever pandas + infers for text: the dtype the SPI support channel's own write gave before + this stage took the column over, as it still does on a household table + without the clone index, with string inference off, as in the legacy + plan, so that plan's object-string columns stay what they were. The graph + casts the column to its declared string dtype. + """ + + household = frame.table("household").copy() + ids = household[UK_NATIONAL_SCHEMA.id_column("household")].tolist() + household[SOURCE_YEAR_COLUMN] = np.full(len(household), source_year, np.int64) + household[SOURCE_HOUSEHOLD_KEY_COLUMN] = pd.Series( + [f"{source_year}:{source}" for source in ids], + index=household.index, + dtype=object, + ) + tables = {entity: frame.table(entity) for entity in frame.entities} + tables["household"] = household + return Frame( + tables, + frame.schema, + {"household": frame.weights_for("household")}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +def expand_uk_geographic_clones( + frame: Frame, *, clones: int +) -> UKGeographicExpandResult: + """``frame`` as ``clones`` geographic clones of every household, clone-major. + + Clone 0 is the frame's own rows and ids. Clone ``k`` repeats every + person, benefit unit and household with each id minted in the + ``geographic_support`` field at ordinal ``k``. Every row carries its + clone in its table's ``{entity}_clone_index``, every household its + ``source_household_id`` (the clone-0 id), ``source_year`` and + ``source_household_key``, and each clone ``w / clones`` of its + household's design weight, with one mass record stating + :data:`GEOGRAPHIC_EXPAND_MASS_CHANGE_REASON`. At one clone the rows, ids + and weights are the frame's own; only those columns and the record are + added. + + Refused, by count or by name: a clone count outside [1, + :data:`GEOGRAPHIC_CLONE_LIMIT`]; weights on any entity but the household, + or of any kind but design; a frame already carrying a clone index, a + source column or a copy step's column; id or membership columns that are + not int64; a time period that is not a year; incoming ids at or above the + field's radix; and every refusal of the shared core. + """ + + count = _clone_count(clones) + if not isinstance(frame, Frame): + raise TypeError("The geographic expansion clones a Frame.") + _refuse_other_weighted_entities(frame) + validate_uk_national_frame(frame) + _refuse_carried_lineage(frame) + _refuse_narrow_ids(frame) + base = _with_source_year_and_key(frame, _source_year(frame)) + expanded = expand_clones(base, clones=count, declaration=UK_GEOGRAPHIC_CLONES) + validate_uk_national_frame(expanded) + sources = record_source_households( + expanded, "household", stage=GEOGRAPHIC_EXPAND_STAGE_NAME + ) + return UKGeographicExpandResult( + frame=expanded, + clone_count=count, + source_households=distinct_carriers(sources.tolist()), + ) + + +@dataclass(frozen=True) +class UKGeographicExpandStageTransform: + """Whole-stage transform for the geographic expansion (G1). + + ``clones`` is the clone count ``K``. Constructing the transform checks + the stage it runs as and the count; running it calls + :func:`expand_uk_geographic_clones`. + """ + + stage: SourceStageSpec + clones: int + last_result: UKGeographicExpandResult | None = field( + default=None, init=False, repr=False + ) + + def __post_init__(self) -> None: + _assert_geographic_expand_stage(self.stage) + _clone_count(self.clones) + + def __call__(self, frame: Frame) -> Frame: + _assert_geographic_expand_stage(self.stage) + result = expand_uk_geographic_clones(frame, clones=self.clones) + object.__setattr__(self, "last_result", result) + return result.frame + + @staticmethod + def output_columns() -> tuple[str, ...]: + return UK_GEOGRAPHIC_EXPAND_OUTPUT_COLUMNS + + def checkpoint_metadata(self) -> dict[str, object]: + if self.last_result is None: + raise RuntimeError("checkpoint metadata requires a completed stage run.") + return {"evidence": self.last_result.evidence()} diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_identity.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_identity.py new file mode 100644 index 0000000000..6bad2b217a --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/geographic_identity.py @@ -0,0 +1,347 @@ +"""Each geographic clone's geography key, from its source and clone path (microcosm#931). + +G1b of the clone-first plan runs right after the geographic expansion +(:mod:`~microcosm.build.uk_runtime.geographic_expand`). It gives every +household the key the area assignment draws on, ``geography_household_key``: +``household_draw_key`` over the FRS source, the household's FRS vintage, its +true source and its copy path, the key the dense line's own identity node gave +its pool before the dense line took its clones from the spine (microcosm#931). + +- **Path.** The shared reader (``source_lineage.copy_paths`` on + ``household_lineage.UK_COPY_LINEAGE``) reads it from the lineage columns, + never from ids, weights or values. Before any copy step it is ``()`` for + clone 0 and ``(("geographic_support", k),)`` for clone ``k``. +- **Vintage.** The FRS vintage frs_release.json pins for the household's + ``source_year`` (:func:`frs_vintages_by_source_year`). A year it pins no + vintage for is refused. +- **Dtype.** The key is written in the graph's ``string`` dtype (pandas' + ``StringDtype("python")``), the dtype the graph stores a ``string`` cell + at, so it is not a legacy object-string column. The step names the dtype + itself, as ``atomic_household_lineage`` does, rather than through the + graph's token table: a stage's transform computes through the Frame's + modules alone, never the executor's. + +So clone 0's key is the key the same rule gives the base household, clone +``k``'s the key it gives that household's clone ``k``, and growing K never +changes an existing household's key. + +Refusals give counts, columns, dtypes and declared constants only, never a +key, an id or a year of a row. The evidence gives the distinct source +households rounded through ``receipt_counts`` (a household's clones counted +once), that the keys are unique, the source and the vintages; no key is ever +printed. + +The spine's ``geographic_identity`` stage runs this step, right after +``geographic_expand``. Its declared operation, ``key_households_by_clone_path``, +must be the reviewed one (:func:`geographic_identity_operation_parameters`), +and its one artifact the FRS release, with ``runtime_sha256_required``, whose +bytes enter the stage's contract. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass, field +from types import MappingProxyType + +import numpy as np +import pandas as pd + +from microcosm.build.source_lineage import copy_paths +from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.uk_runtime.atomic_household_identity import household_draw_key +from microcosm.build.uk_runtime.cgt_structure import _assert_closed_world_operations +from microcosm.build.uk_runtime.frs_release import UKFRSRelease, load_uk_frs_release +from microcosm.build.uk_runtime.geographic_clones import ( + GEOGRAPHY_HOUSEHOLD_KEY_COLUMN, + SOURCE_YEAR_COLUMN, + ladder_clone_index_column, +) +from microcosm.build.uk_runtime.geography_first_roster import ( + GEOGRAPHIC_IDENTITY_STAGE, +) +from microcosm.build.uk_runtime.household_lineage import ( + GEOGRAPHIC_SUPPORT_BRANCH, + UK_COPY_LINEAGE, +) +from microcosm.build.uk_runtime.national_frame import validate_uk_national_frame +from microcosm.build.uk_runtime.receipt_counts import ( + distinct_carriers, + rounded_count, +) +from microcosm.frame import Frame, Weights +from microcosm.frame.concepts import SOURCE_HOUSEHOLD_ID_COLUMN + +__all__ = [ + "FRS_RELEASE_ARTIFACT_ROLE", + "FRS_RELEASE_RESOURCE", + "GEOGRAPHIC_IDENTITY_INPUTS", + "GEOGRAPHIC_IDENTITY_OPERATION_KIND", + "GEOGRAPHIC_IDENTITY_SOURCE", + "GEOGRAPHIC_IDENTITY_STAGE_NAME", + "UK_GEOGRAPHIC_IDENTITY_OUTPUT_COLUMNS", + "UKGeographicIdentityResult", + "UKGeographicIdentityStageTransform", + "frs_vintages_by_source_year", + "geographic_identity_operation_parameters", + "key_uk_households_by_clone_path", +] + +GEOGRAPHIC_IDENTITY_STAGE_NAME = GEOGRAPHIC_IDENTITY_STAGE +#: The source every key names: the FRS, as the dense line's own identity node +#: named it before the dense line took its keys from the spine (microcosm#931). +GEOGRAPHIC_IDENTITY_SOURCE = "frs" +#: The household columns the step reads. +GEOGRAPHIC_IDENTITY_INPUTS = ( + SOURCE_HOUSEHOLD_ID_COLUMN, + SOURCE_YEAR_COLUMN, + ladder_clone_index_column("household"), +) +UK_GEOGRAPHIC_IDENTITY_OUTPUT_COLUMNS = (GEOGRAPHY_HOUSEHOLD_KEY_COLUMN,) +#: The operation the stage declares (microcosm#931). +GEOGRAPHIC_IDENTITY_OPERATION_KIND = "key_households_by_clone_path" +#: The stage's one artifact: the packaged FRS release, which pins the vintage +#: each key names, so its bytes enter the stage's contract. +FRS_RELEASE_ARTIFACT_ROLE = "frs_release" +FRS_RELEASE_RESOURCE = "frs_release.json" + + +def geographic_identity_operation_parameters() -> tuple[ + tuple[str, dict[str, object]], ... +]: + """The reviewed operation the stage must declare verbatim: the source the + keys name and the resource their vintage comes from.""" + + return ( + ( + GEOGRAPHIC_IDENTITY_OPERATION_KIND, + { + "source": GEOGRAPHIC_IDENTITY_SOURCE, + "vintage_resource": FRS_RELEASE_RESOURCE, + }, + ), + ) + + +def frs_vintages_by_source_year( + release: UKFRSRelease | None = None, +) -> Mapping[int, str]: + """The FRS vintage frs_release.json pins for each source year it covers. + + The release pins one vintage, which the loader checks starts with its + survey year. The geographic expansion writes each household's + ``source_year`` from the frame's time period, the release's base year, as + the SPI support channel did before that stage took the column over (and + still does on a household table without the clone index). So both years + map to the vintage (one year when they are equal), and every household's + key names it; any other year maps to none. ``release`` defaults to the + packaged pin. + """ + + release = load_uk_frs_release() if release is None else release + vintage = str(release.vintage) + return MappingProxyType( + {int(release.survey_year): vintage, int(release.base_year): vintage} + ) + + +@dataclass(frozen=True) +class UKGeographicIdentityResult: + """The keyed frame and what the step's evidence reads of it. + + ``source_households`` is the exact count of distinct source households; + the evidence gives it rounded, and neither field prints in the repr. + """ + + frame: Frame = field(repr=False) + source_households: int = field(repr=False) + source_vintages: tuple[str, ...] + + def evidence(self) -> dict[str, object]: + # A result exists only for unique keys: repeated keys are refused. + return { + "stage": GEOGRAPHIC_IDENTITY_STAGE_NAME, + "source_households": rounded_count(self.source_households), + "unique": True, + "source": GEOGRAPHIC_IDENTITY_SOURCE, + "source_vintages": list(self.source_vintages), + "inputs": list(GEOGRAPHIC_IDENTITY_INPUTS), + } + + +def _assert_geographic_identity_stage(stage: SourceStageSpec) -> None: + """The stage a transform runs as: the step's name, household grain, + exactly the key as its output with no rewrites, the reviewed operation + (:func:`geographic_identity_operation_parameters`), and the FRS release as + its one artifact, with ``runtime_sha256_required``.""" + + if not isinstance(stage, SourceStageSpec): + raise TypeError("The geography identity runs as a SourceStageSpec stage.") + if stage.stage != GEOGRAPHIC_IDENTITY_STAGE_NAME: + raise ValueError( + f"Expected stage {GEOGRAPHIC_IDENTITY_STAGE_NAME!r}, got {stage.stage!r}." + ) + if stage.grain != "household": + raise ValueError( + "The geography identity keys households; grain must be 'household', " + f"got {stage.grain!r}." + ) + if tuple(stage.outputs) != UK_GEOGRAPHIC_IDENTITY_OUTPUT_COLUMNS or tuple( + stage.rewrites + ): + raise ValueError( + "The geography identity writes exactly its key; the stage must " + f"declare outputs {UK_GEOGRAPHIC_IDENTITY_OUTPUT_COLUMNS} and no " + f"rewrites, got {tuple(stage.outputs)} / {tuple(stage.rewrites)}." + ) + _assert_closed_world_operations(stage, geographic_identity_operation_parameters()) + releases = [ + artifact + for artifact in stage.artifacts + if artifact.get("role") == FRS_RELEASE_ARTIFACT_ROLE + and artifact.get("resource") == FRS_RELEASE_RESOURCE + and artifact.get("runtime_sha256_required") is True + ] + if len(stage.artifacts) != 1 or len(releases) != 1: + raise ValueError( + "The geography identity reads its vintages from one artifact, role " + f"{FRS_RELEASE_ARTIFACT_ROLE!r} and resource {FRS_RELEASE_RESOURCE!r} " + "with runtime_sha256_required: true; the stage must declare it and no " + f"other, got {len(stage.artifacts)} artifact(s), {len(releases)} of " + "them that one." + ) + + +def _require_int64(household: pd.DataFrame, column: str) -> np.ndarray: + if column not in household.columns: + raise ValueError( + f"The geography identity reads household {column!r}, which the " + "geographic expansion writes; the frame lacks it." + ) + dtype = household[column].dtype + if dtype != np.dtype("int64"): + raise ValueError( + f"The geography identity reads an int64 {column!r}; it holds {dtype}." + ) + return household[column].to_numpy() + + +def _vintages(years: np.ndarray, release: UKFRSRelease | None) -> list[str]: + """Each household's FRS vintage, refusing years the release pins none for.""" + + by_year = frs_vintages_by_source_year(release) + unmapped = int((~np.isin(years, np.fromiter(by_year, np.int64))).sum()) + if unmapped: + raise ValueError( + f"{unmapped} household(s) carry a {SOURCE_YEAR_COLUMN} that " + "frs_release.json pins no FRS vintage for; it pins the source " + f"year(s) {sorted(by_year)} (microcosm#931)." + ) + return [by_year[year] for year in years.tolist()] + + +def key_uk_households_by_clone_path( + frame: Frame, *, release: UKFRSRelease | None = None +) -> UKGeographicIdentityResult: + """``frame`` with every household's ``geography_household_key``. + + Each key is ``household_draw_key`` over :data:`GEOGRAPHIC_IDENTITY_SOURCE`, + the vintage ``release`` (frs_release.json by default) pins for the + household's ``source_year``, its ``source_household_id`` and its copy path + read from the lineage columns, which must hold the geographic expansion's + clone index. Rows, ids, weights, the mass log and the metadata are the + frame's own. + + Refused, by count or by name: a frame that already carries the key, or + lacks the clone index or an int64 ``source_household_id`` or + ``source_year``; malformed lineage, through the shared reader; a source + year the release pins no vintage for; and keys that are not unique. + """ + + if not isinstance(frame, Frame): + raise TypeError("The geography identity keys a Frame.") + validate_uk_national_frame(frame) + household = frame.table("household") + if GEOGRAPHY_HOUSEHOLD_KEY_COLUMN in household.columns: + raise ValueError( + f"The frame already carries {GEOGRAPHY_HOUSEHOLD_KEY_COLUMN!r}; the " + "geography identity keys each household once." + ) + _require_int64(household, SOURCE_HOUSEHOLD_ID_COLUMN) + years = _require_int64(household, SOURCE_YEAR_COLUMN) + sources, paths = copy_paths( + household, UK_COPY_LINEAGE, require=(GEOGRAPHIC_SUPPORT_BRANCH,) + ) + vintages = _vintages(years, release) + keys = [ + household_draw_key( + source=GEOGRAPHIC_IDENTITY_SOURCE, + source_vintage=vintage, + source_household_id=source, + clone_path=path, + ) + for source, vintage, path in zip(sources.tolist(), vintages, paths, strict=True) + ] + repeated = len(keys) - len(set(keys)) + if repeated: + raise ValueError( + f"{repeated} household(s) repeat an earlier household's geography key; " + "every household needs its own source and clone path (microcosm#931)." + ) + keyed = household.copy() + keyed[GEOGRAPHY_HOUSEHOLD_KEY_COLUMN] = pd.Series( + keys, index=household.index, dtype=pd.StringDtype(storage="python") + ) + tables = {entity: frame.table(entity) for entity in frame.entities} + tables["household"] = keyed + # The household weights pass through unchanged. Annotated so the stage's + # static walk meets microcosm.frame's Weights module, as + # test_the_core_modules_are_on_every_nodes_path requires. + weights: Weights = frame.weights_for("household") + result = Frame( + tables, + frame.schema, + {"household": weights}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + validate_uk_national_frame(result) + return UKGeographicIdentityResult( + frame=result, + source_households=distinct_carriers(sources.tolist()), + source_vintages=tuple(sorted(set(vintages))), + ) + + +@dataclass(frozen=True) +class UKGeographicIdentityStageTransform: + """Whole-stage transform for the geography identity (G1b). + + Constructing it checks the stage it runs as; running it calls + :func:`key_uk_households_by_clone_path` with the packaged + frs_release.json. + """ + + stage: SourceStageSpec + last_result: UKGeographicIdentityResult | None = field( + default=None, init=False, repr=False + ) + + def __post_init__(self) -> None: + _assert_geographic_identity_stage(self.stage) + + def __call__(self, frame: Frame) -> Frame: + _assert_geographic_identity_stage(self.stage) + result = key_uk_households_by_clone_path(frame) + object.__setattr__(self, "last_result", result) + return result.frame + + @staticmethod + def output_columns() -> tuple[str, ...]: + return UK_GEOGRAPHIC_IDENTITY_OUTPUT_COLUMNS + + def checkpoint_metadata(self) -> dict[str, object]: + if self.last_result is None: + raise RuntimeError("checkpoint metadata requires a completed stage run.") + return {"evidence": self.last_result.evidence()} diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_first_roster.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_first_roster.py new file mode 100644 index 0000000000..1b19f7ad16 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_first_roster.py @@ -0,0 +1,87 @@ +"""The names that make a UK spine geography-first (microcosm#931). + +A geography-first spine runs every stage ``GEOGRAPHY_FIRST_STAGES`` names and +every operation ``GEOGRAPHY_FIRST_OPERATIONS`` names inside a stage that is +not one of them: ``spi_housing_shell``'s ``derive`` of the Great Britain +council tax of the support channel's copies at their region's expected Band D, +and ``was_wealth``'s ``localise_main_residence_value``. Its build +sidecar records how its area draw was bound in a ``geography`` block under +``SIDECAR_GEOGRAPHY_KEYS``. The dense and national lines build only on such a +spine, the national line at one clone +(:mod:`~microcosm.build.uk_runtime.spine_posture`), and the spine build's +sidecar writer writes the block under these keys. The stages' transforms read +their own names from here, so a stage and the roster cannot name it apart; a +test holds the two together too. Names only, importing nothing first-party. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from types import MappingProxyType + +__all__ = [ + "AREA_BRMA_STAGE", + "ATOMIC_AREA_ASSIGN_STAGE", + "COUNCIL_TAX_LOCALISE_STAGE", + "GEOGRAPHIC_EXPAND_STAGE", + "GEOGRAPHIC_IDENTITY_STAGE", + "GEOGRAPHY_FIRST_OPERATIONS", + "GEOGRAPHY_FIRST_STAGES", + "LOCAL_AUTHORITY_STAGE", + "PRIVATE_RENT_STAGE", + "RFDR_STAGE", + "SIDECAR_GEOGRAPHY_KEYS", + "SPI_COPY_AREAS_STAGE", +] + +#: Each geography-first stage's name, as its transform runs it: the clones, +#: their keys, the area draw, the local authority, the fuel-relief flag, the +#: BRMA, the SPI copies' areas, the private rents and the council tax +#: localised at each household's local authority. +GEOGRAPHIC_EXPAND_STAGE = "geographic_expand" +GEOGRAPHIC_IDENTITY_STAGE = "geographic_identity" +ATOMIC_AREA_ASSIGN_STAGE = "atomic_area_assign" +LOCAL_AUTHORITY_STAGE = "local_authority" +RFDR_STAGE = "rfdr" +AREA_BRMA_STAGE = "area_brma" +SPI_COPY_AREAS_STAGE = "spi_copy_areas" +PRIVATE_RENT_STAGE = "private_rent" +COUNCIL_TAX_LOCALISE_STAGE = "council_tax_localise" + +#: The stages a geography-first spine runs, in the spine's order. +GEOGRAPHY_FIRST_STAGES: tuple[str, ...] = ( + GEOGRAPHIC_EXPAND_STAGE, + GEOGRAPHIC_IDENTITY_STAGE, + ATOMIC_AREA_ASSIGN_STAGE, + LOCAL_AUTHORITY_STAGE, + RFDR_STAGE, + AREA_BRMA_STAGE, + SPI_COPY_AREAS_STAGE, + PRIVATE_RENT_STAGE, + COUNCIL_TAX_LOCALISE_STAGE, +) + +#: The operations a geography-first spine runs inside stages that are not its +#: own, by stage, in the spine's order: the housing shell's derive prices the +#: Great Britain copies' council tax at their region's expected Band D, which +#: the council-tax localisation then localises, and the main-residence +#: localiser runs inside the WAS chain. +GEOGRAPHY_FIRST_OPERATIONS: Mapping[str, tuple[str, ...]] = MappingProxyType( + { + "spi_housing_shell": ("derive",), + "was_wealth": ("localise_main_residence_value",), + } +) + +#: The keys of the build sidecar's ``geography`` block, in the order the +#: writer writes them: the assignment, the identity column it draws on, the +#: definition's digest, the support pins, the draw stream and the register the +#: pins are held to. +SIDECAR_GEOGRAPHY_KEYS: tuple[str, ...] = ( + "assignment", + "identity", + "definition_sha256", + "support_pins", + "stream", + "register", +) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_ladder.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_ladder.py index b8708598fe..cec4a4dd3c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_ladder.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_ladder.py @@ -757,10 +757,7 @@ def uk_geography_ladder_assignment_summary( else np.ones(len(household), dtype=np.float64) ) if len(weights) != len(household): - raise ValueError( - f"weight_values length {len(weights)} does not match household " - f"rows {len(household)}." - ) + raise ValueError("weight_values must give one weight per household row.") summary: dict[str, Any] = { "applied": applied, "household_rows": int(len(household)), @@ -818,10 +815,7 @@ def uk_geography_ladder_gate( details: dict[str, object] = {} weights = np.asarray(weight_values, dtype=np.float64) if len(weights) != len(household): - raise ValueError( - f"weight_values length {len(weights)} does not match household " - f"rows {len(household)}." - ) + raise ValueError("weight_values must give one weight per household row.") missing_columns = [ column @@ -853,10 +847,7 @@ def uk_geography_ladder_gate( [bool(value) and pattern.match(value) is None for value in values] ) if bad.any(): - failures.append( - f"{label}: {int(bad.sum())}/{len(values)} row(s) are not valid " - f"GSS codes; examples {sorted(set(values[bad]))[:5]}" - ) + failures.append(f"{label}: some row(s) are not valid GSS codes") # The engine input must agree with the code it was resolved from on every # row, and every code must sit on the April 2023 roster; reported, not @@ -872,46 +863,33 @@ def uk_geography_ladder_gate( ): mismatched = values != expected if mismatched.any(): - failures.append( - f"{label}: {int(mismatched.sum())}/{len(values)} row(s) disagree " - "with the ITL3 prefix" - ) + failures.append(f"{label}: some row(s) disagree with the ITL3 prefix") region = household["region_code"].astype(str).to_numpy() recognised_regions = frozenset(FRS_REGION_TO_REGION_CODE.values()) unknown_region = np.array([value not in recognised_regions for value in region]) if unknown_region.any(): - failures.append( - f"region_code: {int(unknown_region.sum())}/{len(region)} row(s) are " - "not recognised UK region codes; " - f"examples {sorted(set(region[unknown_region]))[:5]}" - ) + failures.append("region_code: some row(s) are not recognised UK region codes") # Rowwise consistency fence: the assigned region_code must agree with the # household's own pre-assigned region enum, so a mislabelled assignment # (e.g. Scottish households collapsed into English codes) cannot pass on # structural validity alone. - declared_codes = _household_region_codes( - household[region_column], - label=region_column, - ).to_numpy() - inconsistent = declared_codes != region - if inconsistent.any(): - examples = sorted( - { - f"{declared}!={assigned}" - for declared, assigned in zip( - declared_codes[inconsistent], - region[inconsistent], - strict=True, - ) - } - )[:5] - failures.append( - f"region_code: {int(inconsistent.sum())}/{len(region)} row(s) " - "disagree with the household's declared region; " - f"examples {examples}" - ) + try: + declared_codes = _household_region_codes( + household[region_column], + label=region_column, + ).to_numpy() + except ValueError: + # The reader's refusal lists the values it cannot read; the gate's + # failure names the column alone (C-32, microcosm#931). + failures.append(f"{region_column}: some row(s) carry no recognised UK region") + else: + inconsistent = declared_codes != region + if inconsistent.any(): + failures.append( + "region_code: some row(s) disagree with the household's declared region" + ) total = float(weights.sum()) if total <= 0: @@ -929,8 +907,8 @@ def uk_geography_ladder_gate( low, high = london_share_bounds if not low <= london_share <= high: failures.append( - f"London weighted household share {london_share:.4f} outside " - f"[{low}, {high}] (the region-collapse regression)" + f"London weighted household share outside [{low}, {high}] (the " + "region-collapse regression)" ) for label, bounds in ( @@ -942,17 +920,14 @@ def uk_geography_ladder_gate( details[f"{label}_nonempty_weighted_share"] = share low, high = bounds if not low <= share <= high: - failures.append( - f"{label} nonempty weighted share {share:.4f} outside [{low}, {high}]" - ) + failures.append(f"{label} nonempty weighted share outside [{low}, {high}]") constituency = household["constituency_code"].astype(str).to_numpy() constituency_share = float(weights[constituency != ""].sum() / total) details["constituency_nonempty_weighted_share"] = constituency_share if constituency_share < constituency_min_share: failures.append( - f"constituency nonempty weighted share {constituency_share:.4f} below " - f"{constituency_min_share}" + f"constituency nonempty weighted share below {constituency_min_share}" ) return GateResult( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph.py index 65d01c1d6e..5663422b24 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph.py @@ -16,6 +16,7 @@ import hashlib import json +import operator from collections.abc import Iterable, Mapping from dataclasses import dataclass from importlib import metadata @@ -34,6 +35,9 @@ from ..country_spec import CountrySpec, load_country_spec from ..stage_evidence import STAGE_EVIDENCE_TYPE +from .atomic_area_conditioning import SUPPORT_CHANNEL_BRANCH, UK_SPINE_AREA_COLUMNS +from .atomic_area_support import CONDITIONED_SOURCES, SYSTEMS +from .household_lineage import UK_COPY_LINEAGE from .national_sampling import UK_SAMPLE_SEED_DEFAULT __all__ = [ @@ -48,6 +52,7 @@ UK_SPINE_STRUCTURAL_STAGES = frozenset( { + "geographic_expand", "spi_support_channel", "spi_income_band_donors", "high_value_home_support", @@ -59,13 +64,19 @@ ) #: The last stage of the assembled checkpoint: everything through the base -#: FRS mapping and the stochastic draws. A name, not an index — a position -#: standing in for a key is correct only while two independently-maintained -#: orderings happen to agree (the uk-data#468 class). The graph freezes the -#: checkpoint after it, and the graph's spine gates and the plan path read it, -#: so an assembled-phase gate's stage is checked against the boundary the -#: build runs (the third and fourth reviews of microcosm#1422). -UK_SPINE_ASSEMBLED_FINAL_STAGE = "frs_brma" +#: FRS mapping and the stochastic draws, with the HMRC leaves, the annual +#: income spells, the geographic clones, their keys and their areas (the +#: area draw, the local authority and the fuel-relief flag), which run before +#: the draws, and the BRMA, which runs after them, since the engine's +#: LHA_category reads frs_take_up's would_claim_uc (microcosm#931). area_brma +#: took frs_brma's place, and the boundary with it. A name, not an index — a +#: position standing in for a key is correct only while two +#: independently-maintained orderings happen to agree (the uk-data#468 class). +#: The graph freezes the checkpoint after it, and the graph's spine gates and +#: the plan path read it, so an assembled-phase gate's stage is checked +#: against the boundary the build runs (the third and fourth reviews of +#: microcosm#1422). +UK_SPINE_ASSEMBLED_FINAL_STAGE = "area_brma" # The executor's mass ledger is weighted *person* mass per stratum # (``Frame.stratum_mass``: household weights broadcast through membership), so @@ -83,6 +94,9 @@ # person-mass ledger the executor verifies and asserts the household-mass # invariant itself (``UKExpandStageKernel``). _STRUCTURAL_MASS = { + # The geographic clones copy every household K ways at an equal share of + # its weight (microcosm#931), so person mass is conserved, to rounding. + "geographic_expand": "conserve", "spi_support_channel": "declared", # The reserved income rows are funded from the incumbent households of # each donor's region (microcosm#1063): household mass is conserved @@ -101,6 +115,9 @@ } _STRUCTURAL_WEIGHT_KIND = { + # The clones keep the root's design weights: an EXPAND may carry its + # base's kind (microcosm#931). + "geographic_expand": "design", "spi_support_channel": "importance", "spi_income_band_donors": "importance", "high_value_home_support": "importance", @@ -164,10 +181,22 @@ # open-surface SPI income stage, which would otherwise bind the cells # they rewrite (PolicyEngine/uk-autumn-budget-dashboard-2026#316). "company_cars", + # The SPI income stage rewrites the root income and report cells right + # after the open-surface SPI copies' area stage, which would otherwise + # bind them, while the income stage, itself open, reads the keys and + # areas that stage rewrites (microcosm#931). + "hmrc_spi_income_spine", # The student-loan stage moves each household's balance to its # borrowers, rewriting was_wealth's ``student_loan_balance``, which the # open-surface readers before it bind (microcosm#1095 B9). "student_loans", + # G6, the council-tax localisation, rewrites the housing shell's + # council tax, rebate and person council tax reduction right after + # private_rent, whose finite reader anchors on the shell's person + # council tax reduction; without its own version the private rents + # would bind G6's reduction and G6 the private rents' rent, a cycle + # (microcosm#931). + "council_tax_localise", } ) # ``age_tail`` needs no isolation entry: it is declared immediately after the @@ -175,6 +204,14 @@ # owner rule opens its boundary. Every later age reader then binds to the # disaggregated cell — the #785 contract — instead of the top-coded root. +#: The three conditioned atomic-area supports, by the ids sources.yaml +#: declares them under, in the systems' order: graph sources of their raw +#: bytes, which the area stages decode and hold to canonical bytes +#: (microcosm#931). +_ATOMIC_SUPPORT_SOURCES = tuple(CONDITIONED_SOURCES[system] for system in SYSTEMS) +#: The split sources read as raw bytes; every other is a pinned local table. +_RAW_BYTE_SOURCES = frozenset(_ATOMIC_SUPPORT_SOURCES) + _SPLIT_STAGE_SOURCES: Mapping[str, tuple[str, ...]] = { "frs_spine": ("frs",), "frs_relationships": ("frs",), @@ -214,6 +251,15 @@ "frs_annual_income_spells": ("frs",), "spi_income_band_donors": ("spi",), "hmrc_spi_income_spine": ("spi", "hmrc_income"), + # The area draw, the fuel-relief flag's roster, the SPI copies' own draw, + # the council-tax localisation's draw cells and the housing shell's + # reference Band D for SPI copies read the three conditioned atomic-area + # supports, whose bytes enter their nodes' keys (microcosm#931). + "atomic_area_assign": _ATOMIC_SUPPORT_SOURCES, + "rfdr": _ATOMIC_SUPPORT_SOURCES, + "spi_copy_areas": _ATOMIC_SUPPORT_SOURCES, + "spi_housing_shell": _ATOMIC_SUPPORT_SOURCES, + "council_tax_localise": _ATOMIC_SUPPORT_SOURCES, } _SPLIT_SOURCE_DESCRIPTIONS = { @@ -232,8 +278,37 @@ "aps": "Pinned local APS person donor table.", "spi": "Pinned local SPI donor table.", "hmrc_income": "Pinned local HMRC income facts workbook.", + **{ + source: ( + f"Registered local {system} conditioned atomic-area support, as raw bytes." + ) + for system, source in CONDITIONED_SOURCES.items() + }, } +#: The geographic clones' lineage: each household's true source and its clone. +#: frs_take_up, frs_person_draws and frs_household_draws run on every clone, +#: after geographic_expand, and count their receipts' support before the +#: copies, which the shared reader does on the source column wherever the +#: frame carries it, refusing the clone index without it (microcosm#931). +#: frs_take_up's three anchored draws and frs_person_draws' apprentice walk +#: also read household_clone_index to count their targets within each clone. +_GEOGRAPHIC_CLONE_LINEAGE = frozenset( + {("household", "source_household_id"), ("household", "household_clone_index")} +) +#: The SPI support channel's branch of the copy lineage: the household columns +#: marking the rows the channel copied, its copies and income band donors, +#: which the council-tax localisation prices at their own draw cell. The +#: later branches are not yet written where it runs (microcosm#931). +_SUPPORT_CHANNEL_LINEAGE = frozenset( + ("household", column) + for column in next( + branch.columns + for branch in UK_COPY_LINEAGE.branches + if branch.name == SUPPORT_CHANNEL_BRANCH + ) +) + # ``None`` means the implementation genuinely has an open formula/model # surface and therefore receives every live cell. Finite sets are the direct # Frame reads inventoried from the existing transform. Structural ids, @@ -300,7 +375,8 @@ "months_since_last_birthday", ) ) - | frozenset({("benunit", "is_married")}), + | frozenset({("benunit", "is_married")}) + | _GEOGRAPHIC_CLONE_LINEAGE, # is_apprentice draws employees aged 21 to 24 at a rate set by the # frame's weights (microcosm#1372), outside full-time education (Vahid's # review of microcosm#1535, nit 3), and its receipt reads their pay for @@ -313,20 +389,26 @@ "is_in_full_time_education", "employment_income", ) - ), - "frs_household_draws": frozenset(), - "frs_brma": None, + ) + | _GEOGRAPHIC_CLONE_LINEAGE, + "frs_household_draws": _GEOGRAPHIC_CLONE_LINEAGE, + # The BRMA, in frs_brma's place: one engine materialization of + # LHA_category over the whole frame, as frs_brma's, so its surface is + # open (microcosm#931). + "area_brma": None, "was_wealth": None, # The LISA model reads the was_wealth household predictors through the # engine (household net income over the whole frame), an open surface like # the WAS chain it conditions on. "was_lisa": None, # The factor's mean is taken over FRS-base owners only, so the support - # channel is a direct read. + # channel is a direct read, and over the declared denominator, the WAS + # chain's draw that its area home values keep (microcosm#931). "regional_property_uprating": frozenset( { ("household", "region"), ("household", "main_residence_value"), + ("household", "main_residence_value_rank_basis"), ("household", "property_wealth"), ("household", "owned_land"), ("household", "other_residential_property_value"), @@ -339,7 +421,8 @@ # channel and the household reference person's sub-letting rent), the # landlords' ages the FRS-concept finance-cost draw conditions on, and the # rental property values the coherence receipt reads (microcosm#1106, - # microcosm#840). + # microcosm#840). The geographic clone index is declared too, so that a + # check of the stage can read its frame's clone count (microcosm#931). "property_components": frozenset( { ("person", "age"), @@ -351,19 +434,7 @@ ("household", "subrent"), ("household", "other_residential_property_value"), ("household", "non_residential_property_value"), - } - ), - # Engine-free: the owners' region, tenure, value and wealth, the support - # copies the ranking spreads, and the source household the receipt counts - # distinct support by (microcosm#1112, microcosm#1095). - "regional_home_values": frozenset( - { - ("household", "region"), - ("household", "tenure_type"), - ("household", "main_residence_value"), - ("household", "property_wealth"), - ("household", "household_is_high_value_support_copy"), - ("household", "source_household_id"), + ("household", "household_clone_index"), } ), # Engine-free: each kind's stock, the region that marks where SDLT @@ -397,8 +468,49 @@ "gender", ) ), + # The geographic clones copy whole households, so their surface is open + # (microcosm#931). + "geographic_expand": None, + # The geography key reads each household's true source, its source year + # and its clone, the lineage geographic_expand writes (microcosm#931). + "geographic_identity": frozenset( + { + ("household", "source_household_id"), + ("household", "source_year"), + ("household", "household_clone_index"), + } + ), + # Engine-free: the area draw reads each household's geography key, region, + # IMD decile and observed council-tax band, the root's area-conditioning + # carriers, and its true source and clone, which its evidence counts by + # (microcosm#931). + "atomic_area_assign": frozenset( + { + ("household", "geography_household_key"), + ("household", "region"), + ("household", "imd_decile"), + ("household", "council_tax_band"), + ("household", "council_tax_band_observed"), + } + ) + | _GEOGRAPHIC_CLONE_LINEAGE, + # Engine-free: the local authority reads the drawn district code + # (microcosm#931). + "local_authority": frozenset({("household", "local_authority_code")}) + | _GEOGRAPHIC_CLONE_LINEAGE, + # Engine-free: the fuel-relief flag reads the drawn output area and its + # system (microcosm#931). + "rfdr": frozenset( + {("household", "atomic_area_code"), ("household", "atomic_area_system")} + ) + | _GEOGRAPHIC_CLONE_LINEAGE, "spi_support_channel": None, "spi_income_band_donors": None, + # The SPI copies' areas read every household's copy path and rewrite the + # copies' keys and areas, and Northern Ireland's copies take one engine + # materialization of LHA_category over their households: an open surface + # (microcosm#931). + "spi_copy_areas": None, "hmrc_spi_income_spine": None, # Engine-free: the occupation predictors, the drawn-row rule and the channel # split. The rewritten occupation, industry and sector cells arrive as the @@ -421,6 +533,7 @@ ("household", "household_support_channel"), ("household", "household_support_clone_index"), ("household", "source_household_id"), + ("household", "household_clone_index"), } ), # Engine-free: the sub-major predictors, the major group it nests in, the @@ -444,11 +557,15 @@ # Engine-free: the housing predictors and the channel split. The rewritten # housing and benefit cells arrive as the rewrite's incumbents. The # lineage live here lets the model fold geographic clones into each survey - # household (microcosm#931). + # household, and names the support channel's copies, whose Great Britain + # bills the stage prices at their region's expected Band D on the + # conditioned supports it reads as graph sources. It reads no household + # area column (microcosm#931). "spi_housing_shell": frozenset( { ("household", "source_household_id"), ("household", "household_support_clone_index"), + ("household", "household_clone_index"), ("person", "age"), ("person", "is_household_head"), ("person", "employment_income"), @@ -466,6 +583,44 @@ ("household", "household_support_channel"), } ), + # Engine-free: the private rents read each household's geography key, its + # region, tenure, bedrooms, BRMA and channel, and its true source and + # clone, which its rule and evidence count by. The rewritten rent arrives + # as the rewrite's incumbent (microcosm#931). + "private_rent": frozenset( + { + ("household", "geography_household_key"), + ("household", "region"), + ("household", "tenure_type"), + ("household", "num_bedrooms"), + ("household", "brma"), + ("household", "household_support_channel"), + } + ) + | _GEOGRAPHIC_CLONE_LINEAGE, + # Engine-free: the council-tax localisation reads each household's draw + # inputs (its geography key, region, IMD decile and council-tax band, and + # whether the band was observed), its drawn area, system and local + # authority, its bill's provenance and the statute's single-adult flag, + # the support channel's branch of its copy path, and its true source and + # clone, which its evidence counts by. The rewritten bill, rebate and + # person reduction arrive as the rewrite's incumbents (microcosm#931). + "council_tax_localise": frozenset( + { + ("household", "geography_household_key"), + ("household", "region"), + ("household", "imd_decile"), + ("household", "council_tax_band"), + ("household", "council_tax_band_observed"), + ("household", "atomic_area_code"), + ("household", "atomic_area_system"), + ("household", "local_authority_code"), + ("household", "council_tax_bill_provenance"), + ("household", "council_tax_single_adult_raw"), + } + ) + | _SUPPORT_CHANNEL_LINEAGE + | _GEOGRAPHIC_CLONE_LINEAGE, # Runs one temporary engine materialization over the whole frame for its # award screen, so its input surface is genuinely open. "uc_reporter_redraw": None, @@ -547,6 +702,7 @@ ("household", "source_household_id"), ("household", "household_support_channel"), ("household", "household_support_clone_index"), + ("household", "household_clone_index"), ("household", "household_is_high_value_support_copy"), ("household", "high_value_support_copy_index"), ("person", "is_benunit_head"), @@ -844,7 +1000,12 @@ def owned(self, *, rewrite: bool = False) -> Owned: "accommodation_type", "council_tax_band", } -_ROOT_HOUSEHOLD_INT = {"num_bedrooms", "council_tax_single_adult_raw"} +#: imd_decile and council_tax_band_observed are the root's two area- +#: conditioning carriers (frs_spine.FRS_AREA_CONDITIONING_COLUMNS, +#: microcosm#931): the decile int64, 0 where none is observed, and the flag +#: bool. +_ROOT_HOUSEHOLD_INT = {"num_bedrooms", "council_tax_single_adult_raw", "imd_decile"} +_ROOT_HOUSEHOLD_BOOL = {"council_tax_band_observed"} _STRUCTURAL_COLUMNS = { "person_id", "person_benunit_id", @@ -865,6 +1026,8 @@ def _root_cells(stage_outputs: Iterable[str]) -> tuple[_Cell, ...]: cells.append(_Cell("household", column, "string")) elif column in _ROOT_HOUSEHOLD_INT: cells.append(_Cell("household", column, "int64")) + elif column in _ROOT_HOUSEHOLD_BOOL: + cells.append(_Cell("household", column, "bool")) elif column in _ROOT_PERSON_STRING: cells.append(_Cell("person", column, "string")) elif column in _ROOT_PERSON_BOOL: @@ -925,7 +1088,12 @@ def _cells( _Cell("person", "sic_industry_division", "int64"), _Cell("person", "soc2020_major_group", "string"), ), - "frs_council_tax": (_Cell("household", "council_tax", "float64"),), + # The bill, and its provenance: the branch it took, which the council-tax + # localisation alone reads (microcosm#931). + "frs_council_tax": ( + _Cell("household", "council_tax", "float64"), + _Cell("household", "council_tax_bill_provenance", "string"), + ), "frs_disability": ( *_cells( "person", @@ -1019,7 +1187,9 @@ def _cells( ), "bool", ), - "frs_brma": (_Cell("household", "brma", "string"),), + # The BRMA keeps frs_brma's cell, an object column read as a legacy + # string (microcosm#931). + "area_brma": (_Cell("household", "brma", "string"),), "was_wealth": ( *_cells( "household", @@ -1031,6 +1201,10 @@ def _cells( "gross_financial_wealth", "net_financial_wealth", "main_residence_value", + # The chain's draw of the main residence, which its area home + # values keep and the regional uprating divides by + # (microcosm#931). + "main_residence_value_rank_basis", "other_residential_property_value", "non_residential_property_value", "savings", @@ -1082,16 +1256,6 @@ def _cells( "person", ("property_income", "property_finance_costs", "property_rental_income"), ), - # The region's Land Registry values on the spine, and the value they - # replace for the dense node to rank on (microcosm#1112). - "regional_home_values": _cells( - "household", - ( - "main_residence_value", - "property_wealth", - "main_residence_value_rank_basis", - ), - ), # Annual purchases from each kind of property stock (uk-data#573). "property_purchases": _cells( "household", @@ -1215,6 +1379,28 @@ def _cells( "contributory_benefits_annual_share", ), ), + # Each table's geographic clone index and each household's source + # columns (microcosm#931), in the order the transform writes them: the + # source year and key on the base households, then the shared core's + # source id and clone index. The graph appends an EXPAND's new cells in + # this order, so it matches the legacy plan's column order. + "geographic_expand": ( + _Cell("person", "person_clone_index", "int64"), + _Cell("benunit", "benunit_clone_index", "int64"), + _Cell("household", "source_year", "int64"), + _Cell("household", "source_household_key", "string"), + _Cell("household", "source_household_id", "int64"), + _Cell("household", "household_clone_index", "int64"), + ), + "geographic_identity": (_Cell("household", "geography_household_key", "string"),), + # The area draw's three assignment and ten ladder columns, the engine's + # local authority and the fuel-relief flag (microcosm#931), in the order + # each stage writes them. + "atomic_area_assign": _cells("household", UK_SPINE_AREA_COLUMNS, "string"), + "local_authority": (_Cell("household", "local_authority", "string"),), + "rfdr": (_Cell("household", "in_rural_fuel_duty_relief_area", "bool"),), + # The households' source columns are geographic_expand's cells; the + # channel's copies inherit them (microcosm#931). "spi_support_channel": ( _Cell("person", "person_source_id", "int64"), _Cell("person", "person_support_channel", "string"), @@ -1222,9 +1408,6 @@ def _cells( _Cell("benunit", "benunit_source_id", "int64"), _Cell("benunit", "benunit_support_channel", "string"), _Cell("benunit", "benunit_support_clone_index", "int64"), - _Cell("household", "source_household_id", "int64"), - _Cell("household", "source_year", "int64"), - _Cell("household", "source_household_key", "string"), _Cell("household", "household_source_id", "int64"), _Cell("household", "household_support_channel", "string"), _Cell("household", "household_support_clone_index", "int64"), @@ -1247,6 +1430,15 @@ def _cells( _Cell("household", "spi_income_band_donor_lower_bound", "float64"), _Cell("person", "person_is_spi_income_band_carrier", "bool"), ), + # The SPI copies' own keys and areas rewrite the cells the identity and + # area stages own (microcosm#931). + "spi_copy_areas": ( + _Cell("household", "geography_household_key", "string"), + *_cells("household", UK_SPINE_AREA_COLUMNS, "string"), + _Cell("household", "local_authority", "string"), + _Cell("household", "brma", "string"), + _Cell("household", "in_rural_fuel_duty_relief_area", "bool"), + ), "hmrc_spi_income_spine": (), # populated below from typed groups "spi_employment_shell": ( _Cell("person", "soc2020_major_group", "string"), @@ -1279,6 +1471,17 @@ def _cells( ), *_cells("person", ("housing_benefit_reported", "council_tax_benefit_reported")), ), + # Great Britain private renters' rents, read off their BRMA's list + # (microcosm#931). + "private_rent": (_Cell("household", "rent", "float64"),), + # The paid bill kept from before the council-tax localisation, then the + # bill, the rebate and the person council tax reduction it rewrites at + # each household's local authority (microcosm#931). + "council_tax_localise": ( + _Cell("household", "council_tax_frs_bill", "float64"), + *_cells("household", ("council_tax", "council_tax_rebate")), + _Cell("person", "council_tax_benefit_reported", "float64"), + ), "uc_reporter_redraw": (_Cell("person", "universal_credit_reported", "float64"),), "spi_benefit_coherence": ( *_cells( @@ -1697,8 +1900,14 @@ def _source_refs(source_mode: str) -> tuple[SourceRef, ...]: "Content-bound UK FRS and donor fixture/source bundle.", ), ) + # The conditioned supports are one file each, read as raw bytes + # (microcosm#931); every other split source is a table directory or file. return tuple( - SourceRef(name, "csv-tables", description) + SourceRef( + name, + "raw-bytes-v1" if name in _RAW_BYTE_SOURCES else "csv-tables", + description, + ) for name, description in _SPLIT_SOURCE_DESCRIPTIONS.items() ) @@ -1708,6 +1917,11 @@ def _numerical_dependency_versions() -> tuple[tuple[str, str], ...]: The missing marker permits source-only declarations without engine extras; installing the engine then produces a different identity, never a false hit. + microcosm-build does not declare microdf-python, which policyengine-uk + installs, but private_school_attendance stores its own restatement of the + engine's percentile rank, by microdf's weighted percentile rank, wherever + the engine computes the frame clone by clone, so its version is bound too + (microcosm#931; it is declared at the next lock bump). """ versions = [] for name in ( @@ -1717,6 +1931,7 @@ def _numerical_dependency_versions() -> tuple[tuple[str, str], ...]: "pandas", "scikit-learn", "quantile-forest", + "microdf-python", ): try: version = metadata.version(name) @@ -1732,10 +1947,21 @@ def uk_spine_graph( source_mode: str = "bundle", sample_fraction: float = 1.0, sample_seed: int = UK_SAMPLE_SEED_DEFAULT, + geographic_clones: int = 1, ) -> Graph: - """Return the source-bound graph for the packaged UK FRS spine.""" + """Return the source-bound graph for the packaged UK FRS spine. + + ``geographic_clones`` is the spine's clone count ``K``, 1 to the + ``geographic_support`` copy-id field's bound. The geographic_expand node + carries it as a parameter, so it enters that node's key, and through the + chain every later node's, but never a stage contract (microcosm#931). + """ from .frs_spine import UKFRSSpineStageTransform + from .geographic_expand import ( + GEOGRAPHIC_CLONE_LIMIT, + GEOGRAPHIC_EXPAND_STAGE_NAME, + ) if source_mode not in {"bundle", "split"}: raise ValueError("UK graph source_mode must be 'bundle' or 'split'.") @@ -1743,8 +1969,29 @@ def uk_spine_graph( raise ValueError("UK graph sample_fraction must be in (0, 1].") if sample_seed < 0: raise ValueError("UK graph sample_seed must be non-negative.") + try: + clones = ( + None + if isinstance(geographic_clones, bool) + else operator.index(geographic_clones) + ) + except TypeError: + clones = None + if clones is None or not 1 <= clones <= GEOGRAPHIC_CLONE_LIMIT: + raise ValueError( + "UK graph geographic_clones must be a whole number from 1 to " + f"{GEOGRAPHIC_CLONE_LIMIT}, the geographic_support copy-id field's " + "bound." + ) resolved = load_country_spec("uk") if spec is None else spec stages = _manifest_stages(resolved) + if clones != 1 and not any( + stage.stage == GEOGRAPHIC_EXPAND_STAGE_NAME for stage in stages + ): + raise ValueError( + "UK graph geographic_clones above 1 needs the " + f"{GEOGRAPHIC_EXPAND_STAGE_NAME!r} stage, which makes the clones." + ) dependency_versions = _numerical_dependency_versions() # The root transform loads the complete national-frame seed schema even # when a reduced hermetic manifest names only the output under test. @@ -1828,6 +2075,13 @@ def uk_spine_graph( "stage_contract_sha256": _stage_contract_sha256( manifest_stage, resolved ), + # The clone count K is the node's, never its stage + # contract's (microcosm#931). + **( + {"geographic_clones": clones} + if stage_name == GEOGRAPHIC_EXPAND_STAGE_NAME + else {} + ), }, structural=StructuralDelta.EXPAND, artifact_outputs=( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py index 54152fec59..19c6920570 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_build.py @@ -2,13 +2,15 @@ The same graph handles all targets (the default), explicit target filters, dense exports and exact-count exports. A bound spine checkpoint resumes this -composition; an arbitrary historical uk-data H5 is not a build source. +composition; an arbitrary historical uk-data H5 is not a build source. The +full build composes its targets, measures and problem on the spine's own +version, and its pool is that version whole: the spine's geographic clones, +areas and sampling are the build's (microcosm#931). """ from __future__ import annotations import json -from collections.abc import Mapping from dataclasses import asdict, dataclass, field, replace from pathlib import Path @@ -45,158 +47,52 @@ uk_calibration_nodes, ) from .graph_evidence import SPINE_BUILD_STATE_TYPE -from .graph_kernels import UKClaimKernel, UKIdentityKernel, _normalize_create_frame +from .graph_frames import normalize_create_frame +from .graph_kernels import UKClaimKernel, UKIdentityKernel from .graph_population import ( UKPoolCheckpointKernel, - append_uk_population_nodes, population_columns, population_slices, register_uk_population_kernels, - uk_pool_validation_inputs, ) from .graph_targets import append_uk_target_nodes, register_uk_target_kernels -from .local_doctrine import UK_LOCAL_CLONE_COUNT -from .national_sampling import UK_SAMPLE_SEED_DEFAULT SPINE_PROVENANCE_TYPE = ArtifactType("microcosm.uk.bound-spine-provenance", 1) @dataclass(frozen=True) class UKFullBuildConfig: - """Three independent controls: geography target scope, pool K, export k.""" + """Two independent controls, geography target scope and export k, over the + bound spine. + + The spine binds the rest (microcosm#931): ``n_clones`` is its geographic + clone count K and ``source_sample_fraction`` its sampling rung, both read + from the spine the build binds, never chosen here. The full build samples, + expands and places nothing of its own. + """ calibration_year: int time_period: str = field(default_factory=lambda: load_uk_frs_release().time_period) source_year: int = field(default_factory=lambda: load_uk_frs_release().survey_year) geography_levels: tuple[str, ...] | None = None target_families: tuple[str, ...] | None = None - n_clones: int = UK_LOCAL_CLONE_COUNT - sample_fraction: float = 1.0 + n_clones: int = 1 source_sample_fraction: float = 1.0 - sample_seed: int = UK_SAMPLE_SEED_DEFAULT seed: int = 42 engine_blocks: int = 1 - constituency_vintage: str = "2024_pcon" - source_lineage_modulus: int | None = None calibration: UKGraphCalibrationConfig = UKGraphCalibrationConfig() - #: ``atomic`` keys every post-clone household and draws on the shared - #: atomic-geography operators; ``legacy`` keeps the sequential ladder draw - #: for measurement builds (a release candidate refuses it). - geography_assignment: str = "atomic" - #: FRS release vintage folded into every geography identity key. - source_vintage: str = field(default_factory=lambda: load_uk_frs_release().vintage) - #: ``price_paid`` reads each owner clone's home value off its assigned - #: constituency's uprated Land Registry sale distribution and its band off - #: its LSOA's VOA mix (microcosm#1240); ``none`` keeps the spine's values - #: and bands on every clone (measurement builds only; a release candidate - #: refuses it). Unset, the mode follows the geography assignment: - #: ``price_paid`` on the atomic assignment, ``none`` on the legacy one. - property_localisation: str | None = None - #: ``area`` reads each placed Great Britain household's BRMA off its - #: assigned area (microcosm#1262); ``none`` keeps the spine's regional - #: draw on every clone (measurement builds only; a release candidate - #: refuses it). Unset, the mode follows the geography assignment: ``area`` - #: on the atomic assignment, ``none`` on the legacy one. - brma_localisation: str | None = None - #: ``brma_list`` re-draws each Great Britain private renter's rent from its - #: assigned BRMA's list of rents at its FRS rank and region level - #: (microcosm#1262); ``none`` keeps the spine's rents (measurement builds - #: only; a release candidate refuses it). Unset, it follows the BRMA mode: - #: ``brma_list`` with area BRMAs, ``none`` otherwise. - private_rent_localisation: str | None = None - #: ``area`` sets each placed household's Rural Fuel Duty Relief flag off - #: its assigned output area (microcosm#1418); ``none`` leaves the engine's - #: default (measurement builds only; a release candidate refuses it). - #: Unset, the mode follows the geography assignment: ``area`` on the - #: atomic assignment, ``none`` on the legacy one. - rfdr_localisation: str | None = None def __post_init__(self) -> None: - from .graph_population import GEOGRAPHY_ASSIGNMENTS - from .national_sampling import validate_sample_fraction - - if self.geography_assignment not in GEOGRAPHY_ASSIGNMENTS: - raise ValueError( - f"Geography assignment must be one of {GEOGRAPHY_ASSIGNMENTS}." - ) - from .graph_population import PROPERTY_LOCALISATION_MODES - - if self.property_localisation is None: - object.__setattr__( - self, - "property_localisation", - "price_paid" if self.geography_assignment == "atomic" else "none", - ) - if self.property_localisation not in PROPERTY_LOCALISATION_MODES: - raise ValueError( - f"Property localisation must be one of {PROPERTY_LOCALISATION_MODES}." - ) - if ( - self.property_localisation != "none" - and self.geography_assignment != "atomic" - ): - raise ValueError( - "Property localisation needs the atomic geography assignment." - ) - from .graph_population import BRMA_LOCALISATION_MODES + from microcosm.build.frame_sampling import validate_sample_fraction - if self.brma_localisation is None: - object.__setattr__( - self, - "brma_localisation", - "area" if self.geography_assignment == "atomic" else "none", - ) - if self.brma_localisation not in BRMA_LOCALISATION_MODES: - raise ValueError( - f"BRMA localisation must be one of {BRMA_LOCALISATION_MODES}." - ) - if self.brma_localisation != "none" and self.geography_assignment != "atomic": - raise ValueError("Area BRMAs need the atomic geography assignment.") - from .graph_population import PRIVATE_RENT_LOCALISATION_MODES - - if self.private_rent_localisation is None: - object.__setattr__( - self, - "private_rent_localisation", - "brma_list" if self.brma_localisation == "area" else "none", - ) - if self.private_rent_localisation not in PRIVATE_RENT_LOCALISATION_MODES: - raise ValueError( - "Private rent localisation must be one of " - f"{PRIVATE_RENT_LOCALISATION_MODES}." - ) - if ( - self.private_rent_localisation != "none" - and self.brma_localisation != "area" - ): - raise ValueError("Private rents from the lists need the area BRMAs.") - from .graph_population import RFDR_LOCALISATION_MODES - - if self.rfdr_localisation is None: - object.__setattr__( - self, - "rfdr_localisation", - "area" if self.geography_assignment == "atomic" else "none", - ) - if self.rfdr_localisation not in RFDR_LOCALISATION_MODES: - raise ValueError( - f"RFDR localisation must be one of {RFDR_LOCALISATION_MODES}." - ) - if self.rfdr_localisation != "none" and self.geography_assignment != "atomic": - raise ValueError("Area RFDR flags need the atomic geography assignment.") - if not isinstance(self.source_vintage, str) or not self.source_vintage: - raise ValueError("The FRS source vintage must be a non-empty string.") - validate_sample_fraction(self.sample_fraction, label="UK full pool") validate_sample_fraction(self.source_sample_fraction, label="UK source spine") - if self.sample_fraction != 1.0 and self.source_sample_fraction != 1.0: + if type(self.n_clones) is not int or self.n_clones < 1: raise ValueError( - "A sampled spine cannot be sampled a second time in the full build." + "The spine's geographic clone count K must be a positive integer." ) - if type(self.n_clones) is not int or self.n_clones < 1: - raise ValueError("Geographic pool K must be a positive integer.") if self.engine_blocks not in {1, self.n_clones}: raise ValueError( - "Engine blocks must be one or equal the geographic pool K." + "Engine blocks must be one or equal the spine's geographic clone count K." ) if self.geography_levels is not None: if not self.geography_levels or set(self.geography_levels) - { @@ -211,11 +107,6 @@ def __post_init__(self) -> None: ) if len(set(self.geography_levels)) != len(self.geography_levels): raise ValueError("Geographic target levels must not repeat.") - if "brma" in self.geography_levels and self.brma_localisation != "area": - raise ValueError( - "BRMA targets need the area BRMAs: without them the rows " - "would bind on the spine's regional BRMA draw." - ) if self.target_families is not None and ( not self.target_families or any(not isinstance(f, str) or not f for f in self.target_families) @@ -231,7 +122,8 @@ def __post_init__(self) -> None: @property def effective_sample_fraction(self) -> float: - return self.sample_fraction * self.source_sample_fraction + """The spine's sampling rung: the dense line samples nothing more.""" + return self.source_sample_fraction @dataclass(frozen=True) @@ -266,22 +158,39 @@ def operation_inventory(self) -> dict: } +def _declared_spine_clone_count(graph: Graph) -> int | None: + """The clone count K a composed spine's ``geographic_expand`` node declares. + + ``None`` where the graph composes no such node, as a bound checkpoint does: + its K is read from its frame and build sidecar where it is bound. + """ + + from .geographic_expand import GEOGRAPHIC_EXPAND_STAGE_NAME + + for node in graph.nodes: + if node.id == GEOGRAPHIC_EXPAND_STAGE_NAME: + return int(node.params["geographic_clones"]) + return None + + def uk_full_graph( config: UKFullBuildConfig, *, spine: Graph | None = None, spine_population: str | None = None, - spine_weight_kind: str = "importance", optional_target_sources: tuple[str, ...] = (), checkpoint_identity: dict | None = None, review_date: str | None = None, - atomic_geography_definition: Mapping | None = None, ) -> UKFullGraph: """Append the full build to the existing source-owned UK spine graph. - ``atomic_geography_definition`` is the validated UK assignment declaration - (``uk_atomic_assignment_definition``) and is required when the config's - ``geography_assignment`` is ``atomic``. + The targets, measures and problem run on the spine's own version, and the + pool checkpoint keeps every row of it: the dense line takes the spine's + geographic clones, areas and sampling as they stand (microcosm#931). A + composed spine whose ``geographic_expand`` node declares another clone + count than the config's ``n_clones`` is refused, naming both K, and the + measures refuse a frame that does not hold ``n_clones`` clones, whatever + their engine blocks. """ initial = uk_spine_graph(source_mode="split") if spine is None else spine @@ -290,47 +199,32 @@ def uk_full_graph( if spine_population is None else spine_population ) - graph = append_uk_population_nodes( + declared = _declared_spine_clone_count(initial) + if declared is not None and declared != config.n_clones: + raise ValueError( + f"The full build is configured for K = {config.n_clones} geographic " + f"clones, but the spine it composes makes K = {declared}; the " + "dense line takes K from its spine (microcosm#931)." + ) + graph = append_uk_target_nodes( initial, population=endpoint, - time_period=config.time_period, - weight_kind=spine_weight_kind, - sample_fraction=config.sample_fraction, - sample_seed=config.sample_seed, - n_clones=config.n_clones, - seed=config.seed, - source_year=config.source_year, - constituency_vintage=config.constituency_vintage, - source_lineage_modulus=config.source_lineage_modulus, - geography_assignment=config.geography_assignment, - atomic_geography_definition=atomic_geography_definition, - source_vintage=config.source_vintage, - property_localisation_mode=config.property_localisation, - brma_localisation_mode=config.brma_localisation, - private_rent_localisation_mode=config.private_rent_localisation, - rfdr_localisation_mode=config.rfdr_localisation, - # A sampled rung's rent node may rank a region on Great Britain's FRS - # renters; a full-scale build refuses (microcosm#1305 review). - full_scale=config.sample_fraction == 1.0 - and config.source_sample_fraction == 1.0, - ) - graph = append_uk_target_nodes( - graph, calibration_year=config.calibration_year, time_period=config.time_period, geography_levels=config.geography_levels, target_families=config.target_families, engine_blocks=config.engine_blocks, + geographic_clones=config.n_clones, sample_fraction=config.effective_sample_fraction, target_weight_rule=config.calibration.target_weight_rule, optional_sources=optional_target_sources, review_date=review_date, - brma_rows=config.brma_localisation == "area", ) - cells = population_columns(graph, "uk.full.expand") - # This structural checkpoint depends on every pool operation, including - # evidence-only gates and the contribution problem. It binds the complete - # incoming ledger before downstream nodes reconstruct any Frame slices. + cells = population_columns(graph, endpoint) + # This structural checkpoint depends on every operation on the spine's + # version, the evidence-only gates and the contribution problem among + # them. It binds the complete incoming ledger before downstream nodes + # reconstruct any Frame slices, and keeps every row. graph = replace( graph, nodes=( @@ -339,12 +233,9 @@ def uk_full_graph( "uk.full.pool", UKPoolCheckpointKernel.ref, structural=StructuralDelta.FILTER, - base="uk.full.expand", + base=endpoint, inputs=population_slices(cells), - artifact_inputs=uk_pool_validation_inputs( - config.geography_assignment, config.property_localisation - ), - description="Checkpoint the complete geographic pool and selected-target problem.", + description="Checkpoint the bound spine and the selected-target problem.", ), ), ) @@ -388,8 +279,48 @@ class UKBoundSpineKernel(KernelBase): ) def implementation_hash(self) -> str: + from microcosm.data import gate_signing_keys + from microcosm.frame import materialize as frame_materialize + from microcosm.frame.adapters import policyengine_uk as uk_engine_adapter + + from .. import gate_battery, gates, trace + from . import ( + content_identity, + gate_scopes, + geographic_clones, + graph_frames, + hmrc_uprating, + ) + + # The node also computes through the H5 it loads (frame_materialize, + # the engine adapter's contracts), the checkpoint's content identity + # and pinned engine (content_identity, hmrc_uprating), the frame's + # geographic clone count it holds the sidecar's to (geographic_clones), + # the gate digests it derives through a battery run (gate_battery, + # gates, trace) over the spine gate scope and its scoped manifest + # (gate_scopes), the create-frame normalisation it shares with the + # spine's root (graph_frames, not the spine's kernel module, so an + # edit there does not re-key the national line) and the canonical + # encoding of its artifacts (microcosm#931), and the committed + # registry of gate signing keys the checkpoint's signed reports are + # held to (microcosm.data.gate_signing_keys). return source_hash( - type(self), national_frame, calibration_run, release_certification + type(self), + national_frame, + calibration_run, + gate_signing_keys, + gate_scopes, + geographic_clones, + release_certification, + frame_materialize, + uk_engine_adapter, + content_identity, + hmrc_uprating, + gate_battery, + gates, + trace, + graph_frames, + canonical_json, ) def run(self, context: KernelContext) -> KernelResult: @@ -408,7 +339,7 @@ def run(self, context: KernelContext) -> KernelResult: provenance = calibration_run.strict_spine_provenance_from_sidecar( sidecar_path, sidecar, gate_report_path=context.sources["uk_spine_gates"] ) - created = _normalize_create_frame(frame, context) + created = normalize_create_frame(frame, context) return KernelResult( frame=created, artifacts={ diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_calibration.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_calibration.py index c188dc79b4..7387afa408 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_calibration.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_calibration.py @@ -265,26 +265,68 @@ def restore_uk_graph_result( return replace(result, frame=final_frame) +def _size_solve_modules() -> tuple[object, ...]: + """What the size search and refit compute through beyond the shared hash: + the target weights, with the doctrine, local target areas and UK-wide + geography their solves read (microcosm#931).""" + + import microcosm.calibrate.geography_constants as geography_constants + + from . import local_doctrine, local_targets, national_doctrine, target_weights + + return ( + target_weights, + local_doctrine, + local_targets, + national_doctrine, + geography_constants, + ) + + +def _calibration_hash(kernel: KernelBase, *modules: object) -> str: + """The implementation hash every solve kernel shares, over ``modules`` too. + + Every solve kernel computes through the frame it rebuilds on + national_frame's schema, the targets' hierarchy and monetary checks in + microcosm.calibrate and the canonical encoding of its artifacts. The size + search and refit add what :func:`_size_solve_modules` names, which the + dense solve and the other kernels sharing this hash do not compute + through (microcosm#931). test_uk_graph derives each kernel's set. + """ + + import microcosm.calibrate.hierarchy as calibrate_hierarchy + import microcosm.calibrate.monetary_binding as monetary_binding + + from . import national_frame + + return hashlib.sha256( + canonical_json( + { + "solver": CalibrateAdamKernel().implementation_hash(), + "adapter": source_hash( + type(kernel), + dataset_size, + size_checkpoint, + calibration_artifacts, + context_frame, + select_exact_k, + HardConcrete, + contribution_initialization, + national_frame, + calibrate_hierarchy, + monetary_binding, + canonical_json, + *modules, + dependencies=kernel.capabilities.dependencies, + ), + } + ) + ).hexdigest() + + class _CalibrationKernel(KernelBase): def implementation_hash(self): - return hashlib.sha256( - canonical_json( - { - "solver": CalibrateAdamKernel().implementation_hash(), - "adapter": source_hash( - type(self), - dataset_size, - size_checkpoint, - calibration_artifacts, - context_frame, - select_exact_k, - HardConcrete, - contribution_initialization, - dependencies=self.capabilities.dependencies, - ), - } - ) - ).hexdigest() + return _calibration_hash(self) class UKSizeCheckpointImportKernel(_CalibrationKernel): @@ -564,6 +606,9 @@ class UKSizeSearchKernel(_ObservedSolveKernel): dependencies=_DEPENDENCIES, ) + def implementation_hash(self): + return _calibration_hash(self, *_size_solve_modules()) + def run(self, context): frame, problem = _inputs(context) dense = _dense(context, frame, problem) @@ -677,6 +722,9 @@ class UKSizeRefitKernel(_ObservedSolveKernel): dependencies=_DEPENDENCIES, ) + def implementation_hash(self): + return _calibration_hash(self, *_size_solve_modules()) + def run(self, context): frame, problem = _inputs(context) dense = _dense(context, frame, problem) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py index 8c0b3b00dd..4cbb788981 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_evidence.py @@ -41,10 +41,10 @@ decode_stage_evidence, encode_stage_evidence, ) -from . import battery_bindings, energy_pricing, stage_health -from .calibration_run import UK_SPINE_GATE_SCOPE, uk_scoped_gate_manifest +from . import battery_bindings, stage_health from .cgt_asset_type import CGT_ASSET_TYPE_DOMAIN from .frs_relationships import CHRONICLE_ONS_HOUSEHOLD_TYPE_VALUE_IDS +from .gate_scopes import UK_SPINE_GATE_SCOPE, uk_scoped_gate_manifest from .graph import UK_SPINE_ASSEMBLED_FINAL_STAGE #: The spine checkpoint's build state the terminal gates read as the spine @@ -227,6 +227,7 @@ def add_uk_spine_gate_nodes( data-only fixture is not the population its facts describe, so population-fact gates never block such a run. """ + from .gate_phase_codes import UK_SPINE_GATE_PHASE_CODE from .graph import uk_spine_endpoint gates = uk_spine_gate_manifest(spec) @@ -260,6 +261,7 @@ def add_uk_spine_gate_nodes( for phase, population, inputs, phase_stages in checkpoints: if phase == "transferred" and end == len(stages): continue + code = UK_SPINE_GATE_PHASE_CODE.get(phase) nodes.append( Node( id=f"spine.gates.{phase}", @@ -294,11 +296,7 @@ def add_uk_spine_gate_nodes( "phase": phase, "time_period": "2024", "stage_names": phase_stages, - **( - {"energy_pricing_code": source_hash(energy_pricing)} - if phase == "transferred" - else {} - ), + **({f"{phase}_code": code()} if code is not None else {}), "gate_manifest": json.dumps( gate_battery._gates_manifest_payload(gates), sort_keys=True ), @@ -350,9 +348,22 @@ class UKSpineGateKernel(KernelBase): the persisted verdict binds the policy it was judged under: ``synthetic_smoke`` mirrors ``GateBatteryRun(synthetic_smoke=...)`` — on a data-only fixture the population-fact gates are recorded but never block. - The transferred phase also carries ``energy_pricing_code``, the hash of - the module that sets the energy level its rake gate recomputes, so only - that phase re-keys when the level's code changes. + Each phase also carries ``_code``, the hash of the code only its + own gates compute through + (:data:`.gate_phase_codes.UK_SPINE_GATE_PHASE_CODE`), so the assembled + gate's key, on which every later spine stage's chains, carries none of the + transferred phase's (the re-review of microcosm#1420, E; microcosm#931 + ruling 6). An edit to a module only the transferred phase computes through + still re-keys the assembled gate where a stage before it hashes the + module, as for ``stochastic_assignment`` and ``take_up_contract``. The + kernel refuses a stale or missing ``_code`` before it evaluates + anything, as the full gates do + (:func:`.gate_phase_codes.require_current_phase_code`; microcosm#931, + ruled 9 Nov (late)), so a spine gate node whose ``_code`` is stale + or missing is refused whenever it runs, and no spine gate evaluates its + phase under a key that names other phase code. A record stored under the + node's key is reused without running the kernel, so the check does not + guard a cache hit. """ ref = "uk.spine-gates@1" @@ -369,24 +380,58 @@ def __init__(self, *, gates: GatesManifest, engine: object, engine_identity: str self.engine_identity = engine_identity def implementation_hash(self) -> str: + import microcosm.calibrate.geography_constants as geography_constants + from microcosm.frame.adapters import policyengine_uk as uk_engine_adapter from microcosm.graph.canonical import canonical_json + from .. import gates, stage_evidence + from . import ( + council_tax_constants, + graph_kernels, + national_frame, + receipt_counts, + ) from .solve_scope import uk_solve_fingerprint # The spine stages after a gate wait on its verdict, so its key is # part of theirs. No spine gate reads a gate-only register, so it keys # on the solve fingerprint and a sign-off between two release builds - # leaves an in-graph spine a cache hit (microcosm#1301). The stage - # checks live in stage_health, so it keys the gate too (the independent - # review of microcosm#1420, finding 5). Only the transferred phase reads - # the energy level, through energy_pricing, so that phase alone carries - # its hash, as a node param: an energy change re-keys it, not the - # assembled gate every later spine stage waits on (the re-review, E). + # leaves an in-graph spine a cache hit (microcosm#1301). The code is + # what both phases compute through (microcosm#931 ruling 6): this + # kernel's run, the frame it rebuilds (graph_kernels' _minimal_frame, + # national_frame) and the evidence it decodes (stage_evidence); the + # battery and the gate registry (gate_battery, battery_bindings, + # gates); the stage-health checks (stage_health, the independent + # review of microcosm#1420, finding 5), with the receipt bands of the + # take-up checks both phases declare (receipt_counts); the enum + # gate's nation partition (council_tax_constants, the region tier); + # and the engine the kernel is built with, through which each phase's + # enum gates resolve a domain no artifact supplies (the engine + # adapter's PolicyEngineUKEngine.enum_domain). What only one phase's + # gates compute through is that phase's own code, a node param that + # run checks (UK_SPINE_GATE_PHASE_CODE, in gate_phase_codes, which this + # hash leaves out): the energy level, for one, re-keys the transferred + # gate and not the assembled gate every later spine stage waits on, + # since no stage before that gate hashes energy_pricing (the + # re-review, E). A phase-only module a stage before the assembled gate + # hashes still re-keys it by chaining + # (gate_phase_codes.uk_spine_transferred_only_code_hash). return hashlib.sha256( canonical_json( { "code": source_hash( - type(self), gate_battery, battery_bindings, stage_health + type(self), + gate_battery, + battery_bindings, + stage_health, + gates, + stage_evidence, + graph_kernels, + national_frame, + receipt_counts, + council_tax_constants, + geography_constants, + uk_engine_adapter, ), "country_resources": uk_solve_fingerprint(), } @@ -396,8 +441,10 @@ def implementation_hash(self) -> str: def run(self, context: KernelContext) -> KernelResult: from microcosm.frame import Frame, MassChangeRecord + from .gate_phase_codes import require_current_phase_code from .graph_kernels import _minimal_frame + phase = str(context.params["phase"]) expected = json.dumps( gate_battery._gates_manifest_payload(self.gates), sort_keys=True ) @@ -408,6 +455,11 @@ def run(self, context: KernelContext) -> KernelResult: raise ValueError( "Spine gate binding differs from its declared manifest or engine." ) + # The phase's own code keys the node through its parameter, not this + # kernel's hash, so a node whose parameter is stale or missing is + # refused before anything is read or evaluated (microcosm#931, ruled + # 9 Nov (late)), as the full gates refuse theirs. + require_current_phase_code(context.params, phase, battery="spine") release_candidate = bool(context.params["release_candidate"]) synthetic_smoke = bool(context.params["synthetic_smoke"]) previous = context.artifacts.get("previous_gate") @@ -420,7 +472,7 @@ def run(self, context: KernelContext) -> KernelResult: synthetic_smoke=synthetic_smoke, ): report = gate_battery.GatePhaseReport( - phase=str(context.params["phase"]), + phase=phase, outcomes=tuple( gate_battery.GateOutcome( entry=entry, @@ -428,7 +480,7 @@ def run(self, context: KernelContext) -> KernelResult: reason="The assembled spine gate blocked this phase.", ) for entry in self.gates.gates - if entry.phase == context.params["phase"] + if entry.phase == phase ), ) return KernelResult( @@ -464,7 +516,7 @@ def run(self, context: KernelContext) -> KernelResult: ) report = evaluate_phase( self.gates, - str(context.params["phase"]), + phase, EvidenceContext( frame=frame, artifacts={**evidence, **uk_spine_gate_artifacts(self.engine)}, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_frames.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_frames.py new file mode 100644 index 0000000000..9f97750b08 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_frames.py @@ -0,0 +1,53 @@ +"""The owned cells and created frames the UK graph kernels hand the executor. + +A kernel returns each cell its node owns at the dtype the node declares for it, +and a root kernel returns its created frame with those cells cast and the build +period stamped. The spine's create, claim and stage kernels and the national +line's bound spine root share these helpers, so each hashes this module rather +than the module of another kernel family (microcosm#931). +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pandas as pd + +from microcosm.frame import Frame +from microcosm.graph.population import dtype_for_token + +if TYPE_CHECKING: + from microcosm.graph import KernelContext + +__all__ = ["cast_owned", "normalize_create_frame"] + + +def cast_owned(series: pd.Series, dtype: str) -> pd.Series: + """``series`` at the dtype its node declares for it (the token ``dtype``).""" + + target = dtype_for_token(dtype) + if series.dtype != target: + series = series.astype(target) + return series + + +def normalize_create_frame(frame: Frame, context: KernelContext) -> Frame: + """A created frame with its node's owned cells at their declared dtypes + and the build period of the node's ``time_period`` parameter.""" + + tables = {entity: frame.table(entity).copy(deep=True) for entity in frame.entities} + for owned in context.node.outputs: + tables[owned.entity][owned.column] = cast_owned( + tables[owned.entity][owned.column], owned.dtype + ).array + return Frame( + tables, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + frame.strata, + mass_log=frame.mass_log, + metadata={ + **frame.metadata, + "time_period": str(context.params.get("time_period", "2024")), + }, + ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py index c656e6fc37..63e3f99cb5 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_kernels.py @@ -26,11 +26,20 @@ import numpy as np import pandas as pd +from microcosm.calibrate import geography_constants from microcosm.fit import categorical as fit_categorical from microcosm.fit import donors as fit_donors from microcosm.fit import held_out as fit_held_out +from microcosm.fit import model as fit_model +from microcosm.fit import qrf as fit_qrf from microcosm.frame import Frame, MassChangeRecord, WeightKind +from microcosm.frame import materialize as frame_materialize +from microcosm.frame import rules as frame_rules +from microcosm.frame.adapters import engine_identifiers, engine_population from microcosm.frame.adapters import policyengine_uk as uk_engine_adapter +from microcosm.frame.adapters import ( + policyengine_uk_population as uk_engine_population_declaration, +) from microcosm.graph import ( Capabilities, Determinism, @@ -45,45 +54,116 @@ load_source, source_hash, ) +from microcosm.graph import canonical as graph_canonical +from microcosm.graph import randomness as graph_randomness from microcosm.graph.population import dtype_for_token -from .. import source_lineage, stage_evidence, stochastic_assignment +from .. import ( + atomic_geography, + atomic_support_conditioning, + atomic_support_expectations, + bounded_amounts, + chronicle_epoch, + clone_expansion, + gates, + ledger_targets, + raking, + source_lineage, + stage_evidence, + stochastic_assignment, + target_materialization, + trace, +) from . import ( + advani_summers, + annuity_trust_column, + area_home_values, + atomic_area_conditioning, + atomic_area_support, + atomic_household_identity, + birth_month, + brma_area_layer, + bus_fare_pricing, + bus_support_pricing, bus_use_incidence, + carers_allowance_take_up, + cgt_asset_type, cgt_imputation, + cgt_structure, + cgt_support, + chronicle_feed, + copy_id_table, + council_tax_band_d, + council_tax_constants, + dfe_spring_term, + donor_uprating, + employer_pension_contributions, energy_pricing, + etb_vat, + frs_annual_income, + frs_brma, + frs_council_tax, frs_disability, frs_education, frs_hmrc_source, + frs_release, frs_spine, frs_take_up, + gambling_participation, + geographic_clones, + geographic_identity, + geography_first_roster, + geography_sources, + graph_frames, + hmrc_capital_gains, hmrc_company_car_statistics, hmrc_income, hmrc_isa_statistics, hmrc_property_rental, + hmrc_replay, + hmrc_source_contract, hmrc_uprating, + household_income, household_lineage, ledger_fact_vendoring, legacy_closure_take_up, + local_authority_input, + national_frame, + nts_car_attributes, occupation_constants, + pension_coherence, + private_rent_localisation, property_localisation, + qrf_gates, receipt_counts, + regional_uprating, + release_input_coverage, + rfdr_area_layer, rowwise_geography, + spi_band_donors, + spi_housing_council_tax, spi_income, spi_rank_transport, spi_support, + stage_health, + support_clip, + take_up_contract, + tenure_constants, + terminal_gates, uc_capital_coherence, uc_deduction_attributes, uc_relationships, was_wealth, ) +from .graph import UK_SPINE_STRUCTURAL_STAGES +from .graph_frames import cast_owned, normalize_create_frame from .national_frame import UK_NATIONAL_SCHEMA -from .rowwise_geography import id_multiplier_for_values if TYPE_CHECKING: from microcosm.build.source_manifest import SourceStageSpec __all__ = [ + "FIXTURE_SUPPORT_INPUTS", "UKClaimKernel", "UKCreateKernel", "UKExpandStageKernel", @@ -94,6 +174,12 @@ ] +#: The H2 fixture's inputs that hold its toy conditioned atomic-area supports, +#: by system: each is named by the id sources.yaml declares the system's +#: conditioned support under, as the production graph's raw-byte source is +#: (microcosm#931). +FIXTURE_SUPPORT_INPUTS = MappingProxyType(dict(atomic_area_support.CONDITIONED_SOURCES)) + _STAGE_MODULES = { "frs_spine": "frs_spine", "frs_relationships": "frs_relationships", @@ -106,7 +192,16 @@ "frs_take_up": "frs_take_up", "frs_person_draws": "frs_person_draws", "frs_household_draws": "frs_household_draws", - "frs_brma": "frs_brma", + # The area stages' transforms live in spine_geography, the private rents' + # in spine_private_rent and the council-tax localisation's in + # spine_council_tax (microcosm#931). + "atomic_area_assign": "spine_geography", + "local_authority": "spine_geography", + "rfdr": "spine_geography", + "area_brma": "spine_geography", + "spi_copy_areas": "spine_geography", + "private_rent": "spine_private_rent", + "council_tax_localise": "spine_council_tax", "was_wealth": "was_wealth", "was_lisa": "was_lisa", "nts_bus_travel": "nts_bus_travel", @@ -118,6 +213,8 @@ "etb_services": "etb_services", "frs_hmrc_spine_leaves": "spi_spine", "frs_annual_income_spells": "frs_annual_income", + "geographic_expand": "geographic_expand", + "geographic_identity": "geographic_identity", "spi_support_channel": "spi_spine", "spi_income_band_donors": "spi_band_donors", "hmrc_spi_income_spine": "spi_spine", @@ -135,7 +232,6 @@ "carers_allowance_take_up": "carers_allowance_take_up", "uc_deduction_attributes": "uc_deduction_attributes", "high_value_home_support": "high_value_home_support", - "regional_home_values": "regional_home_values", "cgt_support_split": "cgt_support", "cgt_incidence_clone": "cgt_structure", "cgt_incidence_anchor": "cgt_structure", @@ -153,111 +249,759 @@ "age_tail": "age_tail", } -# Imported modules are not traversed by ``source_hash``. Bind relationship -# helpers and the adapter's input-retention checks into every consuming stage. +# What every stage that calls the engine computes through: the adapter, which +# runs a call whose frame carries several clones one clone at a time, and the +# shared engine-block machinery with the UK declaration it judges each batch +# against (microcosm#931). An edit to any of them moves the hash of every such +# stage; test_uk_graph derives the stages from their engine calls and from +# their transforms' engine parameter. Every stage also hashes the adapter +# beside national_frame (_implementation_hash). +_ENGINE_MODULES = ( + uk_engine_adapter, + engine_population, + uk_engine_population_declaration, +) +# What else the injected engine computes through when those stages call it: +# the adapter builds each simulation's dataset through frame.materialize's +# engine_tables, and the ids that simulation holds through engine_identifiers. +# test_uk_graph derives the group from a second walk, rooted at the +# PolicyEngineUKEngine the builds inject and followed from the methods the +# engine stages call on it (microcosm#931). It is kept apart from +# _ENGINE_MODULES, which no other stage binds, because another stage may bind +# one of its modules for its own tables, which test_uk_graph allows only where +# the stage's own transform computes through it: the SPI spine's three stages +# and the band donors bind frame.materialize. No stage reaches +# engine_identifiers except through the engine. +_ENGINE_COMPUTE_MODULES = (frame_materialize, engine_identifiers) +# What every stage that calls the engine binds. +_ENGINE_STAGE_MODULES = (*_ENGINE_MODULES, *_ENGINE_COMPUTE_MODULES) + +# Imported modules are not traversed by ``source_hash``, so each stage binds +# every first-party module holding code its transform computes through, +# beyond its own module (microcosm#931). test_uk_graph derives that set: it +# follows what the transform names into the module defining it, in any +# workspace package, to a fixed point, and for a stage that calls the engine, +# what the methods the stage calls on the injected engine name. It needs no +# binding for a listed refusal-only check, nor for the core modules every node +# runs through (the Frame, the graph executor and the modules that compute +# node keys), which it lists with their reasons. Where it stops at a single +# string, a column-name function or a record, the module is hashed by the +# stage or a stage upstream of it, or the name is listed with its reason; a +# collection of strings, like a number, is a value. +# Every stage whose module or bound helpers import the shared keyed-draw +# helpers (``microcosm.build.stochastic_assignment``), or import another +# module's function that draws through them, binds them as well, so an edit +# to how a uniform is keyed or a binary is placed re-keys every stage it +# moves; test_uk_graph derives that set from the imports and names. + +# A vendored ledger fact is read through ledger_fact_vendoring, which checks +# it against the pinned chronicle feed and the feed's accepted schema epochs. +_FEED_FACT_MODULES = (ledger_fact_vendoring, chronicle_feed, chronicle_epoch) +# A pinned source tab is read through frs_spine's tab reader, which checks the +# tab's digest through trace. +_PINNED_TAB_MODULES = (frs_spine, trace) +# The QRF donor stages fit and draw through microcosm.fit's regime-gated QRF, +# which resolves its fit columns and weights through fit.model. The +# categorical fitters resolve theirs through fit.model too, and their held-out +# scores and folds use the QRF's gate calibration and folds (microcosm#931). +_QRF_MODULES = (fit_qrf, fit_model) + _STAGE_HELPER_MODULES = { # The three FRS draw stages' receipts measure their take-up flags through # frs_take_up's signal measurement and count support by the shared - # disclosure rules (microcosm#1095). - "frs_take_up": (receipt_counts,), - "frs_person_draws": (frs_take_up, receipt_counts), - "frs_household_draws": (frs_take_up, receipt_counts), - "frs_hmrc_spine_leaves": (frs_hmrc_source,), + # disclosure rules (microcosm#1095). The flags draw on the take-up + # contract's rates and age children by the DfE's spring-term rule + # (microcosm#931). + "frs_take_up": ( + dfe_spring_term, + receipt_counts, + take_up_contract, + stochastic_assignment, + ), + "frs_person_draws": ( + dfe_spring_term, + frs_take_up, + receipt_counts, + take_up_contract, + stochastic_assignment, + ), + "frs_household_draws": ( + dfe_spring_term, + frs_take_up, + receipt_counts, + take_up_contract, + stochastic_assignment, + ), + # The FRS stages that read raw tabs and normalise their ids through + # frs_spine; the employment stage maps occupations to the shared codes, and + # the disability flags resolve the benefit year's rules from the FRS + # release and annualise on frs_spine's week count (microcosm#931). The + # council tax stage levels Scotland's bills on CTAS's Band D, which it + # reads as vendored facts (microcosm#1095, S2(a)), and the disability + # stage's carer link places its outside carers through Carer's + # Allowance's balanced positions (microcosm#1095, A3). + "frs_relationships": (*_PINNED_TAB_MODULES, stochastic_assignment), + "frs_employment": ( + *_PINNED_TAB_MODULES, + occupation_constants, + stochastic_assignment, + ), + "frs_council_tax": ( + *_PINNED_TAB_MODULES, + *_FEED_FACT_MODULES, + stochastic_assignment, + ), + "frs_disability": ( + carers_allowance_take_up, + frs_release, + frs_spine, + stochastic_assignment, + ), + "frs_education": (*_PINNED_TAB_MODULES, stochastic_assignment), + # The leaves' employer pension contributions are drawn on keyed uniforms + # through employer_pension_contributions, from the ASHE rates it reads as + # vendored facts; the weekly leaves annualise on frs_spine's week count + # (microcosm#931). Their module, spi_spine, also mints the support + # channel's copies, so the leaves and the SPI income stage bind the copy-id + # table as the support channel does. Its three stages, like the band + # donors, copy their frames' tables through microcosm.frame's + # engine_tables (microcosm#931). + "frs_hmrc_spine_leaves": ( + frs_hmrc_source, + copy_id_table, + employer_pension_contributions, + *_FEED_FACT_MODULES, + frs_spine, + frame_materialize, + stochastic_assignment, + ), # The annual income descriptors' receipt follows the shared disclosure rules - # (microcosm#1164). - "frs_annual_income_spells": (receipt_counts,), + # (microcosm#1164); the spells read their tabs and interview dates through + # frs_spine (microcosm#931). + "frs_annual_income_spells": ( + receipt_counts, + *_PINNED_TAB_MODULES, + stochastic_assignment, + ), + # The geographic clones are the shared clone expansion core's, under the + # UK's declaration, whose ids are minted in the copy-id table's + # geographic_support field (microcosm#931). + # Every geography-first stage's transform reads its stage's name from the + # names-only roster leaf (microcosm#931). + "geographic_expand": (clone_expansion, copy_id_table, geography_first_roster), + # The geography key is household_draw_key, encoded canonically, over the + # FRS vintage the release pins for each source year (microcosm#931). + "geographic_identity": ( + atomic_household_identity, + frs_release, + graph_canonical, + geography_first_roster, + ), + # The area draw runs the shared atomic-area operator and the shared + # fallback-level chooser on the conditioned supports, through + # atomic_area_conditioning's definition and draw inputs, which read the + # support profile, the band order and the FRS region map; the operator + # encodes its definition canonically and keys its draws through + # microcosm.graph's uniforms. The fuel-relief flag decodes the same + # supports for its roster and reads the RFDR layer; the local authority + # reads the names roster (microcosm#931). Their module, spine_geography, + # imports the BRMA stages' Northern Ireland draw, which draws through the + # keyed-draw helpers, so every stage it holds binds them. + "atomic_area_assign": ( + atomic_geography, + atomic_support_conditioning, + atomic_area_conditioning, + atomic_area_support, + council_tax_constants, + rowwise_geography, + geography_first_roster, + graph_canonical, + graph_randomness, + stochastic_assignment, + ), + "local_authority": ( + local_authority_input, + geography_sources, + geography_first_roster, + stochastic_assignment, + ), + "rfdr": ( + atomic_geography, + atomic_support_conditioning, + atomic_area_conditioning, + atomic_area_support, + rfdr_area_layer, + rowwise_geography, + geography_first_roster, + graph_canonical, + stochastic_assignment, + ), + # The SPI copies' areas re-key each copy as the geography identity does, + # draw its area with the area draw's definition, and re-run the local + # authority, the BRMA (Northern Ireland's on the copies' own benefit units + # after one engine call) and the fuel-relief flag (microcosm#931). + "spi_copy_areas": ( + *_ENGINE_STAGE_MODULES, + atomic_geography, + atomic_support_conditioning, + atomic_area_conditioning, + atomic_area_support, + atomic_household_identity, + brma_area_layer, + council_tax_constants, + frs_brma, + frs_release, + geographic_identity, + local_authority_input, + geography_sources, + rfdr_area_layer, + rowwise_geography, + geography_first_roster, + graph_canonical, + graph_randomness, + stochastic_assignment, + ), + # The private rents read the BRMA lists through the localiser and key their + # tie-breaks through microcosm.graph's uniforms, which encode each draw's + # coordinates as canonical JSON (microcosm#931). + "private_rent": ( + private_rent_localisation, + geography_first_roster, + graph_canonical, + graph_randomness, + ), + # The council-tax localisation recomputes each household's draw cell as + # the area draw does, through atomic_area_conditioning's definition, draw + # inputs and levels on the decoded supports, which read the support + # profile, the band order and the FRS region map; the definition is + # encoded canonically and names microcosm.graph's stream. It takes each + # cell's expected Band D through the shared draw-cell expectation, reads + # the Band D and prices the statute through council_tax_band_d, + # Scotland's Band D a vendored fact checked against the pinned feed, reads + # stage 5's bill provenance, the authority code column and the nations of + # the regions, and holds the reductions within their bills through the + # shared bounded amounts. Like every geography-first stage, it reads its + # name from the roster leaf (microcosm#931). + "council_tax_localise": ( + atomic_geography, + atomic_support_conditioning, + atomic_support_expectations, + atomic_area_conditioning, + atomic_area_support, + bounded_amounts, + council_tax_band_d, + council_tax_constants, + frs_council_tax, + local_authority_input, + rowwise_geography, + geography_constants, + *_FEED_FACT_MODULES, + geography_first_roster, + graph_canonical, + graph_randomness, + ), # The SPI income stage's imputation, band-donor resample and receipt live # in spi_income, so an edit there moves the stage's hash (microcosm#1278), - # and its twin transport in spi_rank_transport (microcosm#840). - "hmrc_spi_income_spine": (spi_income, spi_rank_transport), - "frs_spine": (uc_relationships,), - "frs_legacy_proxies": (uk_engine_adapter,), - "frs_education_grant_split": (uk_engine_adapter,), - "frs_brma": (uk_engine_adapter,), - "was_wealth": (uk_engine_adapter,), + # and its twin transport in spi_rank_transport (microcosm#840). The + # imputation refreshes the disability flags, coheres the pension + # contributions on the ASHE rate bands, uprates savings income, floors pay + # on frs_spine's week count and reads its age model and band targets as + # vendored facts and HMRC tables; the stage's parameters bound the band + # donors and the coverage floor, and its result carries the fits' weight + # kinds, gate outcomes and the HMRC replay report (microcosm#931). It + # binds the copy-id table as the leaves do. The imputation refreshes the + # annual income descriptors on frs_annual_income's share columns, and the + # evidence shares iterate HMRC's distributional inputs. It zeros a drawn + # row's annuity and trust descriptor, whose name it reads from + # annuity_trust_column, as the root does (microcosm#931, ruled 9 Nov + # (late)). It takes the carer link again on the copies by frs_disability's + # rule, whose outside carers are placed through Carer's Allowance's + # balanced positions (microcosm#1095, A3). + "hmrc_spi_income_spine": ( + spi_income, + spi_rank_transport, + annuity_trust_column, + carers_allowance_take_up, + copy_id_table, + employer_pension_contributions, + *_FEED_FACT_MODULES, + frs_annual_income, + frs_disability, + frs_release, + frs_spine, + frs_take_up, + gates, + hmrc_income, + hmrc_replay, + hmrc_source_contract, + hmrc_uprating, + pension_coherence, + qrf_gates, + release_input_coverage, + spi_band_donors, + spi_support, + target_materialization, + frame_materialize, + stochastic_assignment, + ), + # The spine's raw ids sit below the copy-id table's first field, and every + # copy layer mints its ids in its own field of the table; the two SPI + # support stages mint through the support channel's clone helper, and the + # high-value split weighs its copies through the CGT split's exact-total + # correction (microcosm#931). The executor reads each id-offset stage's + # declared radix from the table too. The root reads its tabs through + # trace's digest, ages people by birth month, caps pension contributions + # with their receipt, and codes council tax bands in the shared band order + # (microcosm#931). It writes the annuity and trust descriptor under the + # name annuity_trust_column holds (microcosm#931, ruled 9 Nov (late)). It + # reads each household's nation, for its IMD decile and its band flag, + # from the region tier through council_tax_constants (microcosm#931). The + # LCWRA proxy reads the reported disability benefits frs_disability lists + # (microcosm#1095, Y1). + "frs_spine": ( + uc_relationships, + annuity_trust_column, + copy_id_table, + birth_month, + council_tax_constants, + frs_disability, + geography_constants, + pension_coherence, + receipt_counts, + trace, + stochastic_assignment, + ), + # The support channel draws its keyed share and bounds its zero-weight + # gate through the keyed-draw helpers, reads the take-up population's ages + # and gates its zero-weight strata (microcosm#931). + "spi_support_channel": ( + spi_support, + copy_id_table, + frs_take_up, + gates, + terminal_gates, + frame_materialize, + stochastic_assignment, + ), + # The band donors read HMRC's ITL band taxpayers as vendored facts and the + # donors' age model through spi_income (microcosm#931). + "spi_income_band_donors": ( + spi_support, + copy_id_table, + *_FEED_FACT_MODULES, + frs_take_up, + spi_income, + frame_materialize, + stochastic_assignment, + ), + # The exact-total correction the CGT split hands the high-value split is + # spi_support's, and the support stratum is property_localisation's, whose + # owners are the tenure map's (microcosm#931). spi_support draws the SPI + # share through the keyed-draw helpers, so the stages that bind it bind + # them too. + "high_value_home_support": ( + cgt_support, + copy_id_table, + property_localisation, + spi_support, + tenure_constants, + stochastic_assignment, + ), + # The CGT stages read HMRC's CGT tables as vendored facts and the + # Advani-Summers tail from advani_summers, take the CGT policy parameters + # and income proxy from cgt_imputation (whose loss relief evaluates a named + # term), and weigh their copies through spi_support's exact-total + # correction; the support split also takes the adult age and oldest-adult + # rule from cgt_structure and reads the frame's geographic clone count + # through geographic_clones, and the residential split takes its logistic + # from cgt_asset_type; the incidence clone sums cgt_imputation's income proxy + # components into the income behind its propensity; the gains spine names + # HMRC's regional conditioning rows by microcosm.calibrate's UK region tier, + # and it and the asset types key each walk's offset on its label through + # the shared keyed draws (microcosm#931). + "cgt_support_split": ( + copy_id_table, + *_FEED_FACT_MODULES, + cgt_imputation, + cgt_structure, + geographic_clones, + hmrc_capital_gains, + hmrc_income, + spi_support, + target_materialization, + stochastic_assignment, + ), + "cgt_incidence_clone": ( + rowwise_geography, + copy_id_table, + advani_summers, + cgt_imputation, + spi_support, + stochastic_assignment, + ), + "cgt_incidence_anchor": ( + rowwise_geography, + copy_id_table, + advani_summers, + cgt_imputation, + hmrc_capital_gains, + stochastic_assignment, + ), + "cgt_residential_split": ( + copy_id_table, + *_FEED_FACT_MODULES, + cgt_asset_type, + cgt_imputation, + hmrc_capital_gains, + spi_support, + stochastic_assignment, + ), + "hmrc_cgt_gains_spine": ( + *_FEED_FACT_MODULES, + advani_summers, + geography_constants, + hmrc_capital_gains, + hmrc_income, + target_materialization, + stochastic_assignment, + ), + "hmrc_cgt_asset_type_spine": ( + *_FEED_FACT_MODULES, + cgt_imputation, + hmrc_capital_gains, + stochastic_assignment, + ), + # The legacy proxies read raw tabs, and the grant split resolves its + # year's rules from the FRS release (microcosm#931). + "frs_legacy_proxies": ( + *_ENGINE_STAGE_MODULES, + *_PINNED_TAB_MODULES, + stochastic_assignment, + ), + "frs_education_grant_split": (*_ENGINE_STAGE_MODULES, frs_release), + # The BRMA takes frs_brma's engine binding: one engine call materialises + # LHA_category, and Northern Ireland's benefit units draw on frs_brma's + # keyed uniforms; Great Britain's read the area-to-BRMA layer, by the + # systems atomic_area_support names (microcosm#931). + "area_brma": ( + *_ENGINE_STAGE_MODULES, + atomic_area_support, + brma_area_layer, + frs_brma, + geography_first_roster, + stochastic_assignment, + ), + # The donor stages read their donor tabs through frs_spine, record each + # fit's weight kind, gate the fits and clip donor support to the donor's + # range (microcosm#931); the WAS recipients' predictors map benefit units to + # households, count family roles, group tenure by the shared tenure map and + # name the HBAI income predictor. The WAS stage, and the LISA, bus, LCFS, + # ETB, landlord, reporter and housing stages below, fit through the QRF + # modules (microcosm#931). The WAS stage reads its main residences off each + # owner's constituency where its declaration asks (area_home_values): it + # ranks owners and reads the knots through property_localisation, whose + # owners are the tenure map's and whose keyed uniforms are + # microcosm.graph's, which encodes each draw's coordinates as canonical + # JSON; it divides by the regional uprating's factor, reads the knots' + # regions from the FRS region map, and counts its evidence in true source + # households, as its donor-range cap counts the sources it adjusts; so it + # and the LISA stage, which binds it, are counting stages (microcosm#931). + "was_wealth": ( + *_ENGINE_STAGE_MODULES, + *_PINNED_TAB_MODULES, + *_QRF_MODULES, + area_home_values, + frs_brma, + gates, + household_income, + property_localisation, + qrf_gates, + regional_uprating, + rowwise_geography, + support_clip, + tenure_constants, + uc_relationships, + graph_randomness, + graph_canonical, + stochastic_assignment, + ), # The LISA stage reads its household donor through the WAS cleaning and its # household predictors through the WAS recipient surface; the anchor counts # HMRC's payers through the typed ISA facts (microcosm#1415). - "was_lisa": (uk_engine_adapter, was_wealth, hmrc_isa_statistics), + "was_lisa": ( + *_ENGINE_STAGE_MODULES, + was_wealth, + hmrc_isa_statistics, + *_PINNED_TAB_MODULES, + *_FEED_FACT_MODULES, + *_QRF_MODULES, + frs_brma, + gates, + household_income, + support_clip, + tenure_constants, + uc_relationships, + stochastic_assignment, + ), # Landlords' receipts read the vendored PRIS rows through their typed # loader, the tape through the income stage's preparation and uprating, - # and the support channel's names (microcosm#1106). - "property_components": (hmrc_property_rental, spi_income, spi_support), - # The regional values rank as the dense node does and read its knots - # (microcosm#1112). - "regional_home_values": (property_localisation, rowwise_geography), + # and the support channel's names (microcosm#1106); the loader checks the + # feed, the preparation reads the donors' age model over the take-up + # population and uprates savings income, and the fits record their weight + # kinds (microcosm#931). + "property_components": ( + hmrc_property_rental, + spi_income, + spi_support, + *_FEED_FACT_MODULES, + *_QRF_MODULES, + frs_take_up, + gates, + hmrc_uprating, + stochastic_assignment, + ), # The purchases read the vendored HMRC Stamp Tax rows through the feed-pinned # reader and mark SDLT's regions from the FRS region map (microcosm#1095). - "property_purchases": (ledger_fact_vendoring, rowwise_geography), - "nts_bus_travel": (uk_engine_adapter, bus_use_incidence), + "property_purchases": ( + *_FEED_FACT_MODULES, + rowwise_geography, + stochastic_assignment, + ), + # The bus travel donors also read published car availability as vendored + # facts, and the recipients' HBAI gross income (microcosm#931). + "nts_bus_travel": ( + *_ENGINE_STAGE_MODULES, + bus_use_incidence, + *_PINNED_TAB_MODULES, + *_FEED_FACT_MODULES, + *_QRF_MODULES, + gates, + household_income, + qrf_gates, + support_clip, + stochastic_assignment, + ), # The energy level and prices are energy_pricing's (microcosm#1418): a # code-only change to them must re-key the stage (the independent review - # of microcosm#1420, finding 5). - "lcfs_consumption": (uk_engine_adapter, energy_pricing), - "etb_vat": (uk_engine_adapter,), - "etb_services": (uk_engine_adapter,), - "uc_reporter_redraw": (uc_relationships, uk_engine_adapter), + # of microcosm#1420, finding 5). The LCFS and ETB donors are uprated + # through donor_uprating on vendored indices, the recipients' household + # income and family roles are predictors, and bus fares are priced over + # calendar and fiscal-year windows; the LCFS stage also rakes its energy, + # assigns car attributes, tops up gambling and reads the DfT boardings of + # each recipient's region through the FRS region map (microcosm#931). + "lcfs_consumption": ( + *_ENGINE_STAGE_MODULES, + energy_pricing, + *_PINNED_TAB_MODULES, + *_FEED_FACT_MODULES, + *_QRF_MODULES, + bus_fare_pricing, + donor_uprating, + gambling_participation, + gates, + household_income, + ledger_targets, + nts_car_attributes, + qrf_gates, + raking, + rowwise_geography, + support_clip, + stochastic_assignment, + ), + "etb_vat": ( + *_ENGINE_STAGE_MODULES, + *_PINNED_TAB_MODULES, + *_FEED_FACT_MODULES, + *_QRF_MODULES, + donor_uprating, + gates, + household_income, + support_clip, + uc_relationships, + stochastic_assignment, + ), + # The services stage counts its recipients' family roles as the VAT stage + # does, from etb_vat's role table (microcosm#931). + "etb_services": ( + *_ENGINE_STAGE_MODULES, + *_PINNED_TAB_MODULES, + *_FEED_FACT_MODULES, + *_QRF_MODULES, + bus_fare_pricing, + bus_support_pricing, + donor_uprating, + energy_pricing, + etb_vat, + gates, + ledger_targets, + qrf_gates, + support_clip, + uc_relationships, + stochastic_assignment, + ), + # The reporter redraw gates its fits and maps household weights to benefit + # units as the UC capital stage does (microcosm#931). + "uc_reporter_redraw": ( + uc_relationships, + *_ENGINE_STAGE_MODULES, + *_QRF_MODULES, + qrf_gates, + uc_capital_coherence, + stochastic_assignment, + ), # The SPI benefit pass restores from the FRS twin through the support # channel's lineage, re-derives the disability flags and the own-right # rule, draws over the take-up population and maps household weights to - # benefit units as the UC capital stage does (microcosm#1095). + # benefit units as the UC capital stage does (microcosm#1095); the flags + # resolve their year's rules and week count, and the draws read the + # take-up contract (microcosm#931). "spi_benefit_coherence": ( spi_support, frs_disability, frs_education, frs_take_up, uc_capital_coherence, + frs_release, + frs_spine, + take_up_contract, + stochastic_assignment, ), - "uc_capital_coherence": (uc_relationships,), - # The Universal Credit redraw reads the award through the adapter, the + # Unavailable capital is frs_spine's sentinel (microcosm#931). + "uc_capital_coherence": (uc_relationships, frs_spine, stochastic_assignment), # One engine materialization of the proxy, the claimant role and carer - # status; the frs_spine carrier's codes; the receipt's support counts - # (microcosm#1095, Y1). - "uc_limited_capability": (uk_engine_adapter, frs_spine, receipt_counts), + # status, which runs as every engine call does (microcosm#931), with the + # year's rules from the FRS release; the frs_spine carrier's codes, whose + # module draws through the keyed-draw helpers; the receipt's support + # counts (microcosm#1095, Y1). + "uc_limited_capability": ( + *_ENGINE_STAGE_MODULES, + frs_release, + frs_spine, + receipt_counts, + stochastic_assignment, + ), + # The Universal Credit redraw reads the award through the adapter, the # vendored DWP counts, the closure stage's rule and rates, the deduction # attributes, the take-up population and the family-type roles, and counts - # its pools and receipt on the source lineage (microcosm#1334). + # its pools and receipt on the source lineage (microcosm#1334); the + # attributes map benefit units to households and judge their realisation + # by the shared tolerance, and the year's rules come from the FRS release; + # it types its engine as the adapter's ProjectingRulesEngine, a protocol + # that extends microcosm.frame.rules' RulesEngine (microcosm#931). "uc_take_up": ( - uk_engine_adapter, - ledger_fact_vendoring, + *_ENGINE_STAGE_MODULES, + frame_rules, + *_FEED_FACT_MODULES, legacy_closure_take_up, uc_deduction_attributes, frs_take_up, uc_relationships, uc_capital_coherence, receipt_counts, + frs_brma, + frs_release, + stage_health, + stochastic_assignment, ), # The Move to UC claim draw reuses the UC stage's household-to-benefit-unit # weight mapping and reads the vendored DWP rows (microcosm#1095). - "uc_legacy_closure_take_up": (uc_capital_coherence, ledger_fact_vendoring), + "uc_legacy_closure_take_up": ( + uc_capital_coherence, + *_FEED_FACT_MODULES, + stochastic_assignment, + ), # The Pension Credit redraw reuses the UC stage's household-to-benefit-unit - # weight mapping and the adapter's engine materialization. - "pension_credit_take_up": (uc_capital_coherence, uk_engine_adapter), - # The Child Benefit redraw shares the weight mapping and the adapter. - "child_benefit_take_up": (uc_capital_coherence, uk_engine_adapter), + # weight mapping and the adapter's engine materialization; it reads the + # take-up contract and the year's rules, and types its engine by the same + # protocol as the UC redraw (microcosm#931). + "pension_credit_take_up": ( + uc_capital_coherence, + *_ENGINE_STAGE_MODULES, + frame_rules, + frs_release, + take_up_contract, + stochastic_assignment, + ), + # The Child Benefit redraw shares the weight mapping and the adapter, and + # reads the vendored Child Benefit statistics (microcosm#931). + "child_benefit_take_up": ( + uc_capital_coherence, + *_ENGINE_STAGE_MODULES, + *_FEED_FACT_MODULES, + stochastic_assignment, + ), # The Carer's Allowance draw reads the earnings test through the adapter's # engine materialization and counts its receipt's support on the source # lineage (microcosm#1319), and balances its draws through the shared - # placement helper (microcosm#1418). + # placement helper (microcosm#1418); it reads the take-up contract and the + # year's rules (microcosm#931). "carers_allowance_take_up": ( - uk_engine_adapter, + *_ENGINE_STAGE_MODULES, receipt_counts, + frs_release, + take_up_contract, + stochastic_assignment, + ), + # The stored attendance is one engine materialization (microcosm#1095), + # whose year's rules come from the FRS release (microcosm#931). + "private_school_attendance": (*_ENGINE_STAGE_MODULES, frs_release), + # The housing shell keeps council tax bands within the nation, whose + # regions are microcosm.calibrate's UK region tier, and gates its fits. + # It prices the Great Britain bills of the support channel's copies + # through spi_housing_council_tax, which takes each copy's reference Band + # D through the shared draw-cell expectation, on atomic_area_conditioning's + # definition, systems and copy level over the decoded supports, which read + # the support profile, the band order and the FRS region map; the + # definition is encoded canonically and names microcosm.graph's stream. + # It reads the Band D and prices the statute through council_tax_band_d, + # Scotland's Band D a vendored fact checked against the pinned feed, and + # reads the authority code column (microcosm#931). + "spi_housing_shell": ( + atomic_geography, + atomic_support_conditioning, + atomic_support_expectations, + atomic_area_conditioning, + atomic_area_support, + council_tax_band_d, + council_tax_constants, + local_authority_input, + rowwise_geography, + spi_housing_council_tax, + geography_constants, + *_FEED_FACT_MODULES, + *_QRF_MODULES, + qrf_gates, stochastic_assignment, + graph_canonical, + graph_randomness, ), - # The stored attendance is one engine materialization (microcosm#1095). - "private_school_attendance": (uk_engine_adapter,), + # The deduction attributes map benefit units to households and judge + # their realisation by the shared tolerance; the salary sacrifice reads its + # bonus anchor as vendored facts and gates its fits; the student loans + # resolve their year's rules (microcosm#931). + "uc_deduction_attributes": (frs_brma, stage_health, stochastic_assignment), + "salary_sacrifice": (*_FEED_FACT_MODULES, qrf_gates, stochastic_assignment), + "student_loans": (frs_release, stochastic_assignment), # The subscriptions walk the CGT bands on its income proxy (with the # proxy's loss relief), read HMRC's ISA tables through their typed loader - # and round their receipts' support (microcosm#1415). + # and round their receipts' support (microcosm#1415); the loss relief + # evaluates its named term, the donor tabs are pinned, and the regions are + # microcosm.calibrate's UK region tier (microcosm#931). "isa_subscriptions": ( cgt_imputation, + geography_constants, hmrc_income, hmrc_isa_statistics, receipt_counts, + *_PINNED_TAB_MODULES, + *_FEED_FACT_MODULES, + target_materialization, + stochastic_assignment, ), - # The points draws are keyed on identity, read the vendored DWP tables and - # round their receipt's support (microcosm#1095, P1). - "pip_points": (receipt_counts, stochastic_assignment, ledger_fact_vendoring), + # The points draws are keyed on identity, read the vendored DWP tables + # through the vendored-fact reader and round their receipt's support + # (microcosm#1095, P1). + "pip_points": (receipt_counts, stochastic_assignment, *_FEED_FACT_MODULES), # The claimant draw is keyed on identity and its receipt rounds its # support (microcosm#1428). "trading_allowance_receipts": (receipt_counts, stochastic_assignment), @@ -266,7 +1010,8 @@ # index and contribution cap, the SPI channel's EPB and tape-row flag, and # price the cars twice through the engine adapter; their draws are keyed on # identity, and their receipts' support is rounded - # (PolicyEngine/uk-autumn-budget-dashboard-2026#316). + # (PolicyEngine/uk-autumn-budget-dashboard-2026#316). The loader checks the + # feed, and the donor tab is pinned (microcosm#931). "company_cars": ( frs_spine, hmrc_company_car_statistics, @@ -274,21 +1019,35 @@ receipt_counts, spi_income, spi_support, + trace, + *_FEED_FACT_MODULES, stochastic_assignment, - uk_engine_adapter, + *_ENGINE_STAGE_MODULES, ), # The copies' childcare re-draw draws through frs_take_up's childcare # helper, reads each household's clone path, and measures its flags for - # the stage-time signal gate by the shared disclosure rules (microcosm#1095). - "childcare_copy_take_up": (frs_take_up, household_lineage, receipt_counts), + # the stage-time signal gate by the shared disclosure rules (microcosm#1095); + # it maps benefit units to households, ages children by the DfE's rule and + # reads the take-up contract (microcosm#931). + "childcare_copy_take_up": ( + frs_take_up, + household_lineage, + receipt_counts, + dfe_spring_term, + frs_brma, + take_up_contract, + stochastic_assignment, + ), # The employment shell fits, scores and draws through microcosm.fit's # categorical classifier, held-out scores and keyed donors, reads the # shared occupation codes, and counts its receipt's support by the shared - # disclosure rules (microcosm#1490). + # disclosure rules (microcosm#1490); the classifier and held-out scores + # compute through the QRF modules (microcosm#931). "spi_employment_shell": ( fit_categorical, fit_donors, fit_held_out, + *_QRF_MODULES, occupation_constants, receipt_counts, stochastic_assignment, @@ -296,12 +1055,17 @@ # The sub-major draw fits and scores through microcosm.fit's categorical # classifier and held-out scores, reads the shared occupation codes, and # counts its receipt's support by the shared disclosure rules - # (microcosm#1490). + # (microcosm#1490); it reads its donor tab through trace's digest and + # records its fit's weight kind, and its classifier and held-out scores + # compute through the QRF modules (microcosm#931). "aps_occupation": ( fit_categorical, fit_held_out, + *_QRF_MODULES, occupation_constants, receipt_counts, + gates, + trace, stochastic_assignment, ), } @@ -309,26 +1073,62 @@ # The receipts' counting rules and the shared lineage readers: every floor # counts true source households through receipt_counts, and the FRS-trained # fits fold geographic clones through household_lineage, both reading -# source_lineage on household_lineage's declaration (microcosm#931). An edit -# to any of the three moves the hash of every stage whose module or helpers -# import one of them; test_uk_graph derives this set from the imports. -_COUNTING_MODULES = (receipt_counts, household_lineage, source_lineage) +# source_lineage on household_lineage's declaration, which names the +# geographic clone index through geographic_clones (microcosm#931). An edit +# to any of the four moves the hash of every stage whose module or helpers +# import one of the first three; test_uk_graph derives this set from the +# imports. +_COUNTING_MODULES = ( + receipt_counts, + household_lineage, + source_lineage, + geographic_clones, +) _COUNTING_STAGES = frozenset( { + "frs_spine", "frs_take_up", "frs_person_draws", "frs_household_draws", + # The council-tax stage's module counts its bill provenance's receipt + # in true source households (microcosm#931). + "frs_council_tax", + # spi_spine, stage 14's module, rounds the support channel's receipt + # (microcosm#931). + "frs_hmrc_spine_leaves", "frs_annual_income_spells", + # The geographic clones and their keys count their evidence in true + # source households, and the keys read each clone path through the + # shared lineage reader (microcosm#931). + "geographic_expand", + "geographic_identity", + # So do the area stages, the private rents and the council-tax + # localisation; the SPI copies' areas and the localisation read each + # copy's path through the shared reader too (microcosm#931). + "atomic_area_assign", + "local_authority", + "rfdr", + "area_brma", + "spi_copy_areas", + "private_rent", + "council_tax_localise", "hmrc_spi_income_spine", + "spi_support_channel", + "spi_income_band_donors", "company_cars", "spi_housing_shell", + # The WAS stage counts its area home values' evidence and the sources + # its donor-range cap adjusts in true source households, and the LISA + # stage binds it (microcosm#931). + "was_wealth", + "was_lisa", "high_value_home_support", - "regional_home_values", "property_components", "property_purchases", "nts_bus_travel", "lcfs_consumption", "etb_vat", + "etb_services", "uc_reporter_redraw", "spi_benefit_coherence", "uc_capital_coherence", @@ -339,10 +1139,9 @@ "salary_sacrifice", "carers_allowance_take_up", "childcare_copy_take_up", + "uc_deduction_attributes", "isa_subscriptions", "trading_allowance_receipts", - "frs_spine", - "frs_council_tax", "frs_disability", "spi_employment_shell", "aps_occupation", @@ -394,12 +1193,21 @@ def _implementation_hash(kernel: object, stage: str, transform: object | None) - # hermetic registries unhashable and, more importantly, would fail to bind # production edits made elsewhere in that stage's module. A transform may # still declare extra behavior-bearing sources through - # ``graph_implementation_dependencies``. + # ``graph_implementation_dependencies``. Every kernel rebuilds its frame + # on national_frame's schema, and the transforms build, validate and read + # the time period of their frames through it (microcosm#931). It stamps + # each frame it builds with the engine adapter's ISA income contract and + # savings income projection, so every kernel hashes the adapter with it. + # Every kernel casts the cells it owns, and the root normalises the frame + # it creates, through graph_frames. dependencies = getattr(transform, "graph_implementation_dependencies", None) return source_hash( type(kernel), stage_evidence, graph_evidence, + national_frame, + uk_engine_adapter, + graph_frames, _stage_artifacts, _mass_log_payload, _stage_module(stage), @@ -647,7 +1455,7 @@ def _fixture_descriptor( missing = sorted(set(_STAGE_MODULES) - set(stages)) extra = sorted(set(stages) - set(_STAGE_MODULES)) raise ValueError( - "UK parity fixture must describe the current 39-stage spine " + "UK parity fixture must describe every stage of the current spine " f"(missing={missing}, extra={extra})." ) return descriptor, stages @@ -675,7 +1483,6 @@ def _fixture_implementations(source: Path) -> Mapping[str, object]: from .etb_services import UKETBServicesStageTransform from .etb_vat import UKETBVATStageTransform from .frs_annual_income import UKFRSAnnualIncomeStageTransform - from .frs_brma import UKFRSBRMAStageTransform from .frs_council_tax import UKFRSCouncilTaxStageTransform from .frs_disability import UKFRSDisabilityStageTransform from .frs_education import UKFRSEducationStageTransform @@ -686,6 +1493,8 @@ def _fixture_implementations(source: Path) -> Mapping[str, object]: from .frs_person_draws import UKFRSPersonDrawsStageTransform from .frs_relationships import UKFRSRelationshipsStageTransform from .frs_take_up import UKFRSTakeUpStageTransform + from .geographic_expand import UKGeographicExpandStageTransform + from .geographic_identity import UKGeographicIdentityStageTransform from .high_value_home_support import UKHighValueHomeSupportStageTransform from .isa_subscriptions import UKISASubscriptionsStageTransform from .lcfs_consumption import UKLCFSConsumptionStageTransform @@ -699,7 +1508,6 @@ def _fixture_implementations(source: Path) -> Mapping[str, object]: UKPropertyPurchasesStageTransform, UKPropertyPurchaseTargets, ) - from .regional_home_values import UKRegionalHomeValuesStageTransform from .regional_uprating import UKRegionalPropertyUpratingStageTransform from .salary_sacrifice import UKSalarySacrificeStageTransform from .spi_band_donors import UKSPIIncomeBandDonorStageTransform @@ -711,6 +1519,15 @@ def _fixture_implementations(source: Path) -> Mapping[str, object]: UKSPIIncomeSpineStageTransform, UKSPISupportChannelStageTransform, ) + from .spine_council_tax import UKCouncilTaxLocaliseStageTransform + from .spine_geography import ( + UKAreaBRMAStageTransform, + UKAtomicAreaAssignStageTransform, + UKLocalAuthorityStageTransform, + UKRFDRStageTransform, + UKSPICopyAreasStageTransform, + ) + from .spine_private_rent import UKPrivateRentStageTransform from .student_loans import UKStudentLoansStageTransform from .take_up_contract import load_uk_take_up_contract from .trading_allowance_receipts import UKTradingAllowanceReceiptsStageTransform @@ -792,8 +1609,22 @@ def _fixture_implementations(source: Path) -> Mapping[str, object]: _mapping(descriptor.get("cgt_badr_parameters"), label="CGT BADR parameters") ) ) + # The fixture's toy conditioned atomic-area supports, one input each under + # the id sources.yaml declares the system's support by, read as the donor + # tables are (microcosm#931). + supports = { + system: _fixture_input(source, inputs, source_id).read_bytes() + for system, source_id in FIXTURE_SUPPORT_INPUTS.items() + } - engine = PolicyEngineUKEngine() + # The fixture's spine clones every household geographic_clones times, and + # its engine refuses a frame of any other clone count (microcosm#931). + geographic_clones = config.get("geographic_clones") + if isinstance(geographic_clones, bool) or not isinstance(geographic_clones, int): + raise ValueError( + "UK parity fixture config.geographic_clones must be an integer clone count." + ) + engine = PolicyEngineUKEngine(clones=geographic_clones) contract = load_uk_take_up_contract() qrf_estimators = int(config["qrf_estimators"]) donor_sample_size = int(config["spi_donor_sample_size"]) @@ -833,8 +1664,29 @@ def _fixture_implementations(source: Path) -> Mapping[str, object]: "frs_household_draws": UKFRSHouseholdDrawsStageTransform( contract=contract, stage=stages["frs_household_draws"] ), - "frs_brma": UKFRSBRMAStageTransform( - stage=stages["frs_brma"], engine=engine + "atomic_area_assign": UKAtomicAreaAssignStageTransform( + stage=stages["atomic_area_assign"], supports=supports + ), + "local_authority": UKLocalAuthorityStageTransform( + stage=stages["local_authority"] + ), + "rfdr": UKRFDRStageTransform(stage=stages["rfdr"], supports=supports), + "area_brma": UKAreaBRMAStageTransform( + stage=stages["area_brma"], engine=engine + ), + "spi_copy_areas": UKSPICopyAreasStageTransform( + stage=stages["spi_copy_areas"], engine=engine, supports=supports + ), + # The fixture is a sampled rung, as its SPI stages are, so the + # private rents take the sampled rung's region rule (microcosm#931). + "private_rent": UKPrivateRentStageTransform( + stage=stages["private_rent"], sampled_rung=True + ), + # Each clone's council tax at its local authority's committed + # Band D over the expected Band D of its draw cell on the toy + # supports, just before the WAS chain (microcosm#931). + "council_tax_localise": UKCouncilTaxLocaliseStageTransform( + stage=stages["council_tax_localise"], supports=supports ), "was_wealth": UKWASWealthStageTransform( stage=stages["was_wealth"], engine=engine, donor=was @@ -867,9 +1719,6 @@ def _fixture_implementations(source: Path) -> Mapping[str, object]: stage=stages["property_purchases"], targets=property_purchase_targets, ), - "regional_home_values": UKRegionalHomeValuesStageTransform( - stage=stages["regional_home_values"] - ), "lcfs_consumption": UKLCFSConsumptionStageTransform( stage=stages["lcfs_consumption"], engine=engine, @@ -890,6 +1739,12 @@ def _fixture_implementations(source: Path) -> Mapping[str, object]: "frs_annual_income_spells": UKFRSAnnualIncomeStageTransform( raw_dir, stage=stages["frs_annual_income_spells"] ), + "geographic_expand": UKGeographicExpandStageTransform( + stage=stages["geographic_expand"], clones=geographic_clones + ), + "geographic_identity": UKGeographicIdentityStageTransform( + stage=stages["geographic_identity"] + ), "spi_support_channel": UKSPISupportChannelStageTransform( stage=stages["spi_support_channel"], sample_fraction=sample_fraction, @@ -918,8 +1773,13 @@ def _fixture_implementations(source: Path) -> Mapping[str, object]: donor=aps, boosted_max_iter=qrf_estimators, ), + # The Great Britain copies' bills at their region's expected Band + # D on the toy supports, at the committed 2024-25 Band D + # (microcosm#931). "spi_housing_shell": UKSPIHousingShellStageTransform( - stage=stages["spi_housing_shell"], n_estimators=qrf_estimators + stage=stages["spi_housing_shell"], + supports=supports, + n_estimators=qrf_estimators, ), "uc_reporter_redraw": UKUCReporterRedrawStageTransform( stage=stages["uc_reporter_redraw"], engine=engine @@ -1021,11 +1881,13 @@ def fixture_stage_plan_inputs( ) -> tuple[tuple[SourceStageSpec, ...], Mapping[str, object]]: """The H2 bundle as legacy StagePlan inputs: ordered specs, every transform. - The graph's CREATE node loads the captured root frame, so the kernel - mapping carries no ``frs_spine`` transform; the legacy oracle applies one - built from the same ``frs_raw`` tables, added here. The descriptor keys - stages by name, so the committed UK manifest supplies the order the - legacy plan runs them in. + The hermetic registry's transforms (``_fixture_implementations``) carry + no ``frs_spine`` transform, so its CREATE node, with no root transform + bound, loads the captured root frame. This mapping adds one built from + the same ``frs_raw`` tables: the legacy oracle applies it, and H2 parity + binds the graph's CREATE kernel to it, the root's production path. The + descriptor keys stages by name, so the committed UK manifest supplies the + order the legacy plan runs them in. """ from microcosm.build.country_spec import load_country_spec @@ -1134,39 +1996,13 @@ def _id_index(frame: Frame, entity: str) -> pd.Index: return pd.Index(frame.table(entity)[id_column].to_numpy(copy=True), name=id_column) -def _cast_owned(series: pd.Series, dtype: str) -> pd.Series: - target = dtype_for_token(dtype) - if series.dtype != target: - series = series.astype(target) - return series - - def _owned_series(frame: Frame, entity: str, column: str, dtype: str) -> pd.Series: - values = _cast_owned(frame.table(entity)[column].copy(deep=True), dtype) + values = cast_owned(frame.table(entity)[column].copy(deep=True), dtype) values.index = _id_index(frame, entity) values.name = column return values -def _normalize_create_frame(frame: Frame, context: KernelContext) -> Frame: - tables = {entity: frame.table(entity).copy(deep=True) for entity in frame.entities} - for owned in context.node.outputs: - tables[owned.entity][owned.column] = _cast_owned( - tables[owned.entity][owned.column], owned.dtype - ).array - return Frame( - tables, - frame.schema, - {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, - frame.strata, - mass_log=frame.mass_log, - metadata={ - **frame.metadata, - "time_period": str(context.params.get("time_period", "2024")), - }, - ) - - class UKCreateKernel(KernelBase): """Load fixture CSV tables or invoke the bound FRS root transform.""" @@ -1200,7 +2036,7 @@ def run(self, context: KernelContext) -> KernelResult: f"The UK root transform returned {type(frame).__name__}, not Frame." ) return KernelResult( - frame=_normalize_create_frame(frame, context), + frame=normalize_create_frame(frame, context), artifacts=_stage_artifacts( "frs_spine", self.transform, @@ -1217,6 +2053,13 @@ class UKIdentityKernel(KernelBase): ref = "uk.identity@1" capabilities = _FILTER + def implementation_hash(self) -> str: + # The person ids it keeps are read on national_frame's schema + # (microcosm#931). + return source_hash( + type(self), national_frame, dependencies=self.capabilities.dependencies + ) + def run(self, context: KernelContext) -> KernelResult: person = context.tables[UK_NATIONAL_SCHEMA.person_entity] id_column = UK_NATIONAL_SCHEMA.person_id_column @@ -1230,10 +2073,20 @@ class UKClaimKernel(KernelBase): ref = "uk.claim@1" capabilities = _COMPUTE + def implementation_hash(self) -> str: + # The claimed cells are indexed by national_frame's schema's id + # columns and cast through graph_frames (microcosm#931). + return source_hash( + type(self), + national_frame, + graph_frames, + dependencies=self.capabilities.dependencies, + ) + def run(self, context: KernelContext) -> KernelResult: columns = { (owned.entity, owned.column): pd.Series( - _cast_owned( + cast_owned( context.tables[owned.entity][owned.column].copy(deep=True), owned.dtype, ).array, @@ -1320,6 +2173,57 @@ def _expand_cells(context: KernelContext) -> tuple[tuple[str, str, str], ...]: return tuple(tuple(str(part) for part in item) for item in raw) # type: ignore[arg-type] +#: The copy-id field each id-offset structural stage mints (microcosm#931): +#: copy ``k`` takes ``source + k x radix(field)`` on every id column, and +#: every incumbent id lies below the field's radix, so the copy index is the +#: integer quotient. The SPI support stages name each copy's source in +#: ``{entity}_source_id`` instead. +_COPY_ID_FIELDS = MappingProxyType( + { + "geographic_expand": "geographic_support", + "high_value_home_support": "high_value_support", + "cgt_support_split": "cgt_support_split", + "cgt_incidence_clone": "cgt_incidence_clone", + "cgt_residential_split": "cgt_residential_split", + } +) + + +def _declared_copy_radix(stage: str, before: Frame) -> int | None: + """The declared radix an id-offset stage mints at, or None for the others. + + Every incumbent id and membership must sit below it, or the quotient + would name the wrong parent; the refusal prints counts only. + """ + + field = _COPY_ID_FIELDS.get(stage) + if field is None: + return None + person = before.table(before.schema.person_entity) + columns = [ + (before.table(entity), before.schema.entity_id_column(entity)) + for entity in before.entities + ] + [ + (person, before.schema.membership_column(group)) + for group in before.schema.group_entities + ] + for table, column in columns: + copy_id_table.require_below( + table[column], + field, + what=f"UK stage {stage!r} incumbent {column} values", + ) + return copy_id_table.radix(field) + + +def _exact_ids(values: pd.Series, *, label: str) -> np.ndarray: + """An id column as exact int64: ids reach 2**53 and above (microcosm#931).""" + + if not pd.api.types.is_integer_dtype(values.dtype): + raise TypeError(f"{label} must be an integer id column, not {values.dtype}.") + return values.to_numpy(dtype=np.int64) + + def _source_lineage( before: Frame, after: Frame, @@ -1327,62 +2231,66 @@ def _source_lineage( *, id_offset: int | None, ) -> pd.Series: - """Derive new target-to-immediate-source ids from a structural result.""" + """Derive new target-to-immediate-source ids from a structural result. + + Vectorised on exact int64 ids, so it is exact on any frame. A row-by-row + walk reads a row whose columns hold only integers and floats as float64, + which is inexact above 2**53 (an all-integer row stays int64, and a row + with a string or boolean column comes back as objects). The refusals + print counts, never an id. + """ id_column = before.schema.entity_id_column(entity) before_table = before.table(entity) after_table = after.table(entity) + dtype = before_table[id_column].dtype before_ids = pd.Index(before_table[id_column]) + after_ids = _exact_ids(after_table[id_column], label=f"{entity}.{id_column}") + new = ~pd.Index(after_ids).isin(before_ids) # A weights-only structural stage (the #970 incidence anchor) adds no - # row: every target is an incumbent, so the lineage is empty without - # walking the table row by row. - if pd.Index(after_table[id_column]).isin(before_ids).all(): + # row: every target is an incumbent, so the lineage is empty. + if not new.any(): return pd.Series( [], - index=pd.Index([], name=id_column, dtype=before_table[id_column].dtype), - dtype=before_table[id_column].dtype, + index=pd.Index([], name=id_column, dtype=dtype), + dtype=dtype, name=id_column, ) - targets: list[object] = [] - values: list[object] = [] + targets = after_ids[new] source_column = f"{entity}_source_id" - for _, row in after_table.iterrows(): - target = row[id_column] - if target in before_ids: - continue - - # CGT stages retain long-lived source-id provenance from the SPI - # support stage, so their immediate clone lineage is the stage's - # reviewed ID offset, not that older provenance column. Copy ``k`` - # of a CGT structural stage takes ``source + k * offset`` (the clone - # stage mints one copy, the support split up to ``ceil(w / cap)``, - # microcosm#1045), and every incumbent id lies below the offset, so - # the copy index is the integer quotient. - if id_offset is not None: - copy_index = int(target) // int(id_offset) - candidate = target - copy_index * id_offset - if copy_index >= 1 and candidate in before_ids: - targets.append(target) - values.append(candidate) - continue + if id_offset is not None: + # CGT and high-value stages retain long-lived source-id provenance + # from the SPI support stage, so their immediate lineage is the + # stage's declared copy-id radix, not that older provenance column. + copy_index = targets // int(id_offset) + sources = targets - copy_index * int(id_offset) + unmatched = int(((copy_index < 1) | ~pd.Index(sources).isin(before_ids)).sum()) + if unmatched: raise ValueError( - f"UK stage produced {entity!r} target id {target!r} whose " - f"offset lineage {candidate!r} is not an incumbent id." + f"UK stage produced {unmatched} new {entity!r} row(s) whose id " + "less its copy ordinal times the stage's declared copy-id radix " + "is not an incumbent id." ) - - if source_column in after_table and row[source_column] in before_ids: - targets.append(target) - values.append(row[source_column]) - continue - + elif source_column in after_table: + # Only the new rows' sources are read, as the row walk read them. + sources = _exact_ids( + after_table[source_column][new], label=f"{entity}.{source_column}" + ) + unmatched = int((~pd.Index(sources).isin(before_ids)).sum()) + if unmatched: + raise ValueError( + f"UK stage produced {unmatched} new {entity!r} row(s) whose " + f"{source_column!r} is not an incumbent id." + ) + else: raise ValueError( - f"UK stage produced {entity!r} target id {target!r} without an " - f"explicit {source_column!r} or a declared ID-offset lineage rule." + f"UK stage produced {int(new.sum())} new {entity!r} row(s) without an " + f"explicit {source_column!r} or a declared copy-id lineage rule." ) return pd.Series( - values, - index=pd.Index(targets, name=id_column, dtype=before_table[id_column].dtype), - dtype=before_table[id_column].dtype, + sources, + index=pd.Index(targets, name=id_column, dtype=dtype), + dtype=dtype, name=id_column, ) @@ -1421,6 +2329,16 @@ def run(self, context: KernelContext) -> KernelResult: f"UK stage {self.stage!r} has no bound production transform; " "recorded fixture deltas are not accepted as parity evidence." ) + # The geographic clone count is a parameter of the node that makes the + # clones, so it is in that node's key, and the transform must make + # exactly that many (microcosm#931); no other stage has either. + declared_clones = context.params.get("geographic_clones") + transform_clones = getattr(transform, "clones", None) + if transform_clones != declared_clones: + raise ValueError( + f"UK stage {self.stage!r} declares {declared_clones!r} geographic " + f"clone(s) on its node, but its transform makes {transform_clones!r}." + ) before = _minimal_frame(context) after = _invoke_transform(transform, before, context) if not isinstance(after, Frame): @@ -1428,25 +2346,7 @@ def run(self, context: KernelContext) -> KernelResult: f"UK stage {self.stage!r} returned {type(after).__name__}, not Frame." ) cells = _expand_cells(context) - id_offset = None - if self.stage in { - "high_value_home_support", - "cgt_support_split", - "cgt_incidence_clone", - "cgt_residential_split", - }: - id_offset = id_multiplier_for_values( - *( - before.table(entity)[before.schema.entity_id_column(entity)] - for entity in before.entities - ), - *( - before.table(before.schema.person_entity)[ - before.schema.membership_column(group) - ] - for group in before.schema.group_entities - ), - ) + id_offset = _declared_copy_radix(self.stage, before) expand: dict[str, pd.Series] = {} for entity in before.entities: expand[entity] = _source_lineage( @@ -1576,15 +2476,7 @@ def build_uk_registry( fixture_resolver = _FixtureTransformResolver() for stage in sorted(stage_names): transform = implementations.get(stage) - if stage in { - "spi_support_channel", - "spi_income_band_donors", - "high_value_home_support", - "cgt_support_split", - "cgt_incidence_clone", - "cgt_incidence_anchor", - "cgt_residential_split", - }: + if stage in UK_SPINE_STRUCTURAL_STAGES: registry.register( UKExpandStageKernel( stage, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py index c8a574003f..d5da29bc4b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_national.py @@ -9,10 +9,11 @@ * ``uk.full.national_targets`` compiles the national register from the pinned Chronicle artifact (measure exclusions, band edges, the optional frozen scoring register); no ladder and no local surface. -* ``uk.full.national_problem`` resolves the engine measures on the bound - frame, materialises the register and encodes one ordered problem whose - bindings carry the national doctrine's loss weights, mass reason and - bounds. +* ``uk.full.national_problem`` refuses a bound spine the national line cannot + build on (one that is not geography-first, or not of one geographic clone; + microcosm#931), then resolves the engine measures on the bound frame, + materialises the register and encodes one ordered problem whose bindings + carry the national doctrine's loss weights, mass reason and bounds. * ``uk.full.dense`` / ``uk.full.calibrated`` are the existing calibration nodes; the dense node's admission is the bound spine's provenance artifact rather than a source preflight battery, because the national @@ -93,16 +94,16 @@ national_doctrine, ) from . import support_floor as support_floor_rules -from .battery_bindings import UK_GATE_REGISTRY +from .battery_bindings import UK_GATE_REGISTRY, uk_gate_registry_modules from .calibration_run import ( UK_CALIBRATION_GATE_SCOPE_EXCLUSIONS, uk_local_gate_scope_exclusions, - uk_scoped_gate_manifest, ) from .cgt_projection import UK_CGT_PROJECTION_ARTIFACT_KEY from .content_identity import uk_frame_content_identity from .diagnostics import uk_target_geography_levels from .full_targets import CHRONICLE_SOURCE_CODEC, load_uk_national_target_inputs +from .gate_scopes import uk_scoped_gate_manifest from .graph_build import SPINE_PROVENANCE_TYPE from .graph_calibration import ( UKCalibrationNodes, @@ -134,6 +135,7 @@ UK_ROWWISE_NATIONAL_POSTURE, UKRowwisePosture, ) +from .spine_posture import require_national_spine from .weight_year import uk_dataset_year_frame, uk_weight_projection_factor NATIONAL_TARGET_TYPE = ArtifactType("microcosm.uk.national-target-registry", 1) @@ -259,6 +261,10 @@ def operation_inventory(self) -> dict: NATIONAL_PROBLEM_NODE = "uk.full.national_problem" NATIONAL_GATES_NODE = "uk.full.gates.calibrated" NATIONAL_READBACK_NODE = "uk.full.national.readback" +#: The bound spine's build sidecar, the source the bound checkpoint +#: authenticates against its frame; the national problem reads it for the +#: spine's roster and clone count (microcosm#931). +SPINE_SIDECAR_SOURCE = "uk_spine_evidence" def uk_national_gate_manifest(posture: UKRowwisePosture = UK_ROWWISE_NATIONAL_POSTURE): @@ -284,20 +290,26 @@ def uk_national_graph( """Append the national line to a bound spine checkpoint graph. ``spine`` must end in a node that emits the bound spine's provenance - artifact (:class:`~.graph_build.UKBoundSpineKernel`): the national line - solves the checkpoint as bound, never a sampled or cloned pool. + artifact and reads its build sidecar + (:class:`~.graph_build.UKBoundSpineKernel`): the national line solves the + checkpoint as bound, never a sampled or cloned pool, and its problem reads + the same sidecar to refuse a spine it cannot build on (microcosm#931). """ if posture.role != "national": raise ValueError("uk_national_graph composes the national posture only.") holder = spine.node(spine_population) - if not any( - output.name == "spine_provenance" and output.type == SPINE_PROVENANCE_TYPE - for output in holder.artifact_outputs + if ( + not any( + output.name == "spine_provenance" and output.type == SPINE_PROVENANCE_TYPE + for output in holder.artifact_outputs + ) + or SPINE_SIDECAR_SOURCE not in holder.sources ): raise ValueError( "The national line requires a bound spine checkpoint population " - "that emits its spine_provenance artifact." + "that emits its spine_provenance artifact and reads its build " + f"sidecar ({SPINE_SIDECAR_SOURCE})." ) review_date = date.today().isoformat() if review_date is None else review_date date.fromisoformat(review_date) @@ -337,6 +349,9 @@ def uk_national_graph( UKNationalProblemKernel.ref, population=spine_population, inputs=slices, + # The bound spine's build sidecar, for the refusal that runs + # before anything is resolved (microcosm#931). + sources=(SPINE_SIDECAR_SOURCE,), params={ "time_period": str(config.time_period), "target_weight_rule": doctrine.target_weight_rule, @@ -474,8 +489,9 @@ class _NationalKernel(KernelBase): def implementation_hash(self) -> str: import microcosm.calibrate.artifacts as calibrate_artifacts import microcosm.calibrate.matrix as calibrate_matrix + from microcosm.build import chronicle_epoch - from . import receipt_counts, weighted_integrity + from . import graph_targets, receipt_counts, weighted_integrity from .target_compile_modules import uk_target_compile_modules implementation = source_hash( @@ -508,6 +524,17 @@ def implementation_hash(self) -> str: weighted_integrity, receipt_counts, *uk_target_compile_modules(), + # What both kernels also compute through (microcosm#931): the + # Chronicle schema versions and fact-key epochs the facts are + # checked against (chronicle_epoch), the register codec + # (graph_targets) and the canonical encoding of their artifacts. + # What only the problem computes through, it hashes in its own + # implementation hash below, and the targets compute through + # nothing else; test_uk_graph derives each kernel's set from what + # its code names. + chronicle_epoch, + graph_targets, + canonical_json, ) return hashlib.sha256( canonical_json( @@ -520,6 +547,12 @@ def implementation_hash(self) -> str: class UKNationalTargetKernel(_NationalKernel): + # The national register is compiled through ledger_targets, which checks + # the household-type bindings against the ONS household type domain + # (frs_relationships) before it compiles. That check only refuses, so + # test_uk_graph lists it refusal-only, and this kernel computes through + # nothing its shared hash leaves out: it needs no hash of its own + # (microcosm#931, ruled 9 Nov (late)). ref = "uk.full.national_targets@1" def run(self, context: KernelContext) -> KernelResult: @@ -617,9 +650,11 @@ def materialize_uk_national_rows( With a resolver factory the engine route of :func:`~.full_measure.resolve_uk_full_measures` runs (one block, no local - grains). Without one (``None``) the register materialises from the - frame's own columns, the seam's ``measure_resolver=None`` route: the - hermetic tests' path, and a data-only environment's. + grains), on a frame that must hold one geographic clone: the national + line builds at K = 1 (microcosm#931). Without one (``None``) the register + materialises from the frame's own columns, the seam's + ``measure_resolver=None`` route: the hermetic tests' path, and a + data-only environment's. """ if resolver_factory is None: @@ -653,18 +688,49 @@ def materialize_uk_national_rows( band_edge_registry=band_edge_registry, resolver_factory=resolver_factory, blocks=1, + clones=1, local_grains=(), ) return prepared, restore, rows, dict(evidence) +def require_national_bound_spine(sidecar_path: str | Path) -> None: + """Refuse a bound spine the national line cannot build on (microcosm#931). + + The national line builds on a geography-first spine of one geographic + clone. ``sidecar_path`` is the bound spine's build sidecar, which the + bound checkpoint authenticated against its frame (its clone count among + what it holds to the frame's); this reads only the sidecar's roster and + clone count and refuses the spine through + :func:`~microcosm.build.uk_runtime.spine_posture.require_national_spine`, + whose sentence names K and what the roster lacks, never a row count, an + id or a value; a sidecar that is not JSON is refused in the sentence that + refuses one that is not a JSON object. The national problem, the first + national node that reads the bound frame, calls it before it reads the + frame or resolves anything. + """ + + try: + sidecar = json.loads(Path(sidecar_path).read_bytes()) + except (json.JSONDecodeError, UnicodeError): + raise ValueError( + "The national line builds only on a bound spine whose build " + "sidecar is a JSON object (microcosm#931)." + ) from None + require_national_spine(sidecar) + + class UKNationalProblemKernel(_NationalKernel): """One ordered national problem with the doctrine bound into it. ``resolver_factory`` is instance state like the gate kernels' coverage engine: the engine identity the run stands on is provenance recorded by the driver, and the dependency declaration on the capabilities binds the - installed engine version into the node key. + installed engine version into the node key. The first national node that + reads the bound frame, it refuses, before anything else, a bound spine + that is not geography-first or not of one geographic clone, by the + spine's build sidecar (:func:`require_national_bound_spine`, + microcosm#931). """ ref = "uk.full.national_problem@1" @@ -672,7 +738,71 @@ class UKNationalProblemKernel(_NationalKernel): def __init__(self, *, resolver_factory=UKMeasureResolver) -> None: self.resolver_factory = resolver_factory + def implementation_hash(self) -> str: + import microcosm.calibrate.monetary_binding as monetary_binding + from microcosm.frame import materialize as frame_materialize + from microcosm.frame.adapters import policyengine_uk as uk_engine_adapter + + from .. import gate_battery + from . import ( + annuity_trust_column, + atomic_area_support, + dfe_spring_term, + diagnostics, + frs_annual_income, + graph_population, + hmrc_income, + income_tax_rate_classes, + lcfs_consumption, + local_rowwise, + national_frame, + uc_target_measurements, + ) + + # The problem also computes through what the measure resolver and the + # engine run on the bound frame (microcosm#931), as the dense line's + # measures do: the frame it rebuilds on national_frame's schema + # (graph_population, national_frame), the engine's scratch frame + # (atomic_area_support, frame_materialize and the engine adapter's + # contracts), the resolver's inputs and measures (dfe_spring_term, + # frs_annual_income, hmrc_income, income_tax_rate_classes, + # lcfs_consumption and uc_target_measurements), the national rows + # (local_rowwise), the constraint matrix's monetary check + # (monetary_binding), and the target geography levels (diagnostics) + # and JSON-safe values (gate_battery) its payload records. As the + # dense measures do, it binds the annuity and trust column's name, + # since the resolver reads zeros where a frame lacks the column. The + # name lives in annuity_trust_column, which holds nothing else, so + # this hash leaves frs_spine out, and an edit there does not re-solve + # the national line (microcosm#931, ruled 9 Nov (late)). + return hashlib.sha256( + canonical_json( + { + "national": super().implementation_hash(), + "problem": source_hash( + annuity_trust_column, + atomic_area_support, + dfe_spring_term, + diagnostics, + frs_annual_income, + graph_population, + hmrc_income, + income_tax_rate_classes, + lcfs_consumption, + local_rowwise, + national_frame, + uc_target_measurements, + gate_battery, + monetary_binding, + frame_materialize, + uk_engine_adapter, + ), + } + ) + ).hexdigest() + def run(self, context: KernelContext) -> KernelResult: + require_national_bound_spine(context.sources[SPINE_SIDECAR_SOURCE]) frame = context_frame(context) registry_document = _registry_artifact(context) with tempfile.TemporaryDirectory( @@ -1005,9 +1135,44 @@ class UKNationalGateKernel(_NationalKernel): ) def implementation_hash(self) -> str: - from .. import gate_battery - from . import battery_bindings, diagnostics, receipt_counts, weighted_integrity + import microcosm.calibrate.artifacts as calibrate_artifacts + import microcosm.calibrate.hierarchy as calibrate_hierarchy + import microcosm.calibrate.matrix as calibrate_matrix + import microcosm.calibrate.monetary_binding as monetary_binding + import microcosm.calibrate.registry as calibrate_registry + import microcosm.calibrate.solve as calibrate_solve + import microcosm.calibrate.target as calibrate_target + + from .. import gate_battery, gates + from . import ( + battery_bindings, + cgt_projection, + diagnostics, + donor_uprating, + etb_services, + gate_scopes, + geographic_clones, + graph_population, + graph_targets, + national_frame, + receipt_counts, + weighted_integrity, + ) + # The battery evaluates whichever gates its node's gate scope names, + # so it keys on everything the gate registry's bindings compute + # through (uk_gate_registry_modules; microcosm#931 ruling 6). Its own + # code also computes through the shared gates' evaluators (gates), + # the solved result and registry it decodes (microcosm.calibrate's + # artifacts, matrix, registry, target and hierarchy, graph_targets), + # the frame it rebuilds (graph_population, national_frame), the + # manifest's effective sample size (calibrate.solve), the admin + # totals' derived anchors (etb_services), the CGT projection + # (cgt_projection, donor_uprating) and the canonical encoding of its + # artifacts (microcosm#931). Its support-family fold reads the + # geographic clone index columns the national frame carries from a + # clone-first spine, by the names geographic_clones gives them + # (diagnostics.uk_support_family_geographic_clone_column). return hashlib.sha256( canonical_json( { @@ -1017,11 +1182,29 @@ def implementation_hash(self) -> str: gate_battery, battery_bindings, calibration_run, + gate_scopes, weighted_integrity, # The input-mass gate's source-household counts. receipt_counts, diagnostics, + geographic_clones, support_floor_rules, + gates, + calibrate_artifacts, + calibrate_matrix, + calibrate_registry, + calibrate_target, + calibrate_hierarchy, + monetary_binding, + calibrate_solve, + graph_targets, + graph_population, + national_frame, + etb_services, + cgt_projection, + donor_uprating, + canonical_json, + *uk_gate_registry_modules(), ), "country_resources": load_country_spec("uk").fingerprint, } @@ -1126,9 +1309,40 @@ class UKNationalReadbackKernel(KernelBase): capabilities = Capabilities(Determinism.DETERMINISTIC, role=KernelRole.GATE) def implementation_hash(self) -> str: - from . import content_identity, national_frame, weight_year + from microcosm.frame import materialize as frame_materialize + from microcosm.frame.adapters import policyengine_uk as uk_engine_adapter + + from .. import artifact_files, trace + from . import ( + content_identity, + geography_ladder, + graph_population, + hmrc_uprating, + national_frame, + weight_year, + ) - return source_hash(type(self), content_identity, national_frame, weight_year) + # The readback also computes through the file record it takes + # (artifact_files, trace's file digest), the H5 it reads back + # (frame_materialize, the engine adapter's contracts), the export + # frame's area codes (geography_ladder), the frame it rebuilds + # (graph_population), the engine's weight projection factor + # (hmrc_uprating's pins) and the canonical encoding of its report + # (microcosm#931). + return source_hash( + type(self), + content_identity, + national_frame, + weight_year, + artifact_files, + trace, + frame_materialize, + uk_engine_adapter, + geography_ladder, + graph_population, + hmrc_uprating, + canonical_json, + ) def run(self, context: KernelContext) -> KernelResult: from ..artifact_files import file_artifact @@ -1139,11 +1353,13 @@ def run(self, context: KernelContext) -> KernelResult: written, _provenance = load_uk_national_frame(path) failures = [] # The writer drops the reviewed export exclusions (microcosm#1063 c9) - # and writes the weights at the dataset's year (microcosm#796 A19), so - # the expectation is that export frame, not the raw population. + # and, on the national line, the geographic clone indices of its + # one-clone spine (microcosm#931), and writes the weights at the + # dataset's year (microcosm#796 A19), so the expectation is that + # export frame, not the raw population. solve_year = int(context.params["solve_year"]) release = uk_dataset_year_frame( - uk_release_export_frame(frame), solve_year=solve_year + uk_release_export_frame(frame, line="national"), solve_year=solve_year ) expected = uk_frame_content_identity(release) actual = uk_frame_content_identity(written) @@ -1372,7 +1588,7 @@ def replay_uk_dense_gate_battery( """Write the local gate report the dense release contract verifies. The graph evaluated every UK gate once (``uk.full.gates.calibrated``); - the dense line ships the seven-gate local battery, signed and bound to its + the dense line ships the six-gate local battery, signed and bound to its Logbook attempt. The stored terminal outcomes are projected onto the local scope in the scoped manifest's order — nothing is re-evaluated: both manifests hold the same frozen declarations, and the projection diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py index 8c354334b1..8e7e7197aa 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_population.py @@ -1,27 +1,24 @@ -"""Population operations in the single UK full-build graph. - -Source interpretation stays in UK adapters. Selection, row ancestry, typed -weights, content storage and replay are enforced by the shared graph runtime. -The legacy numerical functions remain the only implementations of the draws. +"""The UK full build's pool checkpoint and the population helpers its nodes share. + +The dense line builds only on a geography-first spine +(:mod:`~microcosm.build.uk_runtime.spine_posture`), as that spine stands: its +``geographic_expand`` and ``geographic_identity`` stages give every household +its geographic clones and keys, its geography stages assign each clone its +area, and its root samples at the spine's rung, so the full build samples, +expands, keys and places nothing of its own (microcosm#931). Its targets and +measures read the spine's version, and :class:`UKPoolCheckpointKernel` keeps +every row of it before the solve. Selection, row ancestry, typed weights, +content storage and replay are enforced by the shared graph runtime. """ from __future__ import annotations -import hashlib -import importlib.util -import json from collections.abc import Mapping -from dataclasses import asdict, replace -from pathlib import Path -import numpy as np import pandas as pd -from microcosm.frame import Frame, Weights +from microcosm.frame import Frame from microcosm.graph import ( - ArtifactInput, - ArtifactOutput, - ArtifactType, Capabilities, Determinism, Graph, @@ -29,94 +26,15 @@ KernelContext, KernelRegistry, KernelResult, - Node, Numeric, - Owned, - SeedSource, Slice, - SourceRef, StructuralDelta, - WeightUpdate, compile_graph, source_hash, ) -from microcosm.graph.canonical import canonical_json -from microcosm.graph.population import dtype_for_token -from microcosm.graph.randomness import keyed_uniform -from microcosm.graph.weight_update import weight_update_receipt -from . import ( - atomic_area_support, - atomic_household_identity, - brma_area_layer, - geography_ladder, - local_authority_input, - national_sampling, - private_rent_localisation, - property_localisation, - rfdr_area_layer, - rowwise_dataset, -) -from .geography_ladder import ( - UK_GEOGRAPHY_LADDER_COLUMNS, - derive_uk_ladder_locations, - draw_uk_ladder_locations, - load_uk_oa_ladder, -) -from .household_lineage import ( - HIGH_VALUE_SUPPORT_COPY_COLUMN, - UK_HOUSEHOLD_LINEAGE_COLUMNS, - UK_OPTIONAL_LINEAGE_COLUMNS, - uk_household_clone_paths, -) -from .local_authority_input import ( - UK_LOCAL_AUTHORITY_CODE_COLUMN, - UK_LOCAL_AUTHORITY_INPUT_COLUMN, - UK_LOCAL_AUTHORITY_VINTAGE, - resolve_local_authority_engine_keys, - verify_local_authority_engine_domain, -) +from . import national_frame from .national_frame import UK_NATIONAL_SCHEMA -from .receipt_counts import rounded_count, source_household_ids -from .rowwise_dataset import expand_uk_geographic_pool, ladder_clone_index_column - -POPULATION_RECEIPT_TYPE = ArtifactType("microcosm.uk.population-receipt", 1) -LOCATION_DRAW_TYPE = ArtifactType("microcosm.uk.ladder-location-draw", 1) -GEOGRAPHY_GATE_TYPE = ArtifactType("microcosm.uk.geography-gate", 1) -GEOGRAPHY_ASSIGNMENTS = ("atomic", "legacy") -#: Lineage columns the post-clone household identity is keyed on. The support -#: channel, the CGT support split's copy flag and index (microcosm#1045) and -#: the clone flag come from the spine stages; the geographic clone index from -#: ``uk.full.expand``. Calibration K, weights and remapped ids are not inputs. -UK_GEOGRAPHY_IDENTITY_INPUTS = UK_HOUSEHOLD_LINEAGE_COLUMNS -PROPERTY_LOCALISATION_MODES = property_localisation.PROPERTY_LOCALISATION_MODES -PROPERTY_QUANTILE_TYPE = ArtifactType("microcosm.uk.property-quantile-draw", 1) -#: Household columns the post-geography property nodes read and rewrite. -_PROPERTY_VALUE_COLUMNS = ( - "region", - "tenure_type", - "main_residence_value", - "property_wealth", -) -_PROPERTY_BAND_COLUMNS = ("council_tax_band",) -BRMA_LOCALISATION_MODES = brma_area_layer.BRMA_LOCALISATION_MODES -#: Household columns the post-geography BRMA node reads (microcosm#1262). -_BRMA_AREA_COLUMNS = ("atomic_area_system", "brma", "lsoa_code", "oa_code", "region") -PRIVATE_RENT_LOCALISATION_MODES = ( - private_rent_localisation.PRIVATE_RENT_LOCALISATION_MODES -) -RFDR_LOCALISATION_MODES = rfdr_area_layer.RFDR_LOCALISATION_MODES -#: Household columns the post-geography RFDR node reads (microcosm#1418). -_RFDR_AREA_COLUMNS = ("atomic_area_code", "atomic_area_system") -#: Household columns the post-geography private-rent node reads (microcosm#1262). -_PRIVATE_RENT_COLUMNS = ( - "brma", - "household_support_channel", - "num_bedrooms", - "region", - "rent", - "tenure_type", -) def context_frame(context: KernelContext) -> Frame: @@ -164,1549 +82,42 @@ def population_slices(cells: Mapping[tuple[str, str], str]) -> tuple[Slice, ...] ) -def _series(frame: Frame, entity: str, column: str, dtype: str) -> pd.Series: - table = frame.table(entity) - ids = pd.Index( - table[frame.schema.entity_id_column(entity)], - name=frame.schema.entity_id_column(entity), - ) - return pd.Series(table[column].array, index=ids, name=column).astype( - dtype_for_token(dtype) - ) - - -def _mass_records(before: Frame, after: Frame) -> list[dict]: - if after.mass_log[: len(before.mass_log)] != before.mass_log: - raise ValueError( - "A graph population operation replaced its incoming mass ledger." - ) - return [asdict(record) for record in after.mass_log[len(before.mass_log) :]] - - -def _declared_mass(before: Frame, after: Frame) -> dict: - old, new = before.stratum_mass(), after.stratum_mass() - return { - "policy": "declared", - "before": float(old.sum()), - "after": float(new.sum()), - "stratum_before": {k: float(v) for k, v in old.items()}, - "stratum_after": {k: float(v) for k, v in new.items()}, - } - - -class _PopulationKernel(KernelBase): - def implementation_hash(self) -> str: - from . import graph_evidence - - return source_hash( - type(self), - geography_ladder, - national_sampling, - rowwise_dataset, - graph_evidence, - ) - - -class UKFamilySampleKernel(_PopulationKernel): - ref = "uk.full.sample@1" - capabilities = Capabilities( - Determinism.DETERMINISTIC, - seed_source=SeedSource.PARAM, - structural=StructuralDelta.FILTER, - ) - - def run(self, context: KernelContext) -> KernelResult: - from .graph_evidence import require_uk_spine_gate_admission - - require_uk_spine_gate_admission(context) - before = context_frame(context) - fraction = float(context.params["fraction"]) - seed = int(context.params["seed"]) - if fraction == 1.0: - after = before - receipt = { - "fraction": fraction, - "seed": seed, - "sampled": False, - "pre_household_count": before.n("household"), - "post_household_count": before.n("household"), - "rung_token": "f100", - } - else: - after, receipt = national_sampling.sample_uk_spine_frame( - before, fraction=fraction, seed=seed - ) - receipt = {"sampled": True, **receipt} - ids = before.table("person")["person_id"] - keep = pd.Series( - ids.isin(after.table("person")["person_id"]).to_numpy(), - index=pd.Index(ids, name="person_id"), - dtype=bool, - ) - # The sampling function computes the historical normalization once. - # Installing its weights is a separate declared same-kind operation. - payload = { - "receipt": receipt, - "household_ids": after.table("household")["household_id"].tolist(), - "weights": after.weights_for("household").values.tolist(), - "mass_log_append": _mass_records(before, after), - } - return KernelResult(keep=keep, artifacts={"sampling": canonical_json(payload)}) - - -class UKSampleNormalizationKernel(_PopulationKernel): - ref = "uk.full.normalize@1" - capabilities = Capabilities( - Determinism.DETERMINISTIC, structural=StructuralDelta.REWEIGHT - ) - - def run(self, context: KernelContext) -> KernelResult: - frame = context_frame(context) - payload = json.loads(context.artifacts["sampling"].payload) - ids = frame.table("household")["household_id"].tolist() - if ids != payload["household_ids"]: - raise ValueError("Sample normalization household axis changed.") - weights = Weights( - np.asarray(payload["weights"], dtype=np.float64), - kind=context.weights["household"].kind, - ) - after = Frame( - {e: frame.table(e) for e in frame.entities}, - frame.schema, - {"household": weights}, - frame.strata, - metadata=frame.metadata, - ) - return KernelResult( - weights=weights, - receipt={ - "weight_update": weight_update_receipt(ids), - "mass": _declared_mass(frame, after), - "frame_mass_log_append": payload["mass_log_append"], - }, - ) - - -class UKGeographicExpansionKernel(_PopulationKernel): - ref = "uk.full.expand@1" - capabilities = Capabilities( - Determinism.DETERMINISTIC, structural=StructuralDelta.EXPAND - ) - - def run(self, context: KernelContext) -> KernelResult: - before = context_frame(context) - household = before.table("household").copy() - household["household_weight"] = before.weights_for("household").values - pool = expand_uk_geographic_pool( - person=before.table("person"), - benunit=before.table("benunit"), - household=household, - n_clones=int(context.params["n_clones"]), - source_year=int(context.params["source_year"]), - time_period=str(context.params["time_period"]), - household_weight_kind=before.weights_for("household").kind, - mass_log=before.mass_log, - source_lineage_modulus=context.params.get("source_lineage_modulus"), - ) - after = pool.frame - ancestry = {} - for entity in before.entities: - table = after.table(entity) - id_column = before.schema.entity_id_column(entity) - added = table.iloc[before.n(entity) :] - source_ids = ( - added[id_column].to_numpy() - - added[ladder_clone_index_column(entity)].to_numpy() - * pool.id_multiplier - ) - ancestry[entity] = pd.Series( - source_ids, index=pd.Index(added[id_column], name=id_column) - ) - columns = { - (e, c): _series(after, e, c, dtype) - for e, c, dtype in context.params["expand_cells"] - } - return KernelResult( - expand=ancestry, - columns=columns, - weights=after.weights_for("household"), - artifacts={ - "expansion": canonical_json( - {"n_clones": pool.n_clones, "id_multiplier": pool.id_multiplier} - ) - }, - receipt={"frame_mass_log_append": _mass_records(before, after)}, - ) - - -class UKLocationDrawKernel(_PopulationKernel): - ref = "uk.full.locations@1" - capabilities = Capabilities(Determinism.DETERMINISTIC, seed_source=SeedSource.PARAM) - - def run(self, context: KernelContext) -> KernelResult: - ladder = load_uk_oa_ladder(context.sources["uk_ladder"]) - household = context.tables["household"] - indices = draw_uk_ladder_locations( - household, - ladder, - seed=int(context.params["seed"]), - expected_constituency_vintage=str(context.params["constituency_vintage"]), - ) - return KernelResult( - artifacts={ - "locations": canonical_json( - { - "household_ids": household["household_id"].tolist(), - "indices": indices.tolist(), - "seed": int(context.params["seed"]), - "layer_vintages": ladder.layer_vintages, - "constituency_sampling_basis": ladder.metadata[ - "constituency_sampling_basis" - ], - "oa_sampling_basis": ladder.metadata["oa_sampling_basis"], - } - ) - } - ) - - -class UKGeographyMappingKernel(_PopulationKernel): - ref = "uk.full.geography_mapping@1" - capabilities = Capabilities(Determinism.DETERMINISTIC) - - def run(self, context: KernelContext) -> KernelResult: - payload = json.loads(context.artifacts["locations"].payload) - household = context.tables["household"] - if household["household_id"].tolist() != payload["household_ids"]: - raise ValueError("Location draw is bound to a different household axis.") - assigned = derive_uk_ladder_locations( - household, - load_uk_oa_ladder(context.sources["uk_ladder"]), - np.asarray(payload["indices"], dtype=np.int64), - ) - index = pd.Index(household["household_id"], name="household_id") - return KernelResult( - columns={ - (owned.entity, owned.column): pd.Series( - assigned[owned.column].array, index=index, name=owned.column - ).astype(dtype_for_token(owned.dtype)) - for owned in context.node.outputs - } - ) - - -class UKGeographyIdentityKernel(_PopulationKernel): - """Key every post-clone household by its source id and ordered clone path. - - The path is read from the spine's explicit lineage columns, never inferred - from ids, weights or financial values: ``spi_support_channel`` with the - support clone index when the household is an SPI support copy, - ``cgt_support_split`` with the copy index when it is a support copy - (microcosm#1045), ``cgt_incidence_clone`` (ordinal 1) from its flag, - ``geographic_support`` with the pool clone index when it is not the - original copy. Growing K never changes an existing household's key. - """ - - ref = "uk.full.identity@1" - capabilities = Capabilities(Determinism.DETERMINISTIC) - - def implementation_hash(self) -> str: - return source_hash( - type(self), atomic_household_identity, atomic_area_support, rowwise_dataset - ) - - def run(self, context: KernelContext) -> KernelResult: - household = context.tables["household"] - source = str(context.params["source"]) - vintage = str(context.params["source_vintage"]) - source_ids, paths = uk_household_clone_paths(household) - keys = [ - atomic_household_identity.household_draw_key( - source=source, - source_vintage=vintage, - source_household_id=source_id, - clone_path=path, - ) - for source_id, path in zip(source_ids, paths, strict=True) - ] - if len(set(keys)) != len(keys): - raise ValueError("Geography identity keys are not unique.") - index = pd.Index(household["household_id"], name="household_id") - column = atomic_area_support.IDENTITY_COLUMN - return KernelResult( - columns={ - ("household", column): pd.Series( - keys, index=index, name=column, dtype=dtype_for_token("string") - ) - }, - receipt={ - "households": len(keys), - "source": source, - "source_vintage": vintage, - "inputs": list(UK_GEOGRAPHY_IDENTITY_INPUTS), - }, - ) - - -class UKPropertyValueKernel(_PopulationKernel): - """Read each owner clone's home value off its constituency's distribution. - - microcosm#1240 fixes 3 and 5. Every pool clone of one spine household - keeps one uniform (its weighted rank among the owner households of its - FRS region, ties broken by a keyed draw; high-value support copies take a - keyed draw inside the top stratum), and reads its value at that uniform - from the assigned constituency's uprated latest-sale knots. Weights are - untouched; property wealth moves by the value's change. - """ - - ref = "uk.full.property.value@1" - capabilities = Capabilities( - determinism=Determinism.SEEDED, - seed_source=SeedSource.KEYED, - numeric=Numeric.PLATFORM_BITWISE, - ) - - def implementation_hash(self) -> str: - from . import household_lineage - - return source_hash( - type(self), - property_localisation, - household_lineage, - atomic_household_identity, - ) - - def run(self, context: KernelContext) -> KernelResult: - household = context.tables["household"] - weights = np.asarray(context.weights["household"].values, dtype=float) - knots = property_localisation.load_value_knots( - Path(context.sources[property_localisation.QUANTILES_SOURCE]).read_bytes() - ) - stream = tuple(context.params["stream"]) - keys = property_localisation.spine_household_keys( - household, - source=str(context.params["source"]), - source_vintage=str(context.params["source_vintage"]), - ) - support = ( - household[HIGH_VALUE_SUPPORT_COPY_COLUMN].to_numpy().astype(bool) - if HIGH_VALUE_SUPPORT_COPY_COLUMN in household - else None - ) - u, uniform_receipt = property_localisation.owner_uniforms( - spine_key=keys, - region=household["region"].to_numpy(), - tenure_type=household["tenure_type"].to_numpy(), - # The spine's regional values (microcosm#1112) keep the value they - # replaced; ranking on it keeps this node's draws unchanged. - value=property_localisation.main_residence_rank_basis( - household, - declared=bool(context.params.get("rank_basis_declared", False)), - ), - weight=weights, - stream=stream, - support_copy=support, - support_stratum_lower=float(context.params["support_stratum_lower"]), - ) - new_value, new_wealth, receipt = property_localisation.localise_values( - u=u, - constituency_code=household["constituency_code"].to_numpy(), - tenure_type=household["tenure_type"].to_numpy(), - value=household["main_residence_value"].to_numpy(), - property_wealth=household["property_wealth"].to_numpy(), - knots=knots, - ) - index = pd.Index(household["household_id"], name="household_id") - household_ids = [int(value) for value in household["household_id"].to_numpy()] - payload = { - "schema": "microcosm.uk.property-quantile-draw.v1", - "household_ids": household_ids, - "u": [float(value) for value in u], - "cap": knots.cap, - "stream": list(stream), - "method": property_localisation.METHOD_VERSION, - "knots_inputs_sha256": knots.inputs_sha256, - "basis_year": knots.basis_year, - "rows_value_rewritten": receipt["rows_rewritten"], - } - return KernelResult( - columns={ - ("household", "main_residence_value"): pd.Series( - new_value, index=index, name="main_residence_value", dtype="float64" - ), - ("household", "property_wealth"): pd.Series( - new_wealth, index=index, name="property_wealth", dtype="float64" - ), - }, - artifacts={"quantiles": canonical_json(payload)}, - receipt={ - **receipt, - "uniforms": uniform_receipt, - "weights": "unchanged", - "household_mass": float(weights.sum()), - "knots_inputs_sha256": knots.inputs_sha256, - "basis_year": knots.basis_year, - "stream": list(stream), - }, - ) - - -class UKPropertyBandKernel(_PopulationKernel): - """Draw each England-and-Wales clone's council-tax band from its LSOA's VOA mix. - - microcosm#1240 fix 4: the band is read at the same uniform the value node - placed the home value with, so a clone's value and band agree by - construction. Rows outside England and Wales keep the spine's band. - """ - - ref = "uk.full.property.band@1" - capabilities = Capabilities( - determinism=Determinism.DETERMINISTIC, numeric=Numeric.PLATFORM_BITWISE - ) - - def implementation_hash(self) -> str: - return source_hash(type(self), property_localisation) - - def run(self, context: KernelContext) -> KernelResult: - household = context.tables["household"] - payload = json.loads(context.artifacts["quantiles"].payload) - if payload.get("schema") != "microcosm.uk.property-quantile-draw.v1": - raise ValueError( - "Property band node received an unknown quantile artifact." - ) - ids = [int(value) for value in household["household_id"].to_numpy()] - if payload["household_ids"] != ids: - raise ValueError( - "Property quantile draws do not align with the pool households." - ) - u = np.asarray(payload["u"], dtype=float) - mix = property_localisation.load_band_mix( - Path(context.sources[property_localisation.BAND_MIX_SOURCE]).read_bytes() - ) - scotland_mix = property_localisation.load_scotland_band_mix( - Path( - context.sources[property_localisation.SCOTLAND_BAND_MIX_SOURCE] - ).read_bytes() - ) - band_dtype = household["council_tax_band"].dtype - tenure = household["tenure_type"].to_numpy() - new_band, receipt = property_localisation.localise_bands( - u=u, - lsoa_code=household["lsoa_code"].to_numpy(), - band=household["council_tax_band"].to_numpy(), - tenure_type=tenure, - mix=mix, - scotland_mix=scotland_mix, - local_authority_code=household["local_authority_code"].to_numpy(), - ) - lsoa = household["lsoa_code"].to_numpy().astype(str) - area = household["local_authority_code"].to_numpy().astype(str) - owner = property_localisation.owner_mask(tenure) - england_wales = np.char.startswith(lsoa, "E") | np.char.startswith(lsoa, "W") - scotland = np.char.startswith(area, "S12") & ~england_wales - # Each mix is monotone within its area: an LSOA in England and Wales, - # a council area in Scotland. - violations = property_localisation.monotone_violations( - u[england_wales & owner], - new_band[england_wales & owner], - lsoa[england_wales & owner], - ) + property_localisation.monotone_violations( - u[scotland & owner], new_band[scotland & owner], area[scotland & owner] - ) - if violations: - raise ValueError("Property band draws are not monotone in the uniform.") - index = pd.Index(household["household_id"], name="household_id") - return KernelResult( - columns={ - ("household", "council_tax_band"): pd.Series( - new_band, index=index, name="council_tax_band" - ).astype(band_dtype), - }, - receipt={ - **receipt, - "monotone_violations": violations, - "band_mix_sha256": mix.source_sha256, - "scotland_band_mix_facts_sha256": scotland_mix.source_sha256, - }, - ) - - -class UKLocalAuthorityResolveKernel(_PopulationKernel): - """Resolve the engine's ``local_authority`` input from the derived code. +class UKPoolCheckpointKernel(KernelBase): + """Checkpoint the bound spine before the solve, keeping every row. - The shared derive step writes ``local_authority_code``; the engine input - is the member name resolved fail-closed on the April 2023 roster - (microcosm#953), exactly as the legacy ladder derive writes it. The - engine itself is never imported here; the UK distribution gate checks - the assigned names against the installed enum. - """ - - ref = "uk.full.geography.local_authority@1" - capabilities = Capabilities(Determinism.DETERMINISTIC) - - def implementation_hash(self) -> str: - return source_hash(type(self), local_authority_input) - - def run(self, context: KernelContext) -> KernelResult: - household = context.tables["household"] - keys = resolve_local_authority_engine_keys( - household[UK_LOCAL_AUTHORITY_CODE_COLUMN] - ) - index = pd.Index(household["household_id"], name="household_id") - column = UK_LOCAL_AUTHORITY_INPUT_COLUMN - return KernelResult( - columns={ - ("household", column): pd.Series( - keys, index=index, name=column, dtype=dtype_for_token("string") - ) - }, - receipt={ - "households": int(len(keys)), - "local_authority_vintage": UK_LOCAL_AUTHORITY_VINTAGE, - }, - ) - - -class UKBRMAAreaKernel(_PopulationKernel): - """Read each placed Great Britain household's BRMA off its assigned area. - - microcosm#1262: the spine's ``frs_brma`` stage draws a BRMA within the - household's FRS region before any household has an area, so every clone - carried one BRMA wherever it landed. The committed layer places every - England-and-Wales LSOA and Scottish output area in one BRMA, so a clone's - BRMA, LHA rate, local authority and constituency describe one place. - Northern Ireland has no open BRMA geography and keeps the spine's draw. - """ - - ref = "uk.full.geography.brma@1" - capabilities = Capabilities( - determinism=Determinism.DETERMINISTIC, numeric=Numeric.PLATFORM_BITWISE - ) - - def implementation_hash(self) -> str: - return source_hash(type(self), brma_area_layer) - - def run(self, context: KernelContext) -> KernelResult: - household = context.tables["household"] - payload = Path( - context.sources[brma_area_layer.BRMA_AREA_LAYER_SOURCE] - ).read_bytes() - layer = brma_area_layer.load_brma_area_layer(payload) - brma, counts = brma_area_layer.area_brmas( - layer=layer, - area_system=household["atomic_area_system"].to_numpy(), - lsoa_code=household["lsoa_code"].to_numpy(), - oa_code=household["oa_code"].to_numpy(), - region=household["region"].to_numpy(), - brma=household["brma"].to_numpy(), - ) - index = pd.Index(household["household_id"], name="household_id") - return KernelResult( - columns={ - ("household", "brma"): pd.Series(brma, index=index, name="brma").astype( - household["brma"].dtype - ), - }, - receipt={ - "rows_by_grain": { - grain: rounded_count(count) - for grain, count in sorted(counts.items()) - if grain != "changed" - }, - "rows_changed": rounded_count(counts["changed"]), - "layer_sha256": hashlib.sha256(payload).hexdigest(), - "method": layer.method, - }, - ) - - -class UKRFDRAreaKernel(_PopulationKernel): - """Set each placed household's Rural Fuel Duty Relief flag off its output area. - - microcosm#1418: policyengine-uk relieves a household's petrol and diesel - duty when ``in_rural_fuel_duty_relief_area`` is true. The committed layer - lists the Great Britain output areas in the scheme (HMRC Notice 2001's - islands, post town and postcode districts, by the ONS postcode lookup), so - a clone in one of them carries the flag. Northern Ireland has no area in - the scheme. - """ - - ref = "uk.full.geography.rfdr@1" - capabilities = Capabilities( - determinism=Determinism.DETERMINISTIC, numeric=Numeric.PLATFORM_BITWISE - ) - - def implementation_hash(self) -> str: - return source_hash(type(self), rfdr_area_layer) - - def run(self, context: KernelContext) -> KernelResult: - household = context.tables["household"] - payload = Path( - context.sources[rfdr_area_layer.RFDR_AREA_LAYER_SOURCE] - ).read_bytes() - layer = rfdr_area_layer.load_rfdr_area_layer(payload) - strict = bool(context.params.get("strict_flags", True)) - flags, counts = rfdr_area_layer.area_flags( - layer=layer, - area_system=household["atomic_area_system"].to_numpy(), - area_code=household["atomic_area_code"].to_numpy(), - require_flagged=strict, - ) - index = pd.Index(household["household_id"], name="household_id") - column = rfdr_area_layer.RFDR_COLUMN - return KernelResult( - columns={ - ("household", column): pd.Series(flags, index=index, name=column), - }, - receipt={ - "rows_by_system": { - system: rounded_count(count) - for system, count in sorted(counts.items()) - if system != "flagged" - }, - "rows_flagged": rounded_count(counts["flagged"]), - "zero_flag_refusal": strict, - "layer_sha256": hashlib.sha256(payload).hexdigest(), - "method": layer.method, - }, - ) - - -class UKPrivateRentKernel(_PopulationKernel): - """Read each Great Britain private renter's rent off its BRMA's list of rents. - - microcosm#1262: the spine's rent describes the source household's FRS - region. Every private renter with a positive rent, in either channel, - keeps its rank among the FRS renters of its region and bedroom size (one - keyed tie-break per spine household, shared by all of its pool clones) - and reads that quantile off its assigned BRMA's list for homes of its - size, scaled so the region's mean rent is the FRS level on the census - private-renter mix. Placement follows all households, so at design - weights the level moves with the gap between the two mixes, which - calibration's tenure rows close. Weights are untouched; owners, social - renters, zero rents and Northern Ireland keep their rents bit for bit. The - region scales and ranks stay inside the kernel; the receipt carries - rounded record counts only. - """ - - ref = "uk.full.private_rent@1" - capabilities = Capabilities( - determinism=Determinism.SEEDED, - seed_source=SeedSource.KEYED, - numeric=Numeric.PLATFORM_BITWISE, - ) - - def implementation_hash(self) -> str: - from . import household_lineage - - return source_hash( - type(self), - private_rent_localisation, - property_localisation, - household_lineage, - atomic_household_identity, - ) - - def run(self, context: KernelContext) -> KernelResult: - household = context.tables["household"] - weights = np.asarray(context.weights["household"].values, dtype=float) - quantiles_payload = Path( - context.sources[private_rent_localisation.QUANTILES_SOURCE] - ).read_bytes() - counts_payload = Path( - context.sources[private_rent_localisation.COUNTS_SOURCE] - ).read_bytes() - quantiles = private_rent_localisation.load_rent_quantiles(quantiles_payload) - counts = private_rent_localisation.load_private_renter_counts(counts_payload) - stream = tuple(context.params["stream"]) - keys = property_localisation.spine_household_keys( - household, - source=str(context.params["source"]), - source_vintage=str(context.params["source_vintage"]), - ) - mask = private_rent_localisation.redraw_mask( - household["region"].to_numpy(), - household["tenure_type"].to_numpy(), - household["rent"].to_numpy(), - ) - redrawn = sorted({str(key) for key in keys[mask]}) - draws = keyed_uniform( - stream=stream, keys=[(key, "rent_rank") for key in redrawn] - ) - new_rent, details = private_rent_localisation.localise_rents( - spine_key=keys, - source=source_household_ids(household, stage="uk.full.private_rent"), - region=household["region"].to_numpy(), - tenure_type=household["tenure_type"].to_numpy(), - rent=household["rent"].to_numpy(), - num_bedrooms=household["num_bedrooms"].to_numpy(), - brma=household["brma"].to_numpy(), - channel=household["household_support_channel"].to_numpy(), - weight=weights, - tie_uniform=dict(zip(redrawn, (float(v) for v in draws), strict=True)), - quantiles=quantiles, - counts=counts, - strict=bool(context.params.get("strict_cells", True)), - ) - index = pd.Index(household["household_id"], name="household_id") - return KernelResult( - columns={ - ("household", "rent"): pd.Series( - new_rent, index=index, name="rent" - ).astype(household["rent"].dtype), - }, - receipt={ - "rows_redrawn": rounded_count(details["rows_redrawn"]), - "spine_households_redrawn": rounded_count( - details["spine_households_redrawn"] - ), - "cells_ranked_against_their_region": details["fallback_cells"], - "cells_ranked_against_great_britain": details["great_britain_cells"], - "regions_scaled": details["regions"], - "regions_scaled_on_great_britain": details[ - "regions_scaled_on_great_britain" - ], - "weights": "unchanged", - "quantiles_sha256": hashlib.sha256(quantiles_payload).hexdigest(), - "counts_sha256": hashlib.sha256(counts_payload).hexdigest(), - "method": private_rent_localisation.METHOD_VERSION, - "stream": list(stream), - }, - ) - - -class UKPoolCheckpointKernel(_PopulationKernel): - """Checkpoint the complete pool; refuse unless the shared atomic gate passed. - - The shared ``geography.gate@1`` result is a platform-scoped typed artifact - (amendment 17), and a bitwise UK kernel consuming it as a typed input - would be refused. This FILTER node consumes it and emits no typed - artifact of its own, so that refusal is avoided here. It is only the - typed-artifact edge that stops: the shared assignment kernels are - ``platform_bitwise`` too, and their columns reach the UK target and - calibration chain through population slices, which the executor already - reads at platform scope. Whether the shared kernels can honestly declare - ``bitwise`` (exact keyed draws, integer inverse CDF) is the open question - recorded on microcosm#932. + A keep-all FILTER on the spine's version: the dense pool is the bound + spine, its geographic clones and areas with it (microcosm#931). The node's + base is that version, so it follows every operation on it, the target + compilation, selection, measures and problem among them, and it binds the + complete incoming ledger before the calibration nodes reconstruct any + Frame slices. It reads and emits no typed artifact. """ ref = "uk.full.pool@1" capabilities = Capabilities( Determinism.DETERMINISTIC, - numeric=Numeric.PLATFORM_BITWISE, + numeric=Numeric.BITWISE, structural=StructuralDelta.FILTER, ) + def implementation_hash(self) -> str: + # The checkpoint reads the person axis national_frame's schema names, + # and nothing else (microcosm#931). + return source_hash(type(self), national_frame) + def run(self, context: KernelContext) -> KernelResult: - receipt = {"scope": "complete_pool_checkpoint"} - if "atomic_validation" in context.artifacts: - validation = json.loads(context.artifacts["atomic_validation"].payload) - if validation.get("outcome") != "pass": - raise ValueError("Shared atomic geography validation did not pass.") - receipt["atomic_validation"] = validation - if "property_quantiles" in context.artifacts: - quantiles = json.loads(context.artifacts["property_quantiles"].payload) - household_ids = [ - int(value) - for value in context.tables["household"]["household_id"].to_numpy() - ] - if quantiles.get("household_ids") != household_ids: - raise ValueError( - "Property quantile draws do not align with the checkpointed pool." - ) - receipt["property_localisation"] = { - key: quantiles[key] - for key in ( - "cap", - "stream", - "method", - "knots_inputs_sha256", - "basis_year", - "rows_value_rewritten", - ) - } person = context.tables[UK_NATIONAL_SCHEMA.person_entity] id_column = UK_NATIONAL_SCHEMA.person_id_column ids = pd.Index(person[id_column].to_numpy(copy=True), name=id_column) return KernelResult( - keep=pd.Series(True, index=ids, dtype="bool"), receipt=receipt - ) - - -def uk_pool_validation_inputs( - geography_assignment: str, property_localisation_mode: str = "none" -) -> tuple[ArtifactInput, ...]: - """The typed edges that order the shared atomic gate (and the property draw) before the checkpoint.""" - if geography_assignment != "atomic": - return () - from microcosm.build.graph_atomic_geography import ( - ATOMIC_GEOGRAPHY_VALIDATION_TYPE, - ) - - inputs = ( - ArtifactInput( - "atomic_validation", - "uk.full.geography.gate", - "validation", - ATOMIC_GEOGRAPHY_VALIDATION_TYPE, - ), - ) - if property_localisation_mode == "price_paid": - inputs = ( - *inputs, - ArtifactInput( - "property_quantiles", - "uk.full.property.value", - "quantiles", - PROPERTY_QUANTILE_TYPE, - ), + keep=pd.Series(True, index=ids, dtype="bool"), + receipt={"scope": "complete_pool_checkpoint"}, ) - return inputs - - -class UKGeographyGateKernel(_PopulationKernel): - ref = "uk.full.geography_gate@1" - capabilities = Capabilities(Determinism.DETERMINISTIC) - - def run(self, context: KernelContext) -> KernelResult: - frame = context_frame(context) - household = frame.table("household") - gate = geography_ladder.uk_geography_ladder_gate( - household, frame.weights_for("household").values - ) - # Fail closed against the installed engine (microcosm#953): every - # assigned local_authority member name must exist in this pin's - # LocalAuthority enum. Only assigned keys are checked so a toy ladder - # stays evaluable and an engine-free lane never imports the engine. - if gate.passed and importlib.util.find_spec("policyengine_uk") is not None: - verify_local_authority_engine_domain(household["local_authority"].unique()) - # Persist the failure. Downstream target materialization explicitly - # refuses this outcome; cached failure must never turn into a pass. - payload = asdict(gate) - return KernelResult( - artifacts={"gate": canonical_json(payload)}, - receipt={"outcome": "pass" if gate.passed else "fail", "evidence": payload}, - ) - - -def append_uk_population_nodes( - graph: Graph, - *, - population: str, - time_period: str, - weight_kind: str, - sample_fraction: float = 1.0, - sample_seed: int = national_sampling.UK_SAMPLE_SEED_DEFAULT, - n_clones: int = 1, - seed: int = 42, - source_year: int = 2024, - constituency_vintage: str = "2024_pcon", - source_lineage_modulus: int | None = None, - geography_assignment: str = "atomic", - atomic_geography_definition: Mapping | None = None, - source_vintage: str | None = None, - identity_source: str = "frs", - property_localisation_mode: str = "none", - brma_localisation_mode: str = "none", - private_rent_localisation_mode: str = "none", - rfdr_localisation_mode: str = "none", - full_scale: bool = True, -) -> Graph: - """Compose sampling → replication → identity → assignment → derivation → gate. - - ``property_localisation_mode="price_paid"`` (atomic assignment only) adds - the microcosm#1240 value and band nodes between the local-authority - resolve and the gates; ``"none"`` leaves the spine's values and bands on - every clone. - - ``geography_assignment="atomic"`` (the default) keys every post-clone - household by its lineage and runs the shared atomic-geography operators - on the three UK support artifacts declared by ``atomic_geography_definition`` - (``uk_atomic_assignment_definition``). ``"legacy"`` keeps the sequential - ladder draw for measurement builds only. - """ - - from .graph_kernels import UKClaimKernel - - cells = population_columns(graph, population) - if type(n_clones) is not int or n_clones < 1: - raise ValueError("Geographic replicate count K must be a positive integer.") - if geography_assignment not in GEOGRAPHY_ASSIGNMENTS: - raise ValueError( - f"Geography assignment must be one of {GEOGRAPHY_ASSIGNMENTS}." - ) - if geography_assignment == "atomic": - if atomic_geography_definition is None: - raise ValueError( - "Atomic geography assignment requires the UK assignment definition." - ) - if not isinstance(source_vintage, str) or not source_vintage: - raise ValueError("Atomic geography assignment requires the FRS vintage.") - elif atomic_geography_definition is not None: - raise ValueError("Legacy geography assignment takes no atomic definition.") - if property_localisation_mode not in PROPERTY_LOCALISATION_MODES: - raise ValueError( - f"Property localisation must be one of {PROPERTY_LOCALISATION_MODES}." - ) - if property_localisation_mode != "none" and geography_assignment != "atomic": - raise ValueError("Property localisation needs the atomic geography assignment.") - if brma_localisation_mode not in BRMA_LOCALISATION_MODES: - raise ValueError(f"BRMA localisation must be one of {BRMA_LOCALISATION_MODES}.") - if brma_localisation_mode != "none" and geography_assignment != "atomic": - raise ValueError("Area BRMAs need the atomic geography assignment.") - if private_rent_localisation_mode not in PRIVATE_RENT_LOCALISATION_MODES: - raise ValueError( - f"Private rent localisation must be one of {PRIVATE_RENT_LOCALISATION_MODES}." - ) - if private_rent_localisation_mode != "none" and brma_localisation_mode != "area": - raise ValueError("Private rents from the lists need the area BRMAs.") - if rfdr_localisation_mode not in RFDR_LOCALISATION_MODES: - raise ValueError(f"RFDR localisation must be one of {RFDR_LOCALISATION_MODES}.") - if rfdr_localisation_mode != "none" and geography_assignment != "atomic": - raise ValueError("Area RFDR flags need the atomic geography assignment.") - national_sampling.validate_sample_fraction(sample_fraction, label="UK full build") - national_sampling.validate_sample_seed(sample_seed, label="UK full build") - common = {"time_period": str(time_period)} - spine_gate_inputs = () - spine_gate_params = {} - if any(node.id == "spine.gates.transferred" for node in graph.nodes): - from .graph_evidence import SPINE_GATE_REPORT_TYPE - - gate = graph.node("spine.gates.transferred") - spine_gate_inputs = ( - ArtifactInput("spine_gate", gate.id, "gate_report", SPINE_GATE_REPORT_TYPE), - ) - spine_gate_params = { - "spine_gate_phase": "transferred", - "spine_gate_release_candidate": bool(gate.params["release_candidate"]), - "spine_gate_synthetic_smoke": bool(gate.params["synthetic_smoke"]), - } - nodes = list(graph.nodes) - nodes.append( - Node( - id="uk.full.sample", - kernel=UKFamilySampleKernel.ref, - inputs=population_slices(cells), - base=population, - structural=StructuralDelta.FILTER, - mass="free", - params={ - **common, - "fraction": float(sample_fraction), - "seed": sample_seed, - **spine_gate_params, - }, - artifact_inputs=spine_gate_inputs, - artifact_outputs=(ArtifactOutput("sampling", POPULATION_RECEIPT_TYPE),), - description="Select intact source families; calculate their historical mass normalization.", - ) - ) - nodes.append( - Node( - id="uk.full.normalize", - kernel=UKSampleNormalizationKernel.ref, - inputs=population_slices(cells), - base="uk.full.sample", - structural=StructuralDelta.REWEIGHT, - mass="declared", - weights=WeightUpdate( - "household", weight_kind, "Normalize sampled source-family mass." - ), - artifact_inputs=( - ArtifactInput( - "sampling", "uk.full.sample", "sampling", POPULATION_RECEIPT_TYPE - ), - ), - params=common, - description="Install normalized weights without changing their kind.", - ) - ) - expansion_cells = { - ("household", "source_household_id"): "int64", - ("household", "source_year"): "int64", - ("household", "source_household_key"): "string", - **{ - (entity, ladder_clone_index_column(entity)): "int64" - for entity in UK_NATIONAL_SCHEMA.entities - }, - } - # Existing source lineage is carried unchanged, except the explicitly - # requested historical modulus conversion, which owns its declarations. - if source_lineage_modulus is not None: - expansion_cells.update( - {("household", rowwise_dataset.POOL_SOURCE_LINEAGE_COLUMN): "int64"} - ) - expansion_cells = { - key: cells.get(key, dtype) for key, dtype in expansion_cells.items() - } - nodes.append( - Node( - id="uk.full.expand", - kernel=UKGeographicExpansionKernel.ref, - inputs=population_slices(cells), - structural=StructuralDelta.EXPAND, - base="uk.full.normalize", - mass="conserve", - params={ - **common, - "source_year": source_year, - "n_clones": n_clones, - "source_lineage_modulus": source_lineage_modulus, - "expand_weight_entity": "household", - "expand_weight_kind": weight_kind, - "expand_cells": tuple( - (e, c, dtype) for (e, c), dtype in sorted(expansion_cells.items()) - ), - }, - artifact_outputs=(ArtifactOutput("expansion", POPULATION_RECEIPT_TYPE),), - description="Prepare source lineage and expand linked entities into K geographic copies.", - ) - ) - nodes.append( - Node( - id="uk.full.expand.owned", - kernel=UKClaimKernel.ref, - population="uk.full.expand", - outputs=tuple( - Owned(e, c, dtype, rewrite=(e, c) in cells) - for (e, c), dtype in sorted(expansion_cells.items()) - ), - params={ - "materialized_expand_outputs": tuple( - f"{e}.{c}" for e, c in expansion_cells if (e, c) not in cells - ) - }, - description="Declare the lineage and replicate columns materialized by expansion.", - ) - ) - cells.update(expansion_cells) - if geography_assignment == "legacy": - geography_nodes, geography_sources = _legacy_geography_nodes( - cells, common=common, seed=seed, constituency_vintage=constituency_vintage - ) - else: - geography_nodes, geography_sources = _atomic_geography_nodes( - cells, - common=common, - seed=seed, - definition=atomic_geography_definition, - identity_source=identity_source, - source_vintage=source_vintage, - property_localisation_mode=property_localisation_mode, - brma_localisation_mode=brma_localisation_mode, - private_rent_localisation_mode=private_rent_localisation_mode, - rfdr_localisation_mode=rfdr_localisation_mode, - full_scale=full_scale, - ) - nodes.extend(geography_nodes) - declared = {s.name for s in graph.sources} - sources = ( - *graph.sources, - *(ref for ref in geography_sources if ref.name not in declared), - ) - return replace(graph, nodes=tuple(nodes), sources=tuple(sources)) - - -def _legacy_geography_nodes( - cells: dict, *, common: Mapping, seed: int, constituency_vintage: str -) -> tuple[tuple[Node, ...], tuple[SourceRef, ...]]: - """The sequential ladder draw, kept verbatim for measurement builds.""" - - nodes = ( - Node( - id="uk.full.locations", - kernel=UKLocationDrawKernel.ref, - population="uk.full.expand", - inputs=(Slice("household", ("region",)),), - sources=("uk_ladder",), - params={"seed": seed, "constituency_vintage": constituency_vintage}, - artifact_outputs=(ArtifactOutput("locations", LOCATION_DRAW_TYPE),), - description="Draw constituency then atomic area with the current sequential RNG.", - ), - Node( - id="uk.full.geography_mapping", - kernel=UKGeographyMappingKernel.ref, - population="uk.full.expand", - inputs=(Slice("household", ("region",)),), - sources=("uk_ladder",), - artifact_inputs=( - ArtifactInput( - "locations", "uk.full.locations", "locations", LOCATION_DRAW_TYPE - ), - ), - outputs=tuple( - Owned("household", col, "string", rewrite=("household", col) in cells) - for col in UK_GEOGRAPHY_LADDER_COLUMNS - ), - description="Derive every geography from the drawn atomic-area index.", - ), - ) - cells.update({("household", col): "string" for col in UK_GEOGRAPHY_LADDER_COLUMNS}) - gate = Node( - id="uk.full.geography_gate", - kernel=UKGeographyGateKernel.ref, - population="uk.full.expand", - inputs=population_slices(cells), - params=dict(common), - artifact_outputs=(ArtifactOutput("gate", GEOGRAPHY_GATE_TYPE),), - description="Validate geography integrity before selected-target contributions.", - ) - ladder = SourceRef( - "uk_ladder", "raw-bytes-v1", "Full UK atomic-area ladder with pinned vintages." - ) - return (*nodes, gate), (ladder,) - - -def _atomic_geography_nodes( - cells: dict, - *, - common: Mapping, - seed: int, - definition: Mapping, - identity_source: str, - source_vintage: str, - property_localisation_mode: str = "none", - brma_localisation_mode: str = "none", - private_rent_localisation_mode: str = "none", - rfdr_localisation_mode: str = "none", - full_scale: bool = True, -) -> tuple[tuple[Node, ...], tuple[SourceRef, ...]]: - """Identity-keyed assignment on the shared operators (release path order).""" - - from microcosm.build.atomic_geography import validate_assignment_spec - from microcosm.build.graph_atomic_geography import atomic_geography_nodes - - spec = validate_assignment_spec(definition) - if spec["identity"] != [atomic_area_support.IDENTITY_COLUMN]: - raise ValueError("UK atomic assignment must key on the geography identity.") - if spec["stream"][3] != seed: - raise ValueError("UK atomic assignment seed differs from the build seed.") - if {system["source"] for system in spec["systems"]} != set( - atomic_area_support.SOURCES.values() - ): - raise ValueError("UK atomic assignment must declare the three UK supports.") - lineage_inputs = tuple( - sorted( - column - for column in (*UK_HOUSEHOLD_LINEAGE_COLUMNS, *UK_OPTIONAL_LINEAGE_COLUMNS) - if ("household", column) in cells - ) - ) - identity = Node( - id="uk.full.identity", - kernel=UKGeographyIdentityKernel.ref, - population="uk.full.expand", - inputs=(Slice("household", lineage_inputs),), - outputs=(Owned("household", atomic_area_support.IDENTITY_COLUMN, "string"),), - params={ - **common, - "source": identity_source, - "source_vintage": source_vintage, - }, - description="Key every post-clone household by its source id and ordered clone path.", - ) - cells[("household", atomic_area_support.IDENTITY_COLUMN)] = "string" - shared = atomic_geography_nodes( - spec, - tuple(Owned(e, c, dtype) for (e, c), dtype in sorted(cells.items())), - base="uk.full.expand", - prefix="uk.full.geography", - emit_validation_artifact=True, - ) - for node in shared: - cells.update({(o.entity, o.column): o.dtype for o in node.outputs}) - # The engine's local_authority enum input (microcosm#953) is UK-side: the - # shared derive writes the GSS code, this node resolves the member name - # the export gate and the engine require, between derive and the gates. - local_authority = Node( - id="uk.full.geography.local_authority", - kernel=UKLocalAuthorityResolveKernel.ref, - population="uk.full.expand", - inputs=(Slice("household", (UK_LOCAL_AUTHORITY_CODE_COLUMN,)),), - outputs=(Owned("household", UK_LOCAL_AUTHORITY_INPUT_COLUMN, "string"),), - params=dict(common), - description="Resolve the engine local_authority input from the derived GSS code.", - ) - cells[("household", UK_LOCAL_AUTHORITY_INPUT_COLUMN)] = "string" - brma_nodes: tuple[Node, ...] = () - brma_sources: tuple[SourceRef, ...] = () - if brma_localisation_mode == "area": - brma_nodes, brma_sources = _brma_area_nodes(cells, common=common) - if private_rent_localisation_mode == "brma_list": - rent_nodes, rent_sources = _private_rent_nodes( - cells, - common=common, - seed=seed, - identity_source=identity_source, - source_vintage=source_vintage, - lineage_inputs=lineage_inputs, - strict=full_scale, - ) - brma_nodes = (*brma_nodes, *rent_nodes) - brma_sources = (*brma_sources, *rent_sources) - rfdr_nodes: tuple[Node, ...] = () - rfdr_sources: tuple[SourceRef, ...] = () - if rfdr_localisation_mode == "area": - rfdr_nodes, rfdr_sources = _rfdr_area_nodes( - cells, common=common, strict=full_scale - ) - property_nodes: tuple[Node, ...] = () - property_sources: tuple[SourceRef, ...] = () - if property_localisation_mode == "price_paid": - property_nodes, property_sources = _property_localisation_nodes( - cells, - common=common, - seed=seed, - identity_source=identity_source, - source_vintage=source_vintage, - lineage_inputs=lineage_inputs, - ) - # The shared gate's typed validation artifact is platform-scoped, so it is - # consumed by the pool checkpoint (uk.full.pool, no typed outputs), not by - # this distribution gate, whose own artifact feeds the bitwise UK chain. - gate = Node( - id="uk.full.geography_gate", - kernel=UKGeographyGateKernel.ref, - population="uk.full.expand", - inputs=population_slices(cells), - params=dict(common), - artifact_outputs=(ArtifactOutput("gate", GEOGRAPHY_GATE_TYPE),), - description="Validate the UK geography distribution on the identity-keyed assignment.", - ) - sources = ( - SourceRef( - "uk_ladder", - "raw-bytes-v1", - "Full UK atomic-area ladder with pinned vintages.", - ), - *( - SourceRef( - atomic_area_support.SOURCES[system], - "raw-bytes-v1", - f"UK atomic-area support artifact for {system}.", - ) - for system in atomic_area_support.SYSTEMS - ), - *property_sources, - *brma_sources, - *rfdr_sources, - ) - return ( - identity, - *shared[:-1], - local_authority, - *brma_nodes, - *rfdr_nodes, - *property_nodes, - shared[-1], - gate, - ), sources - - -def _property_localisation_nodes( - cells: dict, - *, - common: Mapping, - seed: int, - identity_source: str, - source_vintage: str, - lineage_inputs: tuple[str, ...], -) -> tuple[tuple[Node, ...], tuple[SourceRef, ...]]: - """The microcosm#1240 value and band nodes, after derivation and before the gates.""" - for column in (*_PROPERTY_VALUE_COLUMNS, *_PROPERTY_BAND_COLUMNS): - if ("household", column) not in cells: - raise ValueError( - f"Property localisation needs the household column {column!r} on the pool." - ) - for column in ("constituency_code", "lsoa_code", "local_authority_code"): - if ("household", column) not in cells: - raise ValueError( - f"Property localisation runs after geography derives {column!r}." - ) - stream = ( - "sha256-u53-v1", - property_localisation.PROPERTY_STREAM_EXPERIMENT, - 0, - int(seed), - ) - # The regional home values stage (microcosm#1112) keeps the value it - # replaced; the node must receive it to rank owners as it did before. - rank_basis_declared = ( - "household", - property_localisation.RANK_BASIS_COLUMN, - ) in cells - rank_basis_inputs = ( - (property_localisation.RANK_BASIS_COLUMN,) if rank_basis_declared else () - ) - value = Node( - id="uk.full.property.value", - kernel=UKPropertyValueKernel.ref, - population="uk.full.expand", - inputs=( - Slice( - "household", - tuple( - sorted( - { - *lineage_inputs, - *_PROPERTY_VALUE_COLUMNS, - *rank_basis_inputs, - "constituency_code", - } - ) - ), - ), - ), - outputs=( - Owned( - "household", - "main_residence_value", - cells[("household", "main_residence_value")], - rewrite=True, - ), - Owned( - "household", - "property_wealth", - cells[("household", "property_wealth")], - rewrite=True, - ), - ), - sources=(property_localisation.QUANTILES_SOURCE,), - params={ - **common, - "source": identity_source, - "source_vintage": source_vintage, - "stream": stream, - "support_stratum_lower": property_localisation.SUPPORT_STRATUM_LOWER, - "method": property_localisation.METHOD_VERSION, - "rank_basis_declared": rank_basis_declared, - }, - artifact_outputs=(ArtifactOutput("quantiles", PROPERTY_QUANTILE_TYPE),), - description="Read each owner clone's home value off its constituency's uprated sale distribution at its region rank.", - ) - band = Node( - id="uk.full.property.band", - kernel=UKPropertyBandKernel.ref, - population="uk.full.expand", - inputs=( - Slice( - "household", - tuple( - sorted( - { - *_PROPERTY_BAND_COLUMNS, - "local_authority_code", - "lsoa_code", - "tenure_type", - } - ) - ), - ), - ), - outputs=( - Owned( - "household", - "council_tax_band", - cells[("household", "council_tax_band")], - rewrite=True, - ), - ), - sources=( - property_localisation.BAND_MIX_SOURCE, - property_localisation.SCOTLAND_BAND_MIX_SOURCE, - ), - params={**common, "method": property_localisation.METHOD_VERSION}, - artifact_inputs=( - ArtifactInput("quantiles", value.id, "quantiles", PROPERTY_QUANTILE_TYPE), - ), - description=( - "Draw each England-and-Wales owner clone's council-tax band from its " - "LSOA's VOA mix, and each Scottish owner clone's from its council " - "area's CTAXBASE mix, at the same uniform." - ), - ) - sources = ( - SourceRef( - property_localisation.QUANTILES_SOURCE, - "raw-bytes-v1", - "Per-constituency quantile knots of the uprated Land Registry sale values.", - ), - SourceRef( - property_localisation.BAND_MIX_SOURCE, - "raw-bytes-v1", - "Per-LSOA cumulative VOA council-tax band shares.", - ), - SourceRef( - property_localisation.SCOTLAND_BAND_MIX_SOURCE, - "raw-bytes-v1", - "Per-council-area chargeable dwellings by council-tax band " - "(Scottish Government CTAXBASE), vendored from Chronicle.", - ), - ) - return (value, band), sources - - -def _brma_area_nodes( - cells: dict, *, common: Mapping -) -> tuple[tuple[Node, ...], tuple[SourceRef, ...]]: - """The microcosm#1262 area-BRMA node, after the local-authority resolve.""" - - for column in _BRMA_AREA_COLUMNS: - if ("household", column) not in cells: - raise ValueError( - f"The area BRMA node needs the household column {column!r} on the pool." - ) - node = Node( - id="uk.full.geography.brma", - kernel=UKBRMAAreaKernel.ref, - population="uk.full.expand", - inputs=(Slice("household", tuple(sorted(_BRMA_AREA_COLUMNS))),), - outputs=( - Owned("household", "brma", cells[("household", "brma")], rewrite=True), - ), - sources=(brma_area_layer.BRMA_AREA_LAYER_SOURCE,), - params={**common, "method": brma_area_layer.METHOD_VERSION}, - description="Read each placed Great Britain household's BRMA off its assigned area.", - ) - source = SourceRef( - brma_area_layer.BRMA_AREA_LAYER_SOURCE, - "raw-bytes-v1", - "BRMA of every England-and-Wales LSOA and Scottish output area.", - ) - return (node,), (source,) - - -def _rfdr_area_nodes( - cells: dict, *, common: Mapping, strict: bool = True -) -> tuple[tuple[Node, ...], tuple[SourceRef, ...]]: - """The microcosm#1418 area RFDR node, after the local-authority resolve. - - ``strict`` (the build's full scale) keeps the refusal of a support system - none of whose households is flagged, as the private-rent node keeps its - strict cells; malformed codes refuse at every scale. - """ - - for column in _RFDR_AREA_COLUMNS: - if ("household", column) not in cells: - raise ValueError( - f"The area RFDR node needs the household column {column!r} on the pool." - ) - column = rfdr_area_layer.RFDR_COLUMN - node = Node( - id="uk.full.geography.rfdr", - kernel=UKRFDRAreaKernel.ref, - population="uk.full.expand", - inputs=(Slice("household", _RFDR_AREA_COLUMNS),), - outputs=(Owned("household", column, "bool"),), - sources=(rfdr_area_layer.RFDR_AREA_LAYER_SOURCE,), - params={ - **common, - "method": rfdr_area_layer.METHOD_VERSION, - "strict_flags": bool(strict), - }, - description="Set each placed household's Rural Fuel Duty Relief flag off its area.", - ) - cells[("household", column)] = "bool" - source = SourceRef( - rfdr_area_layer.RFDR_AREA_LAYER_SOURCE, - "raw-bytes-v1", - "Great Britain output areas in the Rural Fuel Duty Relief scheme.", - ) - return (node,), (source,) - - -def _private_rent_nodes( - cells: dict, - *, - common: Mapping, - seed: int, - identity_source: str, - source_vintage: str, - lineage_inputs: tuple[str, ...], - strict: bool = True, -) -> tuple[tuple[Node, ...], tuple[SourceRef, ...]]: - """The microcosm#1262 private-rent node, after the area-BRMA node. - - ``strict`` holds at full scale; a sampled development rung relaxes it so - that a region its sample leaves without FRS renters ranks and scales on - Great Britain's (``private_rent_localisation.localise_rents``). - """ - - for column in _PRIVATE_RENT_COLUMNS: - if ("household", column) not in cells: - raise ValueError( - f"The private-rent node needs the household column {column!r} on the pool." - ) - stream = ( - "sha256-u53-v1", - private_rent_localisation.STREAM_EXPERIMENT, - 0, - int(seed), - ) - node = Node( - id="uk.full.private_rent", - kernel=UKPrivateRentKernel.ref, - population="uk.full.expand", - inputs=( - Slice( - "household", tuple(sorted({*lineage_inputs, *_PRIVATE_RENT_COLUMNS})) - ), - ), - outputs=( - Owned("household", "rent", cells[("household", "rent")], rewrite=True), - ), - sources=( - private_rent_localisation.QUANTILES_SOURCE, - private_rent_localisation.COUNTS_SOURCE, - ), - params={ - **common, - "source": identity_source, - "source_vintage": source_vintage, - "stream": stream, - "method": private_rent_localisation.METHOD_VERSION, - "strict_cells": bool(strict), - }, - description=( - "Read each Great Britain private renter's rent off its assigned BRMA's " - "list at its FRS rank, scaled to its region's FRS level." - ), - ) - sources = ( - SourceRef( - private_rent_localisation.QUANTILES_SOURCE, - "raw-bytes-v1", - "Weekly rent quantiles by BRMA and LHA category B to E.", - ), - SourceRef( - private_rent_localisation.COUNTS_SOURCE, - "raw-bytes-v1", - "Census private-rented households by FRS region, BRMA and bedrooms.", - ), - ) - return (node,), sources def register_uk_population_kernels(registry: KernelRegistry) -> None: - from microcosm.build.graph_atomic_geography import ( - register_atomic_geography_kernels, - ) + """Register the pool checkpoint. The shared atomic-geography kernels are + not the UK's to register: a geography-first spine assigns its areas in its + own stages, and NZ's transport registers the shared kernels it runs.""" - for kernel in ( - UKFamilySampleKernel(), - UKSampleNormalizationKernel(), - UKGeographicExpansionKernel(), - UKGeographyIdentityKernel(), - UKPropertyValueKernel(), - UKPropertyBandKernel(), - UKLocalAuthorityResolveKernel(), - UKBRMAAreaKernel(), - UKRFDRAreaKernel(), - UKPrivateRentKernel(), - UKLocationDrawKernel(), - UKGeographyMappingKernel(), - UKGeographyGateKernel(), - UKPoolCheckpointKernel(), - ): - registry.register(kernel) - register_atomic_geography_kernels(registry) + registry.register(UKPoolCheckpointKernel()) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py index ff981b7994..32b5ab5c7e 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_targets.py @@ -65,7 +65,6 @@ from .full_targets import CHRONICLE_SOURCE_CODEC, load_chronicle_source_bytes from .geography_ladder import load_uk_oa_ladder, uk_area_region_codes from .graph_population import ( - GEOGRAPHY_GATE_TYPE, context_frame, population_columns, population_slices, @@ -259,6 +258,11 @@ def implementation_hash(self) -> str: weighted_integrity, receipt_counts, *uk_target_compile_modules(), + # The canonical encoding of the artifacts all four kernels store + # and read (microcosm#931). What only one of them computes + # through, it hashes in its own implementation hash below; + # test_uk_graph derives each kernel's set from what its code names. + canonical_json, ) # The solve fingerprint: every packaged resource but the gate-only # registers, which no compile module reads (microcosm#1301), so a @@ -276,6 +280,26 @@ def implementation_hash(self) -> str: class UKFullTargetCompilationKernel(_TargetKernel): ref = "uk.full.target_compilation@1" + def implementation_hash(self) -> str: + from microcosm.build import chronicle_epoch + + # The compilation also computes through the Chronicle schema versions + # and fact-key epochs ledger_artifact checks the facts against and + # records in their provenance (chronicle_epoch) (microcosm#931). It + # does not compute through the ONS household type domain + # (frs_relationships): ledger_targets checks the household-type + # bindings against it before it compiles, and that check only + # refuses, so test_uk_graph lists it refusal-only (microcosm#931, + # ruled 9 Nov (late)). + return hashlib.sha256( + canonical_json( + { + "targets": super().implementation_hash(), + "compilation": source_hash(chronicle_epoch), + } + ) + ).hexdigest() + def run(self, context: KernelContext) -> KernelResult: from .full_targets import load_uk_full_target_inputs @@ -432,16 +456,6 @@ def run(self, context: KernelContext) -> KernelResult: or f"{spec.family}/{target_geography(spec)}" in families ) ) - brma_left_out = 0 - if context.params.get("brma_rows", True) is False: - # Without area BRMAs the BRMA rows would bind on the spine's - # regional draw, unrelated to placement (microcosm#1305 review). - brma_left_out = sum( - target_geography(spec) == "brma" for spec in registry.specs - ) - registry = registry.select( - predicate=lambda spec: target_geography(spec) != "brma" - ) selected = select_targets( registry, geography_levels=levels, geography_resolver=target_geography ) @@ -451,19 +465,6 @@ def run(self, context: KernelContext) -> KernelResult: "registry": registry_payload(selected.registry), "receipt": selected.receipt, "target_families": None if families is None else list(families), - **( - {} - if context.params.get("brma_rows", True) - else { - "brma_rows_left_out": { - "rows": brma_left_out, - "reason": ( - "no area BRMAs (--brma-localisation none): the rows " - "would bind on the spine's regional BRMA draw" - ), - } - } - ), } return KernelResult(artifacts={"selection": canonical_json(payload)}) @@ -471,12 +472,78 @@ def run(self, context: KernelContext) -> KernelResult: class UKFullMeasureKernel(_TargetKernel): ref = "uk.full.measures@1" - def run(self, context: KernelContext) -> KernelResult: - gate = json.loads(context.artifacts["geography_gate"].payload) - if not gate["passed"]: - raise ValueError( - "Geography integrity gate failed: " + "; ".join(gate["failures"]) + def implementation_hash(self) -> str: + import microcosm.calibrate.monetary_binding as monetary_binding + from microcosm.build import chronicle_epoch + from microcosm.frame import materialize as frame_materialize + from microcosm.frame.adapters import policyengine_uk as uk_engine_adapter + + from . import ( + annuity_trust_column, + atomic_area_support, + dfe_spring_term, + frs_annual_income, + graph_population, + hmrc_income, + income_tax_rate_classes, + lcfs_consumption, + national_calibration, + national_frame, + uc_target_measurements, + ) + + # The measures also compute through what the measure resolver and the + # engine blocks run (microcosm#931): the frame they rebuild on + # national_frame's schema (graph_population, national_frame), the + # engine's scratch frame (atomic_area_support, frame_materialize and + # the engine adapter's contracts), the resolver's inputs and measures + # (dfe_spring_term, frs_annual_income, hmrc_income, + # income_tax_rate_classes, lcfs_consumption and + # uc_target_measurements), the vendored facts' schema checks + # (chronicle_epoch), the measure inputs the calibration frame adapter + # injects (national_calibration), the national rows (local_rowwise) + # and the constraint matrix's monetary check (monetary_binding). The + # resolver reads the annuity and trust column where the frame carries + # it and zeros where it does not, so the measures bind its name too. + # It lives in annuity_trust_column, which holds nothing else, not in + # frs_spine, which writes the column, so this hash leaves frs_spine + # out, and on a bound spine an edit there leaves the measures' key + # alone (microcosm#931, ruled 9 Nov (late)). + return hashlib.sha256( + canonical_json( + { + "targets": super().implementation_hash(), + "measures": source_hash( + annuity_trust_column, + atomic_area_support, + chronicle_epoch, + dfe_spring_term, + frs_annual_income, + graph_population, + hmrc_income, + income_tax_rate_classes, + lcfs_consumption, + local_rowwise, + national_calibration, + national_frame, + uc_target_measurements, + monetary_binding, + frame_materialize, + uk_engine_adapter, + ), + } ) + ).hexdigest() + + def run(self, context: KernelContext) -> KernelResult: + from .graph_evidence import require_uk_spine_gate_admission + + # The spine's transferred gates admit the measures before they read + # the spine's version, where the composed spine declares them + # (microcosm#931). The dense line's own geography gate retired with + # its area draw; the spine's geography gates, registered later in + # #931, check the areas. + require_uk_spine_gate_admission(context) frame = context_frame(context) full = json.loads(context.artifacts["surface"].payload) selected = registry_from_payload( @@ -501,7 +568,9 @@ def run(self, context: KernelContext) -> KernelResult: ) as scratch: # The national problem is compiled per engine block and stitched # in the pool's household order; the whole pool is never - # materialized (microcosm#932 follow-up, 2026-10-05). + # materialized (microcosm#932 follow-up, 2026-10-05). The frame + # must hold the spine's K geographic clones at every block count + # (microcosm#931). national_problem, rows, metrics, evidence = ( resolve_uk_full_national_problem( frame, @@ -512,6 +581,7 @@ def run(self, context: KernelContext) -> KernelResult: full["band_edge_registry"] ), blocks=int(context.params["engine_blocks"]), + clones=int(context.params["geographic_clones"]), local_grains=grains, ) ) @@ -668,6 +738,10 @@ class UKFullProblemKernel(_TargetKernel): ref = "uk.full.problem@1" def implementation_hash(self) -> str: + import microcosm.calibrate.monetary_binding as monetary_binding + + from . import graph_population, local_geography, national_frame + # The loss weights the problem binds are built by # ``local_rowwise.prepare_uk_full_solve`` from the rules in # ``target_weights`` and ``national_doctrine`` (microcosm#1124), so an @@ -677,7 +751,11 @@ def implementation_hash(self) -> str: # ``calibrate.solve.default_target_loss_scales`` (microcosm#1492). All # of ``calibrate.solve`` is hashed: ``calibrate.adam@1``, whose hash # every UK solve kernel embeds, already hashes it, so an edit there - # re-solves anyway and re-keys only this node besides. + # re-solves anyway and re-keys only this node besides. The problem + # also computes through the frame it rebuilds on national_frame's + # schema (graph_population, national_frame), the area columns of the + # local surface matrix local_rowwise builds (local_geography) and the + # constraint matrix's monetary check (monetary_binding) (microcosm#931). return hashlib.sha256( canonical_json( { @@ -689,6 +767,12 @@ def implementation_hash(self) -> str: geography_constants, calibrate_solve, ), + "problem": source_hash( + graph_population, + national_frame, + local_geography, + monetary_binding, + ), } ) ).hexdigest() @@ -791,33 +875,63 @@ def run(self, context: KernelContext) -> KernelResult: def append_uk_target_nodes( graph: Graph, *, - population: str = "uk.full.expand", + population: str, calibration_year: int, time_period: str, geography_levels: tuple[str, ...] | None = None, target_families: tuple[str, ...] | None = None, engine_blocks: int = 1, + geographic_clones: int = 1, sample_fraction: float = 1.0, target_weight_rule: str = "uniform", optional_sources: tuple[str, ...] = (), review_date: str | None = None, - brma_rows: bool = True, ) -> Graph: """Default all geographies. Scope is independent of K, k and solver outcomes. - ``brma_rows=False`` (no area BRMAs) leaves the BRMA-grain rows out of the - selection: they bind on each placed household's own BRMA, and the spine's - regional draw is unrelated to placement (microcosm#1305 review). + ``population`` is the spine's version the targets read, the full build's + pool (microcosm#931). Where the graph declares the spine's transferred + gates, the measures take their admission: they refuse to read the version + unless that stored report admits it. ``geographic_clones`` is the spine's + clone count K: the measures refuse a frame that does not hold K clones, + at every block count, and ``engine_blocks`` must be one or K. """ if geography_levels is not None and not geography_levels: raise ValueError("An explicit geography selector must contain levels.") if target_families is not None and not target_families: raise ValueError("An explicit family selector must contain families.") + if ( + isinstance(geographic_clones, bool) + or not isinstance(geographic_clones, int) + or geographic_clones < 1 + ): + raise ValueError( + "The spine's geographic clone count K must be a positive integer." + ) + if engine_blocks not in {1, geographic_clones}: + raise ValueError( + "Engine blocks must be one or one per geographic clone, " + f"K = {geographic_clones} (microcosm#931)." + ) cells = population_columns(graph, population) slices = population_slices(cells) compile_sources = ("uk_ladder", "uk_ledger_facts", *optional_sources) common = {"time_period": time_period} + spine_gate_inputs: tuple[ArtifactInput, ...] = () + spine_gate_params: dict[str, object] = {} + if any(node.id == "spine.gates.transferred" for node in graph.nodes): + from .graph_evidence import SPINE_GATE_REPORT_TYPE + + gate = graph.node("spine.gates.transferred") + spine_gate_inputs = ( + ArtifactInput("spine_gate", gate.id, "gate_report", SPINE_GATE_REPORT_TYPE), + ) + spine_gate_params = { + "spine_gate_phase": "transferred", + "spine_gate_release_candidate": bool(gate.params["release_candidate"]), + "spine_gate_synthetic_smoke": bool(gate.params["synthetic_smoke"]), + } review_date = date.today().isoformat() if review_date is None else review_date date.fromisoformat(review_date) # The solve fingerprint, as the target kernels hash it (microcosm#1301). @@ -849,7 +963,6 @@ def append_uk_target_nodes( params={ "geography_levels": geography_levels, "target_families": target_families, - **({} if brma_rows else {"brma_rows": False}), }, artifact_inputs=(surface,), artifact_outputs=(ArtifactOutput("selection", TARGET_SELECTION_TYPE),), @@ -860,17 +973,13 @@ def append_uk_target_nodes( UKFullMeasureKernel.ref, population=population, inputs=slices, - params={**common, "engine_blocks": engine_blocks}, - artifact_inputs=( - surface, - selection, - ArtifactInput( - "geography_gate", - "uk.full.geography_gate", - "gate", - GEOGRAPHY_GATE_TYPE, - ), - ), + params={ + **common, + "engine_blocks": engine_blocks, + "geographic_clones": geographic_clones, + **spine_gate_params, + }, + artifact_inputs=(surface, selection, *spine_gate_inputs), artifact_outputs=(ArtifactOutput("measures", MEASURE_TYPE),), description="Evaluate selected measures and compile temporary national contributions.", ), diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py index 15f0f6e85b..e3d4363486 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph_terminal.py @@ -14,7 +14,6 @@ import sys from collections.abc import Mapping from dataclasses import asdict, replace -from functools import cache from importlib import metadata from pathlib import Path from types import SimpleNamespace @@ -54,6 +53,7 @@ from . import geography_ladder, national_frame, weight_year from .atomic_area_support import uk_area_code_frames, without_uk_native_alias_columns from .dataset_size import UK_SIZE_L2_PARAM_KEYS +from .geographic_clones import ARTIFACT_CLONE_INDEX_COLUMN, ladder_clone_index_column from .geography_ladder import ( UK_EXPORT_AREA_CODE_COLUMNS, export_area_code_columns, @@ -78,11 +78,7 @@ uk_time_period, validate_uk_national_frame, ) -from .rowwise_dataset import ( - ARTIFACT_CLONE_INDEX_COLUMN, - ladder_clone_index_column, - load_uk_rowwise_dataset, -) +from .rowwise_dataset import load_uk_rowwise_dataset from .weight_year import uk_dataset_year_frame, uk_weight_projection_factor EXPORT_DESCRIPTOR_TYPE = ArtifactType("microcosm.uk.full-export-descriptor", 1) @@ -358,18 +354,51 @@ class UKExportPrepareKernel(KernelBase): ) def implementation_hash(self) -> str: + from microcosm.frame import materialize as frame_materialize + + from .. import gates + from . import ( + atomic_area_support, + geographic_clones, + graph_population, + hmrc_uprating, + local_authority_input, + rowwise_dataset, + rowwise_geography, + ) + + # The descriptor also computes through the ladder gate's results and + # region codes (gates, rowwise_geography, local_authority_input), the + # area code frames (atomic_area_support), the engine tables + # (frame_materialize) and the clone index columns geographic_clones + # names, the engine's weight projection factor (hmrc_uprating's + # pins), the frame it rebuilds (graph_population) and the canonical + # encoding of the descriptor (microcosm#931). The gate report it reads + # only admits it (_require_full_gate_admission). return source_hash( - sys.modules[__name__], national_frame, geography_ladder, weight_year + sys.modules[__name__], + national_frame, + geography_ladder, + weight_year, + gates, + atomic_area_support, + graph_population, + hmrc_uprating, + local_authority_input, + rowwise_dataset, + geographic_clones, + rowwise_geography, + frame_materialize, + canonical_json, ) def run(self, context: KernelContext) -> KernelResult: for value in context.artifacts.values(): if value.type == FULL_GATE_REPORT_TYPE: - _, enforcement = decode_full_gate_report(value.payload) - if not enforcement["artifact_permitted"]: - raise ValueError( - "UK export preparation refused by a structural full-build gate." - ) + _require_full_gate_admission( + value.payload, + "UK export preparation refused by a structural full-build gate.", + ) bindings = json.loads(str(context.params["bindings"])) bindings["artifacts"] = { name: value.key for name, value in context.artifacts.items() @@ -392,7 +421,26 @@ class UKExportReadbackKernel(KernelBase): ) def implementation_hash(self) -> str: - return source_hash(sys.modules[__name__], national_frame) + from microcosm.frame import materialize as frame_materialize + from microcosm.frame.adapters import policyengine_uk as uk_engine_adapter + + from .. import artifact_files, trace + + # The readback also computes through the file record it takes + # (artifact_files, trace's file digest), the H5 it reads back + # (frame_materialize's table reader, the engine adapter's contracts), + # the area code columns it checks (geography_ladder) and the + # canonical encoding of its report (microcosm#931). + return source_hash( + sys.modules[__name__], + national_frame, + artifact_files, + trace, + frame_materialize, + uk_engine_adapter, + geography_ladder, + canonical_json, + ) def run(self, context: KernelContext) -> KernelResult: descriptor = json.loads(context.artifacts["export_descriptor"].payload) @@ -409,6 +457,19 @@ class UKPackageInventoryKernel(KernelBase): Determinism.DETERMINISTIC, numeric=Numeric.BITWISE, seed_source=SeedSource.NONE ) + def implementation_hash(self) -> str: + from .. import artifact_files, trace + + # The inventory records each evidence file through artifact_files + # (trace's file digest) and encodes itself canonically (microcosm#931). + return source_hash( + type(self), + artifact_files, + trace, + canonical_json, + dependencies=self.capabilities.dependencies, + ) + def run(self, context: KernelContext) -> KernelResult: report = json.loads(context.artifacts["export_readback"].payload) if ( @@ -683,6 +744,16 @@ def _require_one_reference(supporting: Mapping[str, Any]) -> None: ) +def _require_full_gate_admission(payload: bytes | Mapping, refusal: str) -> None: + """Refuse with ``refusal`` unless a full gate report permits the artifact. + + An admission read only: its caller computes nothing from the report. + """ + _, enforcement = decode_full_gate_report(payload) + if not enforcement["artifact_permitted"]: + raise ValueError(refusal) + + def _spine_gate_evidence(context: KernelContext): from ..stage_evidence import decode_stage_evidence @@ -746,56 +817,18 @@ def _source_gate_evidence(context: KernelContext, engine): } -@cache -def uk_terminal_only_code_hash() -> str: - """The source hash of the code only the calibrated gates run. - - The high-value homes evidence is computed on the calibrated pool alone. - The source preflight runs the gate kernel's code and the dense solve keys - on the preflight, so this code keys the calibrated node through its - parameters, not the kernel's hash (microcosm#1306 review). An edit to - ``high_value_home_gates`` re-keys the calibrated gates and what follows - them, never the solve. ``high_value_homes`` also feeds the target - kernels, which hash it, so an edit to it still re-solves: a disclosure fix - between two builds that must reuse the solve belongs in - ``high_value_home_gates`` alone. - - The calibrated node (``uk.full.gates.calibrated``) also publishes its - ``calibration_diagnostics`` output as schema 9: - ``catalog_calibration_diagnostics`` with the UK label choices (which read - ``UK_GEOGRAPHY_ID_TO_LABEL``), dumped from ``CalibrationDiagnosticsV9``. - Those modules decide the published labels and bytes, so an edit to them - re-keys the calibrated node the same way. ``geography_constants``, like - ``high_value_homes``, also feeds the target kernels, which hash it - (microcosm#1492), so an edit to it still re-solves; an edit to the other - three never does. - - Hashed once per process, as the code was imported: an edit to the files - while a build runs cannot fail the calibrated gates after the solve. - """ - import microcosm.calibrate.geography_constants as geography_constants - import microcosm.calibrate.hierarchy_labels as hierarchy_labels - import microcosm.diagnostics.catalogs as diagnostics_catalogs - import microcosm.diagnostics.schema as diagnostics_schema - - from . import high_value_home_gates, high_value_homes - - return source_hash( - high_value_home_gates, - high_value_homes, - diagnostics_catalogs, - diagnostics_schema, - hierarchy_labels, - geography_constants, - ) - - class UKFullGateKernel(KernelBase): """The calibrated gate battery: every terminal gate, on the final weights. Its terminal gates read the gate-only registers, so it keys on the full country fingerprint and a sign-off re-keys it. The source preflight runs - under :class:`UKFullPreflightGateKernel` (microcosm#1301). + under :class:`UKFullPreflightGateKernel` (microcosm#1301). Both phases + key on the code both compute through, this kernel's hash, and each on + its own, which its node carries as ``_code`` and the kernel + checks before it evaluates anything + (:data:`.gate_phase_codes.UK_FULL_GATE_PHASE_CODE`, + :func:`.gate_phase_codes.require_current_phase_code`; microcosm#931 + ruling 6, and the module as ruled on 9 Nov (late)). """ ref = "uk.full-gates@1" @@ -820,17 +853,51 @@ def _country_resources(self) -> str: return load_country_spec("uk").fingerprint def implementation_hash(self) -> str: - # The gate evaluators (``gates``) and the register loaders - # (``terminal_gates``) decide the report as much as the bindings do, - # so an edit to either re-keys the gate nodes (microcosm#1301). - from .. import gate_battery, gates + # The code both phases compute through (microcosm#931 ruling 6), as + # test_uk_graph derives it from the gates each declares and the branch + # of run each takes: this kernel's run, with the manifests, + # enforcement and source registries it reads and the reports it + # encodes; the battery, the gate registry and the shared gates' + # evaluators; and what both phases' gates compute through, among it + # the release input coverage, the frame surface it reads, the cut's + # reference surface it requires and the true-source counts it + # withholds by (reference_surface, receipt_counts; microcosm#1095), + # the target surfaces and their expected sets, the register loaders + # (terminal_gates), and the spine and local gate scopes (gate_scopes, + # not the runner that re-exports them). What only one phase computes + # through is that phase's own code, a node param that run checks + # (UK_FULL_GATE_PHASE_CODE, in gate_phase_codes, which this hash + # leaves out), so the preflight's own key, and with it the dense + # solve's, carries none of the terminal phase's; an edit to one of + # those still re-solves wherever a node upstream of the solve hashes + # it (gate_phase_codes.uk_terminal_only_code_hash). + import microcosm.calibrate.geography_constants as geography_constants + import microcosm.calibrate.hierarchy as calibrate_hierarchy + import microcosm.calibrate.registry as calibrate_registry + import microcosm.calibrate.target as calibrate_target + import microcosm.frame.materialize as frame_materialize + import microcosm.graph.canonical as canonical_encoder + from microcosm.frame.adapters import policyengine_uk as uk_engine_adapter + + from .. import gate_battery, gates, stage_evidence from . import ( battery_bindings, - diagnostics, + council_tax_constants, full_gates, - local_rowwise, + gate_scopes, + geography_ladder, + graph_evidence, + graph_targets, + high_value_homes, + ledger_targets, + local_targets, + national_frame, + parity_reference, + receipt_counts, + reference_surface, + release_identity, + release_input_coverage, terminal_gates, - weighted_integrity, ) return hashlib.sha256( @@ -842,10 +909,29 @@ def implementation_hash(self) -> str: gates, battery_bindings, full_gates, - local_rowwise, terminal_gates, - weighted_integrity, - diagnostics, + stage_evidence, + graph_evidence, + graph_targets, + gate_scopes, + release_input_coverage, + reference_surface, + receipt_counts, + national_frame, + uk_engine_adapter, + frame_materialize, + release_identity, + parity_reference, + ledger_targets, + local_targets, + high_value_homes, + geography_ladder, + council_tax_constants, + geography_constants, + calibrate_registry, + calibrate_target, + calibrate_hierarchy, + canonical_encoder, ), "country_resources": self._country_resources(), } @@ -876,6 +962,7 @@ def run(self, context: KernelContext) -> KernelResult: uk_full_gate_manifest, uk_full_gate_scope_receipt, ) + from .gate_phase_codes import require_current_phase_code from .geography_ladder import load_uk_oa_ladder from .local_rowwise import uk_ladder_area_support_summary from .reference_surface import load_uk_reference_surface @@ -886,13 +973,7 @@ def run(self, context: KernelContext) -> KernelResult: raise ValueError( "UK full gate engine identity differs from its declared binding." ) - if ( - context.params["phase"] == "terminal" - and context.params.get("terminal_code") != uk_terminal_only_code_hash() - ): - raise ValueError( - "UK full gate terminal-only code differs from its declared binding." - ) + require_current_phase_code(context.params, phase, battery="full") selection = json.loads(context.artifacts["selection"].payload)["receipt"] gates = uk_full_gate_manifest(selection) from ..gate_battery import _gates_manifest_payload, _json_safe @@ -1124,7 +1205,9 @@ class UKFullPreflightGateKernel(UKFullGateKernel): preflight's key is part of the solve's. Its gates read no gate-only register (``solve_scope``), so it keys on the solve fingerprint: a sign-off between the two release builds leaves it, and so the solve, a - cache hit (microcosm#1301). The code it runs is the calibrated kernel's. + cache hit (microcosm#1301). The code it runs is the calibrated kernel's, + and it keys on that kernel's hash and on its own code, which its node + carries as ``preflight_code`` (microcosm#931 ruling 6). """ ref = "uk.full-gates.preflight@1" @@ -1167,6 +1250,7 @@ def append_uk_full_gate_nodes( from ..gate_battery import _gates_manifest_payload from ..stage_evidence import STAGE_EVIDENCE_TYPE from .full_gates import uk_full_gate_manifest + from .gate_phase_codes import UK_FULL_GATE_PHASE_CODE from .graph_evidence import SPINE_BUILD_STATE_TYPE, SPINE_GATE_REPORT_TYPE from .graph_targets import TARGET_SELECTION_TYPE, TARGET_SURFACE_TYPE @@ -1233,7 +1317,16 @@ def append_uk_full_gate_nodes( "uk.full.gates.preflight", UKFullPreflightGateKernel.ref, population=input_population, - params={**params, "phase": "preflight"}, + # The preflight keys on its own code, as the calibrated node keys on + # its own, so neither phase's own key carries the other's + # (microcosm#931 ruling 6). Key chaining still re-keys a phase wherever + # a node upstream of it hashes the edited module, as gate_phase_codes' + # docstrings say. + params={ + **params, + "phase": "preflight", + "preflight_code": UK_FULL_GATE_PHASE_CODE["preflight"](), + }, artifact_inputs=common, artifact_outputs=(ArtifactOutput("gate_report", FULL_GATE_REPORT_TYPE),), description="Validate complete source/reference registries and spine stage ownership before dense calibration.", @@ -1278,12 +1371,14 @@ def append_uk_full_gate_nodes( population=calibration.population, inputs=population_slices(population_columns(graph, calibration.population)), sources=sources, - # Only the calibrated node keys on the code only it runs, so an edit to - # that code never re-keys the preflight and the solve after it. + # Only the calibrated node keys on the code only it runs, so the + # preflight's own key, and the solve's, leave it out; an edit to it + # still re-solves wherever a node upstream of the solve hashes the + # module (gate_phase_codes.uk_terminal_only_code_hash). params={ **gate_params, "phase": "terminal", - "terminal_code": uk_terminal_only_code_hash(), + "terminal_code": UK_FULL_GATE_PHASE_CODE["terminal"](), }, artifact_inputs=( *common, @@ -1353,16 +1448,61 @@ class UKFullHoldoutKernel(KernelBase): ) def implementation_hash(self) -> str: + import microcosm.calibrate.artifacts as calibrate_artifacts + import microcosm.calibrate.exact_k as calibrate_exact_k + import microcosm.calibrate.gates as calibrate_gates + import microcosm.calibrate.geography_constants as geography_constants + import microcosm.calibrate.hierarchy as calibrate_hierarchy + import microcosm.calibrate.initialization as calibrate_initialization + import microcosm.calibrate.matrix as calibrate_matrix + import microcosm.calibrate.monetary_binding as monetary_binding + import microcosm.calibrate.registry as calibrate_registry + import microcosm.calibrate.solve as calibrate_solve + import microcosm.calibrate.target as calibrate_target + from microcosm.frame.adapters import policyengine_uk as uk_engine_adapter + + from .. import cross_grain, gate_battery, holdout, stage_evidence from . import ( + council_tax_constants, + cross_grain_declarations, dataset_size, + full_problem, + graph_population, graph_targets, + high_value_homes, + ledger_targets, local_doctrine, + local_geography, local_rowwise, + local_target_census, + local_targets, national_doctrine, + off_surface_cells, + scotland_brma_la_overlap, + size_checkpoint, + solve_progress, + support_floor, target_weights, + uprating_holds, + weighted_integrity, ) from .solve_scope import uk_solve_fingerprint + # The holdout also computes through the problem it reconstructs (the + # local surface: full_problem, ledger_targets, local_targets, + # cross_grain and its declarations, the off-surface cells, the + # uprating holds, the Scotland BRMA overlap, the council tax bands, + # high-value homes' metric names, the support floor, local_geography, + # geography_ladder, and the frame on national_frame's schema through + # graph_population), the folds and rowwise solves it runs (holdout, + # local_target_census, size_checkpoint, solve_progress, + # weighted_integrity, the engine adapter's contracts, and + # microcosm.calibrate's solver, matrix, gates, exact-K, initialisation, + # targets, registry, hierarchy, monetary checks and UK-wide geography), + # the problem it decodes (calibrate's artifacts), and its report's + # encoding (gate_battery's _json_safe, stage_evidence and the + # canonical encoder) (microcosm#931). The preflight report only + # admits it (_require_full_gate_admission). return hashlib.sha256( canonical_json( { @@ -1374,6 +1514,41 @@ def implementation_hash(self) -> str: local_doctrine, national_doctrine, target_weights, + full_problem, + ledger_targets, + local_targets, + cross_grain, + cross_grain_declarations, + off_surface_cells, + uprating_holds, + scotland_brma_la_overlap, + council_tax_constants, + high_value_homes, + support_floor, + local_geography, + geography_ladder, + graph_population, + national_frame, + holdout, + local_target_census, + size_checkpoint, + solve_progress, + weighted_integrity, + uk_engine_adapter, + calibrate_solve, + calibrate_matrix, + calibrate_gates, + calibrate_exact_k, + calibrate_initialization, + calibrate_target, + calibrate_registry, + calibrate_hierarchy, + monetary_binding, + geography_constants, + calibrate_artifacts, + gate_battery, + stage_evidence, + canonical_json, dependencies=self.capabilities.dependencies, ), # The holdout repeats the solve: it keys on the solve @@ -1393,11 +1568,10 @@ def run(self, context: KernelContext) -> KernelResult: from .graph_targets import reconstruct_uk_full_problem_inputs from .local_rowwise import rotated_uk_local_holdout - _, preflight = decode_full_gate_report(context.artifacts["preflight"].payload) - if not preflight["artifact_permitted"]: - raise ValueError( - "Rotated holdout cannot run after a blocking source preflight." - ) + _require_full_gate_admission( + context.artifacts["preflight"].payload, + "Rotated holdout cannot run after a blocking source preflight.", + ) if context.params["skip_holdout"]: report = { "report_only": True, @@ -1703,8 +1877,8 @@ def rowwise_candidate_manifest_from_graph( from microcosm.calibrate.artifacts import decode_problem from .battery_bindings import uk_area_support_below_floor - from .calibration_run import UK_LOCAL_GATE_SCOPE from .diagnostics import uk_fit_by_family, uk_support_limited_misses + from .gate_scopes import UK_LOCAL_GATE_SCOPE from .graph_targets import registry_from_payload from .national_sampling import UK_SAMPLE_RUNG_TOKENS from .rowwise_cli import ( @@ -1756,12 +1930,6 @@ def rowwise_candidate_manifest_from_graph( holdout = json.loads( _graph_payload(final_manifest, store, "uk.full.holdout", "holdout") ) - geography_gate = json.loads( - _graph_payload(final_manifest, store, "uk.full.geography_gate", "gate") - ) - sampling = json.loads( - _graph_payload(final_manifest, store, "uk.full.sample", "sampling") - )["receipt"] size_receipt = _optional_graph_json( final_manifest, store, "uk.full.size_refit", "size" ) @@ -1771,17 +1939,33 @@ def rowwise_candidate_manifest_from_graph( ) or {} ) - # The request's geography binding (assignment mode, definition digest, - # support pins, identity column, stream) travels in the export descriptor. + # The request's bindings travel in the export descriptor: the spine the + # build binds (its clone count, its sampling and its release posture) and + # the spine's geography binding (assignment, identity column, definition + # digest, support pins, stream, register and clone count), since the + # dense line takes its pool, sampling and areas from the spine + # (microcosm#931). The ladder gate is the descriptor's own geography + # integrity check, on the calibrated release frame. export_descriptor = ( _optional_graph_json( final_manifest, store, "uk.full.export.prepare", "export_descriptor" ) or {} ) - geography_binding = dict( - (export_descriptor.get("bindings") or {}).get("geography") or {} - ) + request_bindings = dict(export_descriptor.get("bindings") or {}) + geography_binding = dict(request_bindings.get("geography") or {}) + spine_binding = dict(request_bindings.get("spine") or {}) + if not isinstance(spine_binding.get("sampling"), Mapping): + raise ValueError( + "The export descriptor's request binds no spine sampling: the dense " + "line's rung is its spine's (microcosm#931)." + ) + sampling = dict(spine_binding["sampling"]) + spine_fraction = float(sampling["fraction"]) + ladder_gate = { + **dict(export_descriptor.get("geography_integrity") or {}), + "phase": "post_calibration", + } enforcement = dict(gate_document["enforcement"]) gate_rows = _gate_rows(gate_document) local_scope = { @@ -1805,7 +1989,7 @@ def rowwise_candidate_manifest_from_graph( ) size_drift = size_doctrine_drift(args) releasable, release_posture = release_verdict( - sample_fraction=args.sample_fraction, + sample_fraction=spine_fraction, engine_blocks=args.engine_blocks, release_blocking_gates_passed=bool( enforcement["release_blocking_gates_passed"] @@ -1875,8 +2059,7 @@ def rowwise_candidate_manifest_from_graph( "uk.full.gates.calibrated", "uk.full.holdout", "uk.full.package", - "uk.full.sample", - "uk.full.geography_gate", + "uk.full.export.prepare", "uk.full.size_refit", "uk.full.spine_checkpoint", } @@ -1930,8 +2113,8 @@ def rowwise_candidate_manifest_from_graph( "sampling": dict(sampling), "rung_surface": { **dict(bindings.get("rung_surface", {})), - "rung": UK_SAMPLE_RUNG_TOKENS[float(args.sample_fraction)], - "fraction": float(args.sample_fraction), + "rung": UK_SAMPLE_RUNG_TOKENS[spine_fraction], + "fraction": spine_fraction, "unreachable_check": "completed", }, "outputs": { @@ -1957,20 +2140,20 @@ def rowwise_candidate_manifest_from_graph( ].nunique() ), "missing_geography_rows": 0, - "ladder_gate": {**geography_gate, "phase": "post_calibration"}, + "ladder_gate": dict(ladder_gate), "assignment": geography_binding, }, - "gate": {**geography_gate, "phase": "post_calibration"}, + "gate": dict(ladder_gate), "weights": { "household_weight_kind": weight_kind, "household_weight_kind_chain": [ {"stage": "staging", "kind": "importance"}, *( [] - if float(args.sample_fraction) == 1.0 + if spine_fraction == 1.0 else [{"stage": "sample", "kind": "importance"}] ), - {"stage": "ladder_clone", "kind": "importance"}, + {"stage": "geographic_expand", "kind": "importance"}, {"stage": "rowwise_calibration", "kind": weight_kind}, ], "mass_log_records_before_calibration": len(frame.mass_log) - 1, diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/high_value_home_support.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/high_value_home_support.py index 4459b95625..b03c96168e 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/high_value_home_support.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/high_value_home_support.py @@ -1,26 +1,19 @@ """UK high-value home support: light copies of the top value stratum of owner households (microcosm#1240, fix 1). -The dense graph reads every owner clone's home value off its assigned -constituency's Land Registry sale distribution at the household's within-region -rank (``uk.full.property.value``), so where the rank lands decides whether a -clone is a £2m-plus home. The top of that rank is thin: a few hundred FRS owner -households carry the top five per cent of each region, and each seat's £2m-plus -cell then rests on the handful of them whose clones landed there. This stage -copies every owner household in the top stratum of its FRS region ``m`` times -at equal weight, each copy carrying its index in +This stage copies every owner household in the top stratum of its FRS region +``m`` times at equal weight, each copy carrying its index in ``high_value_support_copy_index`` (0 on roots and unsplit households), the -explicit lineage the geography identity keys on; the value node then gives each -copy its own keyed rank inside the stratum instead of the household's single -rank, so the copies spread across the tail and across seats. - -Deterministic and mass-conserving: no draw, no seed, no salt. Every -household's mass and the total household mass are conserved exactly. The -stratum is selected on the spine's weights with ties broken by household id; -the value node ranks the pool's weights with a keyed tie-break and ranks the -copies beside their root, so the two agree up to ties and up to any weight -change between the spine and the pool, not by construction. No value moves -here. +explicit lineage the geography identity keys on. Since microcosm#931 the +stratum ranks England and Wales owners on the values ``was_wealth`` reads off +their constituency's knots, which the regional property uprating returns them +to, and every copy keeps its parent's value, area and geography key +(``uk_copy_area_agreement`` checks the area and the key). + +Deterministic and mass-conserving: every household's mass and the total +household mass are conserved exactly, and the one draw is the stratum's +tie-break, a uniform keyed on each household's own id that orders +households tied on value (microcosm#931). No value moves here. """ from __future__ import annotations @@ -31,8 +24,13 @@ import pandas as pd from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.stochastic_assignment import keyed_tie_order +from microcosm.build.uk_runtime import copy_id_table from microcosm.build.uk_runtime.cgt_structure import _assert_closed_world_operations -from microcosm.build.uk_runtime.cgt_support import _exact_support_weights +from microcosm.build.uk_runtime.cgt_support import ( + CGT_SUPPORT_EXACT_TOTAL_EXCEPTION, + _exact_support_weights, +) from microcosm.build.uk_runtime.household_lineage import ( HIGH_VALUE_SUPPORT_COPY_COLUMN, HIGH_VALUE_SUPPORT_INDEX_COLUMN, @@ -46,19 +44,22 @@ SUPPORT_STRATUM_LOWER, owner_mask, ) -from microcosm.build.uk_runtime.rowwise_geography import id_multiplier_for_values from microcosm.frame import Frame, MassChangeRecord, WeightKind __all__ = [ "HIGH_VALUE_SUPPORT_COPIES", "HIGH_VALUE_SUPPORT_COPIES_COLUMN", + "HIGH_VALUE_SUPPORT_COPY_ID_FIELD", "HIGH_VALUE_SUPPORT_MASS_CHANGE_REASON", "HIGH_VALUE_SUPPORT_OPERATION_KIND", "HIGH_VALUE_SUPPORT_STAGE_NAME", "HIGH_VALUE_SUPPORT_STRATUM_LOWER", + "HIGH_VALUE_SUPPORT_TIE_SALT", + "HIGH_VALUE_SUPPORT_TIE_SEED", "UKHighValueHomeSupportResult", "UKHighValueHomeSupportStageTransform", "high_value_support_operation_parameters", + "high_value_support_tie_break", "select_high_value_support_households", "split_high_value_support_households", ] @@ -73,6 +74,14 @@ HIGH_VALUE_SUPPORT_STRATUM_LOWER = SUPPORT_STRATUM_LOWER #: Copies per stratum household, root included (ruled six). HIGH_VALUE_SUPPORT_COPIES = 6 +#: The copy-id field copy ``k`` takes ``k`` in (microcosm#931). +HIGH_VALUE_SUPPORT_COPY_ID_FIELD = "high_value_support" +#: The seed and salt of the stratum's tie-break: owners tied on value take +#: the order of a uniform keyed on their own household_id, then of their ids, +#: not of their ids alone, which always ranked a household's highest copy or +#: clone ordinal highest (microcosm#931). +HIGH_VALUE_SUPPORT_TIE_SEED = 0 +HIGH_VALUE_SUPPORT_TIE_SALT = "high_value_home_support:value_tie" HIGH_VALUE_SUPPORT_MASS_CHANGE_REASON = ( "High-value home support divides every owner household in the top value " "stratum of its region into light copies at equal weight; every household's " @@ -138,19 +147,30 @@ def high_value_support_operation_parameters() -> tuple[ "stratum_rule": ( "within each FRS region, owner households (tenure owned outright " "or with a mortgage, main_residence_value > 0, positive weight) " - "ordered by main_residence_value ascending then household_id " - "ascending; a household whose weighted mid-rank " - "(cumulative weight before it plus half its own, over the " - "region's owner weight) is at or above stratum_lower is selected" + "ordered by main_residence_value ascending, households tied on " + "it in ascending order of a uniform keyed on their own " + "household_id under seed and salt, then household_id ascending; " + "a household whose weighted mid-rank (cumulative weight before " + "it plus half its own, over the region's owner weight) is at or " + "above stratum_lower is selected" + ), + "seed": HIGH_VALUE_SUPPORT_TIE_SEED, + "salt": HIGH_VALUE_SUPPORT_TIE_SALT, + "copy_rule": ( + "every selected household becomes copies rows at weight / " + f"copies, {CGT_SUPPORT_EXACT_TOTAL_EXCEPTION}" ), - "copy_rule": "every selected household becomes copies rows at weight / copies", "id_remapping": ( - "id_multiplier_for_values; copy k takes every entity id + k x multiplier" + "copy-id table field high_value_support; copy k takes every " + "entity id + k x the field's radix (10**12)" ), "flag_column": HIGH_VALUE_SUPPORT_COPY_COLUMN, "copies_column": HIGH_VALUE_SUPPORT_COPIES_COLUMN, "copy_index_column": HIGH_VALUE_SUPPORT_INDEX_COLUMN, - "values": "every column copied unchanged; no draw, no seed, no salt", + "values": ( + "every column copied unchanged; the stratum's tie-break is the " + "stage's only draw" + ), "weight_kind_out": "importance", "conservation": "exact_total", "declared_factor": 1.0, @@ -160,6 +180,18 @@ def high_value_support_operation_parameters() -> tuple[ ) +def high_value_support_tie_break() -> tuple[int, str]: + """The declared seed and salt of the stratum's tie-break. + + Read from the reviewed operation the manifests restate verbatim + (:func:`high_value_support_operation_parameters`), so the stage keys its + ties on the declared pair (microcosm#931). + """ + + ((_, operation),) = high_value_support_operation_parameters() + return int(operation["seed"]), str(operation["salt"]) + + def select_high_value_support_households( *, household_ids: np.ndarray, @@ -168,13 +200,20 @@ def select_high_value_support_households( tenure_type: np.ndarray, value: np.ndarray, stratum_lower: float = HIGH_VALUE_SUPPORT_STRATUM_LOWER, + seed: int, + salt: str, ) -> tuple[np.ndarray, tuple[dict[str, object], ...]]: """The owner households in the top value stratum of each region. - Returns the selection mask aligned to the inputs and one receipt row per - region (owner households, owner mass, households selected, selected mass - and the value rank the stratum starts at). A region without owner - households selects nothing and says so. + Owners are ranked by value ascending, owners tied on value by a uniform + keyed on their own household id under ``seed`` and ``salt``, then by id + (:func:`microcosm.build.stochastic_assignment.keyed_tie_order`); ``seed`` + and ``salt`` have no default (the stage passes + :func:`high_value_support_tie_break`). Returns the selection mask aligned + to the inputs and one receipt row per region (owner households, owner + mass, households selected, selected mass and the value rank the stratum + starts at). A region without owner households selects nothing and says + so. """ ids = np.asarray(household_ids) weight = np.asarray(weights, dtype=float) @@ -206,7 +245,7 @@ def select_high_value_support_households( } ) continue - order = pool[np.lexsort((ids[pool], values[pool]))] + order = pool[keyed_tie_order(values[pool], ids[pool], seed=seed, salt=salt)] pool_weight = weight[order] total = float(pool_weight.sum()) cumulative = np.cumsum(pool_weight) @@ -235,10 +274,16 @@ def split_high_value_support_households( Each selected household of weight ``w`` becomes ``copies`` rows at ``w / copies``; the root keeps its ids, copy ``k`` takes every entity id - plus ``k`` times the frame's id multiplier, records ``k`` in - ``high_value_support_copy_index`` and carries a full copy of its person - and benefit-unit rows. The exact-total correction lands on the heaviest - unselected incumbent so every copy weight stays bitwise ``w / copies``. + plus ``k`` times the declared radix of the copy-id table's + ``high_value_support`` field (so every incoming id must sit below it), + records ``k`` in ``high_value_support_copy_index`` and carries a full copy + of its person and benefit-unit rows. The exact-total correction lands on + an unselected household, the first of the 64 heaviest that can carry it + (:data:`microcosm.build.uk_runtime.cgt_support.CGT_SUPPORT_CORRECTION_CANDIDATES`), + so every root and copy weighs bitwise ``w / copies``. Only where none of + them can does spi_support's shared fallback place it, and that row can be + a root or copy, then off ``w / copies`` by about one ulp of the total + (:func:`microcosm.build.uk_runtime.cgt_support._exact_support_weights`). Refuses a frame this stage has already split. """ if isinstance(copies, bool) or copies < 2: @@ -265,6 +310,7 @@ def split_high_value_support_households( weights = np.asarray(frame.weights_for("household").values, dtype=float) old_total = frame.weights_for("household").total household_ids = household["household_id"].to_numpy() + tie_seed, tie_salt = high_value_support_tie_break() selected, region_rows = select_high_value_support_households( household_ids=household_ids, weights=weights, @@ -272,16 +318,22 @@ def split_high_value_support_households( tenure_type=household[_TENURE_COLUMN].to_numpy(), value=household[_VALUE_COLUMN].to_numpy(), stratum_lower=stratum_lower, + seed=tie_seed, + salt=tie_salt, ) copy_counts = np.ones(len(household), dtype="int64") copy_counts[selected] = int(copies) - multiplier = id_multiplier_for_values( - person["person_id"], - person["person_household_id"], - person["person_benunit_id"], - benunit["benunit_id"], - household["household_id"], - ) + for table, columns in ( + (person, _PERSON_ID_COLUMNS), + (benunit, ("benunit_id",)), + (household, ("household_id",)), + ): + for column in columns: + copy_id_table.require_below( + table[column], + HIGH_VALUE_SUPPORT_COPY_ID_FIELD, + what=f"incoming {column} values", + ) household[HIGH_VALUE_SUPPORT_COPY_COLUMN] = False household[HIGH_VALUE_SUPPORT_COPIES_COLUMN] = copy_counts household[HIGH_VALUE_SUPPORT_INDEX_COLUMN] = np.zeros(len(household), dtype="int64") @@ -299,14 +351,15 @@ def split_high_value_support_households( benunit["benunit_id"].isin(set(family_person["person_benunit_id"])) ].copy() family_household = household.loc[family].copy() - offset = k * multiplier for column in _PERSON_ID_COLUMNS: - family_person[column] = family_person[column].astype("int64") + offset - family_benunit["benunit_id"] = ( - family_benunit["benunit_id"].astype("int64") + offset + family_person[column] = copy_id_table.mint( + family_person[column], HIGH_VALUE_SUPPORT_COPY_ID_FIELD, k + ) + family_benunit["benunit_id"] = copy_id_table.mint( + family_benunit["benunit_id"], HIGH_VALUE_SUPPORT_COPY_ID_FIELD, k ) - family_household["household_id"] = ( - family_household["household_id"].astype("int64") + offset + family_household["household_id"] = copy_id_table.mint( + family_household["household_id"], HIGH_VALUE_SUPPORT_COPY_ID_FIELD, k ) family_household[HIGH_VALUE_SUPPORT_COPY_COLUMN] = True family_household[HIGH_VALUE_SUPPORT_INDEX_COLUMN] = np.full( @@ -323,7 +376,7 @@ def split_high_value_support_households( exact = _exact_support_weights( final_weights, target=old_total, - unselected=np.flatnonzero(~selected & (incumbent_weights > 0.0)), + whole=np.flatnonzero(~selected & (incumbent_weights > 0.0)), ) receipt = MassChangeRecord( entity="household", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/household_lineage.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/household_lineage.py index 2adfc71827..8959982d19 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/household_lineage.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/household_lineage.py @@ -1,14 +1,15 @@ """The ordered clone path of every post-clone UK household, from its lineage columns. -The geography identity kernel keys each pool household on its source id and -its explicit clone path (microcosm#931); the post-geography property node -(microcosm#1240) needs the same path without the geographic clone branch, so -that every pool clone of one spine household shares one key. Both read the -path through this module, which validates the lineage columns once and never -infers a branch from ids, weights or values. The UK's copy lineage is declared -here (:data:`UK_COPY_LINEAGE`) for the shared readers in -``microcosm.build.source_lineage``: the receipts' true sources and the -FRS-trained fits' collapse of geographic clones read it too. +The spine's ``geographic_identity`` stage keys each household on its source id +and its explicit clone path, and the dense line keys no household of its own +(microcosm#931). The copy take-up stages read the same path through +:func:`uk_household_clone_paths`, which validates the lineage columns once and +never infers a branch from ids, weights or values. The UK's copy lineage is +declared here (:data:`UK_COPY_LINEAGE`) for the shared readers in +``microcosm.build.source_lineage``: the geography identity, the receipts' true +sources and the FRS-trained fits' collapse of geographic clones read it, and +:func:`geographic_fold_receipt` records how a fit's fold met its survey +records. """ from __future__ import annotations @@ -21,28 +22,38 @@ from microcosm.build.source_lineage import ( CopyBranch, CopyLineage, - collapse_replicas, + ReplicaFold, copy_paths, + fold_replicas, replica_keys, ) +from microcosm.build.uk_runtime.geographic_clones import ( + geographic_clone_count, + ladder_clone_index_column, +) from microcosm.build.uk_runtime.national_frame import UK_NATIONAL_SCHEMA -from microcosm.build.uk_runtime.rowwise_dataset import ladder_clone_index_column __all__ = [ + "GEOGRAPHIC_FOLD_EVIDENCE", "GEOGRAPHIC_SUPPORT_BRANCH", "HIGH_VALUE_SUPPORT_COPY_COLUMN", "HIGH_VALUE_SUPPORT_INDEX_COLUMN", "UK_COPY_LINEAGE", "UK_HOUSEHOLD_LINEAGE_COLUMNS", "collapse_geographic_clones", + "fold_geographic_clones", + "geographic_fold_receipt", "uk_household_clone_paths", ] -#: Lineage columns the post-clone household identity is keyed on. The support -#: channel, the CGT support split's copy flag and index (microcosm#1045), the -#: high-value home support copy flag and index (microcosm#1240) and the clone -#: flags come from the spine stages; the geographic clone index from -#: ``uk.full.expand``. Calibration K, weights and remapped ids are not inputs. +#: Lineage columns a post-clone household's clone path is read from. The +#: support channel, the CGT support split's copy flag and index +#: (microcosm#1045), the high-value home support copy flag and index +#: (microcosm#1240) and the clone flags come from the spine stages. So do the +#: source id and the geographic clone index, which the spine's +#: ``geographic_expand`` stage writes (microcosm#931); a spine built before +#: that stage carries no index. Calibration K, weights and remapped ids are +#: not inputs. HIGH_VALUE_SUPPORT_COPY_COLUMN = "household_is_high_value_support_copy" HIGH_VALUE_SUPPORT_INDEX_COLUMN = "high_value_support_copy_index" UK_HOUSEHOLD_LINEAGE_COLUMNS: tuple[str, ...] = ( @@ -56,13 +67,11 @@ "cgt_residential_clone_index", ladder_clone_index_column("household"), ) -#: The two optional columns the high-value support stage adds (microcosm#1240); -#: a spine built before that stage carries neither and keys as before. -UK_OPTIONAL_LINEAGE_COLUMNS: tuple[str, ...] = ( - HIGH_VALUE_SUPPORT_COPY_COLUMN, - HIGH_VALUE_SUPPORT_INDEX_COLUMN, -) GEOGRAPHIC_SUPPORT_BRANCH = "geographic_support" +#: The evidence key under which a stage whose fit folds geographic clones +#: records the fold (:func:`geographic_fold_receipt`), on a frame of more +#: than one clone only. +GEOGRAPHIC_FOLD_EVIDENCE = "geographic_fold" _BASE_SUPPORT_CHANNEL = "frs" _SPI_SUPPORT_CHANNEL = "spi" #: The UK's copy lineage for the shared readers in @@ -104,8 +113,8 @@ #: The branches every post-clone household carries: the SPI support channel #: and the CGT steps. High-value support (a spine built before that stage has -#: neither of its columns) and the geographic clone index (a spine frame has -#: none yet) are read where present. +#: neither of its columns) and the geographic clone index (a spine built +#: before ``geographic_expand`` has none) are read where present. _REQUIRED_BRANCHES = ( "spi_support_channel", "cgt_support_split", @@ -139,45 +148,55 @@ def uk_household_clone_paths( required = [c for c in UK_HOUSEHOLD_LINEAGE_COLUMNS if c != clone_column] missing = [c for c in required if c not in household] if missing: - raise ValueError(f"Geography identity lacks lineage column(s) {missing}.") + raise ValueError( + f"The household table lacks lineage column(s) {missing} that " + "its clone paths are read from." + ) present = [ column for column in (UK_COPY_LINEAGE.source, *UK_COPY_LINEAGE.copy_columns) if column in household ] if household.loc[:, present].isna().any().any(): - raise ValueError("Geography identity refuses null lineage values.") + raise ValueError("Household clone paths refuse null lineage values.") for column in _INT64_LINEAGE_COLUMNS: if column in household and household[column].dtype != np.dtype("int64"): - raise ValueError(f"Geography identity requires an int64 {column}.") + raise ValueError(f"Household clone paths require an int64 {column}.") sources, paths = copy_paths(household, UK_COPY_LINEAGE, require=_REQUIRED_BRANCHES) return [int(source) for source in sources], paths -def collapse_geographic_clones( +def fold_geographic_clones( tables: Mapping[str, pd.DataFrame], entity: str, rows: np.ndarray, weights: np.ndarray, *, inputs: pd.DataFrame, -) -> tuple[np.ndarray, np.ndarray]: +) -> ReplicaFold: """A fit's training rows with each record's geographic clones folded in. - A survey record a fit trains on keeps one row, its first among ``rows``, - with the weight of all its geographic clones; every other copy (SPI - support, high-value support and the CGT steps) stays a row of its own - (microcosm#931). The record's key is its true source and its clone path - without the geographic branch, through the shared reader - (``microcosm.build.source_lineage``); a frame without geographic clones, - as every spine is today, keeps ``rows`` and ``weights`` value for value. + A survey record a fit trains on keeps one row, its clone with the lowest + id among ``rows``, with the weight of all its geographic clones; every + other copy (SPI support, high-value support and the CGT steps) stays a + row of its own (microcosm#931). The record's key is its true source and + its clone path without the geographic branch, through the shared reader + (``microcosm.build.source_lineage``); a frame of one clone, with or + without the clone index, keeps ``rows`` and ``weights`` value for value. + + The fold orders a key's clones by ``entity``'s id column. Their ids + differ only by each clone's offset (the copy-id table's + ``geographic_support`` digits on a clone-first spine), so each variant + trains on its lowest clone among ``rows``: clone 0 whenever clone 0 is + selected and agrees with it, whatever the row order. ``inputs`` is what the fit reads, one row per row of ``entity``. Clones fold only where they agree on it. Clones that differ, through a draw keyed on their own ids or a stage run per area, stay rows of their own, - each variant with its clones' summed weight, so the fit sees each - clone's value at its share. Returns the kept rows and the weights over - every row of ``entity``. + each variant on its lowest-id clone with its clones' summed weight, so + the fit sees each clone's value at its share. Returns the shared + :class:`~microcosm.build.source_lineage.ReplicaFold` over every row of + ``entity``. """ keys = replica_keys( @@ -187,7 +206,93 @@ def collapse_geographic_clones( UK_COPY_LINEAGE, merge=(GEOGRAPHIC_SUPPORT_BRANCH,), ) - return collapse_replicas(keys, rows, weights, inputs=inputs) + ids = tables[entity][UK_NATIONAL_SCHEMA.entity_id_column(entity)].to_numpy() + return fold_replicas(keys, rows, weights, inputs=inputs, order=ids) + + +def collapse_geographic_clones( + tables: Mapping[str, pd.DataFrame], + entity: str, + rows: np.ndarray, + weights: np.ndarray, + *, + inputs: pd.DataFrame, +) -> tuple[np.ndarray, np.ndarray]: + """The kept rows and weights of :func:`fold_geographic_clones`. + + Each survey record trains on its lowest-id clone among ``rows``, with + the weight of the clones that agree with it on ``inputs``; clones that + differ stay rows of their own. Returns the kept rows and the weights + over every row of ``entity``. + """ + + fold = fold_geographic_clones(tables, entity, rows, weights, inputs=inputs) + return fold.kept, fold.weights + + +def geographic_fold_receipt( + tables: Mapping[str, pd.DataFrame], + entity: str, + fold: ReplicaFold, + *, + stage: str, +) -> dict[str, object] | None: + """How a fit's geographic fold met its survey records, or None at one clone. + + ``fold`` is :func:`fold_geographic_clones`' fold of ``entity``'s rows + over ``tables``. On a frame of more than one geographic clone + (:func:`~microcosm.build.uk_runtime.geographic_clones.geographic_clone_count`) + the receipt holds: + + - ``clone_count``: the frame's clone count ``K``; + - ``records``: the true source households of the records the fit + selected; + - ``varied_records``: the source households of the selected records + whose clones did not all fold into one row, since they differ on what + the fit reads; + - ``varied_share``: the share of the selected source households that + vary, floored to the hundredth, where both the households that vary + and those that do not read as rounded numbers rather than bands; + otherwise it repeats ``varied_records``. A share to the hundredth of a + few dozen sources would give back where in its band either count + lies. + + Both counts are of distinct true sources on rows of positive weight, + rounded as every receipt count is (``receipt_counts.rounded_count``), so + a household's clones and copies count once. A frame of one clone returns + None and its evidence is left as it was. Variety is recorded, never + refused: clones that differ stay rows of their own in the fit. + """ + + clone_count = geographic_clone_count(tables["household"], stage=stage) + if clone_count == 1: + return None + # receipt_counts reads this module's lineage, so it is imported here. + from microcosm.build.uk_runtime import receipt_counts + + sources = receipt_counts.record_source_households(tables, entity, stage=stage) + selected = fold.representative >= 0 + records = receipt_counts.distinct_count(sources, selected, fold.weights) + varied = receipt_counts.distinct_count(sources, fold.varied, fold.weights) + varied_band = receipt_counts.rounded_count(varied) + # A varied record is a selected one, so the rest are the selected sources + # none of whose records varied. + steady_band = receipt_counts.rounded_count(records - varied) + share_given = ( + varied > 0 + and not isinstance(varied_band, str) + and not isinstance(steady_band, str) + ) + return { + "clone_count": clone_count, + "records": receipt_counts.rounded_count(records), + "varied_records": varied_band, + "varied_share": ( + receipt_counts.floored_share(varied / records) + if share_given + else varied_band + ), + } def without_geographic_branch( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/inspection_export.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/inspection_export.py index 60e8a7ba0e..68b39c7c12 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/inspection_export.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/inspection_export.py @@ -39,7 +39,45 @@ class UKInspectionExportPrepareKernel(graph_terminal.UKExportPrepareKernel): ref = "uk.full-export.prepare-inspection@1" def implementation_hash(self) -> str: - return source_hash(sys.modules[__name__], graph_terminal) + from microcosm.frame import materialize as frame_materialize + + from .. import gates + from . import ( + atomic_area_support, + geographic_clones, + geography_ladder, + graph_population, + hmrc_uprating, + local_authority_input, + national_frame, + rowwise_dataset, + rowwise_geography, + weight_year, + ) + + # The description computes through what the export kernel's does + # (microcosm#931): the ladder gate's results and region codes, the + # area code frames, the engine tables and the clone index columns + # geographic_clones names, the dataset year's weights and the + # engine's projection factor, the frame it rebuilds, and the + # descriptor's canonical encoding. + return source_hash( + sys.modules[__name__], + graph_terminal, + gates, + atomic_area_support, + geographic_clones, + geography_ladder, + graph_population, + hmrc_uprating, + local_authority_input, + national_frame, + rowwise_dataset, + rowwise_geography, + weight_year, + frame_materialize, + canonical_json, + ) def run(self, context: KernelContext) -> KernelResult: if context.params.get("inspection_export") is not True: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/isa_subscriptions.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/isa_subscriptions.py index 37dafd7564..c4debc1b8c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/isa_subscriptions.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/isa_subscriptions.py @@ -27,7 +27,8 @@ cells of the same income band with room; nobody is made a holder. 3. **How much (2023-24).** Within each (type, income band) the subscribers walk Table 9.7's band masses from the top, ranked by a key that blends their balance - rank with a keyed uniform, so larger holders tend to pay in more. Amounts are + rank with a keyed uniform, so larger holders tend to pay in more; holders tied + on a balance share its rank, so the keyed uniform alone orders them. Amounts are spread evenly inside each band on weight-proportional strata; GBP 20,000 is a point mass. 4. **The split for people paying into both** (María's ruling, 28 October 2026). @@ -61,8 +62,12 @@ import pandas as pd from microcosm.build.source_manifest import SourceStageSpec -from microcosm.build.stochastic_assignment import stable_identity_uniforms +from microcosm.build.stochastic_assignment import ( + stable_identity_uniform, + stable_identity_uniforms, +) from microcosm.build.uk_runtime.cgt_imputation import ( + _KeyedOffset, _truncated_exponential_quantile, _walk_bands, uk_cgt_component_sum_income, @@ -199,18 +204,10 @@ class ISASubscriptionsError(ValueError): """Raised when the inputs or the declaration do not carry what the stage needs.""" -class _KeyedOffset: - """One identity-keyed offset in the shape ``_walk_bands`` reads (``random()``).""" - - def __init__(self, value: float) -> None: - self._value = float(value) - - def random(self) -> float: - return self._value - - def _keyed_offset(seed: int, salt: str, key: str) -> _KeyedOffset: - return _KeyedOffset(stable_identity_uniforms([key], seed=seed, salt=salt)[0]) + """One walk's offset, keyed on its label, in the shape ``_walk_bands`` reads.""" + + return _KeyedOffset(stable_identity_uniform(key, seed=seed, salt=salt)) # -------------------------------------------------------------------------- @@ -995,17 +992,38 @@ def _subscription_bands() -> list[tuple[float, float]]: SUBSCRIPTION_BANDS = _subscription_bands() -def _weighted_rank(values: np.ndarray, weights: np.ndarray) -> np.ndarray: - """Each value's weighted mid-rank in [0, 1] (ties share a rank).""" +def _weighted_rank( + values: np.ndarray, weights: np.ndarray, ids: np.ndarray +) -> np.ndarray: + """Each value's weighted mid-rank in [0, 1]; tied values share one rank. + + A value held by one row keeps its own mid-rank, the weight below it plus + half its own, over the total. Every row of a group tied on one value takes + the group's mid-rank, ``(F(x-) + F(x)) / 2``: the weight below the group + plus half the group's. The weights accumulate in (value, id) order, so the + ranks do not depend on row order, and the keyed uniform each caller blends + in orders tied rows exchangeably. Tied rows used to rank by row position, + so the copies and clones that carry their root's balance ranked by where + they sat in the table, clone 0 lowest (microcosm#931). + """ - order = np.argsort(values, kind="stable") - cumulative = np.cumsum(weights[order]) + order = np.lexsort((ids, values)) + ordered = values[order] + ordered_weights = weights[order] + cumulative = np.cumsum(ordered_weights) total = float(cumulative[-1]) if len(cumulative) else 0.0 ranks = np.empty(len(values), dtype=float) if total <= 0: ranks[:] = 0.5 return ranks - ranks[order] = (cumulative - weights[order] / 2.0) / total + # A singleton's mid-rank, computed as before so it is bit for bit what it + # was wherever the rows arrive in id order (the spine's tables do). + mid = cumulative - ordered_weights / 2.0 + first = np.searchsorted(ordered, ordered, side="left") + last = np.searchsorted(ordered, ordered, side="right") - 1 + below = np.concatenate(([0.0], cumulative))[first] + mid = np.where(last > first, (below + cumulative[last]) / 2.0, mid) + ranks[order] = mid / total return ranks @@ -1036,7 +1054,8 @@ def draw_band_amounts( person_ids[rows], seed=seed, salt=f"{salt}:rank" ) rank_key = ( - balance_rank_weight * _weighted_rank(balance[rows], row_weights) + balance_rank_weight + * _weighted_rank(balance[rows], row_weights, person_ids[rows]) + (1.0 - balance_rank_weight) * uniforms ) ranked = np.lexsort((person_ids[rows], -rank_key)) @@ -1064,9 +1083,9 @@ def draw_band_amounts( cumulative = np.cumsum(row_weights[ordered]) tops = cumulative / cumulative[-1] bottoms = np.concatenate(([0.0], tops[:-1])) - jitter = stable_identity_uniforms( - [f"{key}:{band}"], seed=seed, salt=f"{salt}:jitter" - )[0] + jitter = stable_identity_uniform( + f"{key}:{band}", seed=seed, salt=f"{salt}:jitter" + ) quantiles = bottoms + (0.05 + 0.9 * jitter) * (tops - bottoms) amounts[ordered] = lower + quantiles * (upper - lower) return amounts @@ -1294,7 +1313,9 @@ def draw_lifetime_isa_payments( key = ( spec.balance_rank_weight * _weighted_rank( - recipients["lifetime_isa_balance"].to_numpy(float)[rows], payer_weights + recipients["lifetime_isa_balance"].to_numpy(float)[rows], + payer_weights, + person_ids, ) + (1.0 - spec.balance_rank_weight) * uniforms ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py index 8cfa63163a..8474840e2a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py @@ -67,6 +67,7 @@ gambling_participation_operation, top_up_gambling_participation, ) +from microcosm.build.uk_runtime.geographic_clones import geographic_clone_count from microcosm.build.uk_runtime.household_income import ( UK_HBAI_GROSS_INCOME_DEDUCTIONS, UK_HBAI_NET_INCOME_PREDICTOR, @@ -1110,7 +1111,12 @@ def __call__(self, frame: Frame) -> Frame: gate_calibration=_qrf_gate_calibration(self.stage), ) clip_result = support_clip_to_donor( - imputation.draws, donor, exempt=support_clip_exempt(self.stage) + imputation.draws, + donor, + exempt=support_clip_exempt(self.stage), + clone_count=geographic_clone_count( + frame.table("household"), stage=UK_LCFS_CONSUMPTION_STAGE_NAME + ), ) household_draws, shares_outside = clip_excise_shares( clip_result.clipped, shares @@ -5517,6 +5523,7 @@ def support_clip_to_donor( donor: pd.DataFrame, *, exempt: set[str] | None = None, + clone_count: int = 1, ) -> UKSupportClipResult: return support_clip_to_donor_with_receipt( draws, @@ -5524,6 +5531,7 @@ def support_clip_to_donor( columns=UK_LCFS_CONSUMPTION_TARGET_COLUMNS, stage=UK_LCFS_CONSUMPTION_STAGE_NAME, exempt=exempt, + clone_count=clone_count, ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/local_authority_input.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/local_authority_input.py index 6a80b7b6ec..4753c7d6e5 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/local_authority_input.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/local_authority_input.py @@ -15,11 +15,16 @@ ``local_authority`` disagrees with its code; and the rowwise clone verifies the assigned member names against the installed engine (:func:`verify_local_authority_engine_domain`), with the whole roster asserted -by ``test_uk_local_authority_input.py``. No battery ``enum_domain`` gate is -declared for this column: that family needs a ``rules_engine`` evidence -artifact the rowwise local battery never supplies, and -``uk_local_geography_ladder_post_calibration`` already reruns the ladder gate, -consistency check included, on the calibrated candidate (microcosm#953). +by ``test_uk_local_authority_input.py``. The spine writes the column in its +``local_authority`` stage, and ``spi_copy_areas`` for the SPI copies and +income band donors, so the spine battery checks it at its transferred +boundary: ``uk_local_authority_enum_domain`` holds every row to the installed +engine's enum, whose domain the battery's rules engine supplies, and +``uk_geography_ladder`` runs the ladder gate, consistency check included +(microcosm#953, microcosm#931). The rowwise local battery supplies no +``rules_engine`` evidence artifact, so it declares no ``enum_domain`` gate for +the column; ``uk_local_geography_ladder_post_calibration`` reruns the ladder +gate on the calibrated candidate. """ from __future__ import annotations @@ -189,13 +194,15 @@ def local_authority_consistency_failures( """Report rows whose engine input disagrees with their code (gate use). Returns failure messages and never raises, so the ladder gate can carry - the verdict through the local battery. + the verdict through the local battery. Each names the check and the + column, never a count of rows or an example (C-32): the gates that read + them run on frames whose rows are SPI copies and geographic clones of + survey households (microcosm#931). """ missing = [c for c in (code_column, input_column) if c not in household.columns] if missing: return [f"household table is missing local authority column(s): {missing}"] - total = len(household) codes = ( household[code_column].fillna("").astype(str).str.strip().to_numpy(dtype=object) ) @@ -210,14 +217,13 @@ def local_authority_consistency_failures( failures: list[str] = [] blank = np.array([key == "" for key in keys], dtype=bool) if blank.any(): - failures.append(f"{input_column}: {int(blank.sum())}/{total} row(s) are blank") + failures.append(f"{input_column}: some row(s) are blank") expected = [table.get(code) for code in codes] off_roster = np.array([value is None for value in expected], dtype=bool) if off_roster.any(): - examples = sorted(set(codes[off_roster].tolist()))[:5] failures.append( - f"{code_column}: {int(off_roster.sum())}/{total} row(s) are not on " - f"the April 2023 local authority roster; examples {examples}" + f"{code_column}: some row(s) are not on the April 2023 local " + "authority roster" ) mismatched = np.array( [ @@ -229,16 +235,7 @@ def local_authority_consistency_failures( dtype=bool, ) if mismatched.any(): - examples = sorted( - { - f"{code}->{key}" - for code, key in zip(codes[mismatched], keys[mismatched], strict=True) - } - )[:5] - failures.append( - f"{input_column}: {int(mismatched.sum())}/{total} row(s) disagree with " - f"{code_column}; examples {examples}" - ) + failures.append(f"{input_column}: some row(s) disagree with {code_column}") return failures diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py index ef2cff31f5..bbf1fff656 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/measure_simulation.py @@ -5,7 +5,7 @@ import functools import json import warnings -from collections.abc import Mapping +from collections.abc import Iterable, Mapping from datetime import date from importlib import resources as importlib_resources from pathlib import Path @@ -19,6 +19,9 @@ evaluate_named_term, named_term_leaves, ) +from microcosm.build.uk_runtime.annuity_trust_column import ( + ANNUITY_AND_TRUST_INCOME_COLUMN, +) from microcosm.build.uk_runtime.atomic_area_support import ( without_uk_native_alias_columns, ) @@ -42,7 +45,6 @@ SELF_EMPLOYMENT_INCOME_ANNUAL_SHARE, SELF_EMPLOYMENT_INCOME_OTHER_SPELLS, ) -from microcosm.build.uk_runtime.frs_spine import ANNUITY_AND_TRUST_INCOME_COLUMN from microcosm.build.uk_runtime.high_value_homes import ( high_value_home_mask, uk_high_value_home_thresholds, @@ -86,6 +88,12 @@ ) from microcosm.calibrate import TargetRegistry from microcosm.frame import Frame, MassChangeRecord, Weights +from microcosm.frame.adapters.engine_population import ( + MEASURE_PASS, + EngineBlock, + PoolShare, + validate_pool_shares, +) from microcosm.frame.adapters.policyengine_uk import ( frame_isa_income_contract, uk_dataset_through, @@ -93,6 +101,7 @@ uk_whole_rebalancing_years, validate_uc_claimant_input, ) +from microcosm.frame.adapters.policyengine_uk_population import UK_ENGINE_POPULATION _ENTITY_LINK = {"benunit": "person_benunit_id", "household": "person_household_id"} _ENTITY_ID = { @@ -299,10 +308,10 @@ def savings_interest_outside_isas(frame: Any) -> np.ndarray: #: private_pension_income, where the engine's means tests, household income #: and tax read them (microcosm#1428). The frame stores each person's annuity #: and trust payments at the dataset's year -#: (frs_spine.ANNUITY_AND_TRUST_INCOME_COLUMN), and their share of the stored -#: private pension carries them to the measurement year on the engine's own -#: private pension path. A frame without the column (the enhanced FRS -#: reference) reads the engine's other income alone. A reserved name. +#: (annuity_trust_column.ANNUITY_AND_TRUST_INCOME_COLUMN), and their share of +#: the stored private pension carries them to the measurement year on the +#: engine's own private pension path. A frame without the column (the enhanced +#: FRS reference) reads the engine's other income alone. A reserved name. UK_SPI_OTHER_INVESTMENT_INCOME = "other_investment_income_spi_concept" @@ -421,6 +430,24 @@ def annuity_and_trust_share(frame: Any) -> np.ndarray: ("esa_contrib_reported", CONTRIBUTORY_BENEFITS_ANNUAL_SHARE, None), ("incapacity_benefit_reported", CONTRIBUTORY_BENEFITS_ANNUAL_SHARE, None), ) +#: The growth series policyengine-uk's dataset projection carries each input +#: with other spells by (its ``data/uprating_indices.yaml``: pay, stored as +#: ``employment_income``, by average earnings; profit by mixed income per +#: head). The other spells grow from the dataset's year by the product of the +#: engine's own rates in that series, read from its parameters, so the factor +#: is the same in every per-clone engine block and in one engine over the pool +#: (María's ruling of 2026-11-06, microcosm#931; it was the median of the +#: engine's own year-on-year ratios, a statistic of whichever block ran). +_ANNUAL_INCOME_OTHER_SPELLS_GROWTH: Mapping[str, str] = MappingProxyType( + { + "employment_income_before_lsr": ( + "gov.economic_assumptions.yoy_growth.obr.average_earnings" + ), + "self_employment_income": ( + "gov.economic_assumptions.yoy_growth.obr.per_capita.mixed_income" + ), + } +) #: Taxable benefits whose entitlement reads earnings, held at their baseline #: values: a carer was paid in the year at the earnings of the time, which the #: current concept records, not at the year's total. @@ -916,9 +943,11 @@ def _engine_scratch_frame( (and only those: the resolver's own frame keeps the block's true mass). A per-clone engine block carries a K-th of the pool, so a formula that allocates a national total by weighted share would hand every block the - whole total; scaled to the pool's mass the block's weighted sums equal - the pool's and the formula is exact for identical clone copies. The - scaling is declared on the engine frame's mass log. + whole total; scaled to the pool's mass each block hands out a share of it + instead. That share is the pool's for identical clone copies; for clones + that differ, the resolver's pool-share correction (``engine_pool_shares``) + sets each share formula to the pool's value. The scaling is declared on + the engine frame's mass log. The identifiers leave as the release's too (microcosm#1194, ``national_frame.uk_release_identifier_tables``): policyengine-uk stores @@ -1051,7 +1080,19 @@ def engine_bus_pricing_parameters(tax_benefit_system: Any, year: int) -> dict[st class UKMeasureResolver: - """B2 measure provider backed by a policyengine-uk Microsimulation.""" + """B2 measure provider backed by a policyengine-uk Microsimulation. + + ``engine_pool_shares`` makes the resolver one per-clone engine block of + the dense measure step (microcosm#931): a :class:`PoolShare` for each of + policyengine-uk's share formulas (``UK_ENGINE_POPULATION.corrected``), + carrying the block's ``engine_weight_scale``. Right after its simulation + is built the resolver opens the shared :class:`EngineBlock` on it, with + the UK declaration and the measure pass, which records and never raises + (María's ruling of 2026-11-06). Every simulation the resolver builds then + reads the pool's shares, and :meth:`receipt` says how; + :meth:`close_engine_block` judges what the block computed, once it has + computed everything. Scratch mode only. + """ def __init__( self, @@ -1063,11 +1104,25 @@ def __init__( microsimulation_factory: Any | None = None, engine_weight_scale: float | None = None, engine_identifier_universe: Any | None = None, + engine_pool_shares: Mapping[str, PoolShare] | None = None, ): self.year = int(year) self._engine_weight_scale = ( None if engine_weight_scale is None else float(engine_weight_scale) ) + self._engine_pool_block: EngineBlock | None = None + if engine_pool_shares is not None: + # A block's shares carry the factor its engine weights do (1 when + # unscaled); checked before any engine is built. + validate_pool_shares( + UK_ENGINE_POPULATION, + engine_pool_shares, + weight_scale=( + 1.0 + if self._engine_weight_scale is None + else self._engine_weight_scale + ), + ) policyengine_uk = _policyengine_uk_module() factory = ( microsimulation_factory @@ -1100,6 +1155,11 @@ def __init__( "an identifier universe needs scratch-mode resolution; a direct " "H5 source carries its own identifiers." ) + if engine_pool_shares is not None: + raise ValueError( + "pool shares need scratch-mode resolution; a direct H5 source " + "carries its own weights and is no engine block of a pool." + ) dropped = () source_path = Path(simulation_source) mode = "direct_h5" @@ -1161,6 +1221,10 @@ def __init__( validate_uc_claimant_input(self.simulation, frame.table("person"), self.year) self.contract_targets = _uk_contract_targets() bound_cgt_periods = {} + # The years a dated measure is computed at besides this one: an engine + # block corrects its shares at each one its scratch engine holds + # (microcosm#931). + bound_years: set[int] = set() for target_id, target in self.contract_targets.items(): binding = target["bindings"]["policyengine"] for name in _binding_variable_names(binding): @@ -1173,6 +1237,8 @@ def __init__( raise ValueError( f"company car benefit year/period mismatch: {target_id}" ) + if benefit_year is not None: + bound_years.add(int(benefit_year)) cgt_measure = _cgt_model_measure(name, self.year) if cgt_measure is None: continue @@ -1185,6 +1251,7 @@ def __init__( "model_variable": model_variable, "measurement_period": measure_year, } + bound_years.add(int(measure_year)) self._receipt = { "mode": mode, "source_path": str(source_path), @@ -1222,6 +1289,55 @@ def __init__( self._annual_income_fallback: str | None = None self._annual_income_simulation_cache: Any | None = None self._annual_income_factors: dict[str, float] = {} + if engine_pool_shares is not None: + # After the simulation is built and its roles validated, before + # anything reads it: the dataset's year, this year and every + # dated measure's year up to this one are the periods the block + # computes at. Pool shares need a scratch engine, which holds no + # later year: a measure dated later is refused before it is + # computed (:meth:`_refuse_dated_measure_after_window`), so the + # block neither checks the engine's growth nor sets the pool's + # shares there. + dataset_year = self._dataset_year() + held_years = { + bound_year for bound_year in bound_years if bound_year <= self.year + } + self._engine_pool_block = EngineBlock( + self.simulation, + engine_pool_shares, + declaration=UK_ENGINE_POPULATION, + policy=MEASURE_PASS, + periods=(dataset_year, self.year, *held_years), + dataset_year=dataset_year, + ) + self._factory = self._engine_pool_block.wrap(self._factory) + + def close_engine_block(self) -> None: + """Judge what this engine block's simulations computed (microcosm#931). + + The dense measure step calls it once the block has resolved every + measure and local metric, before reading :meth:`receipt`; the + receipt then carries the verdict (``engine_population_reach``). A + resolver that is no engine block has nothing to judge. + """ + + block = getattr(self, "_engine_pool_block", None) + if block is not None: + block.close() + + def _note_engine_request(self, variable: str) -> None: + block = getattr(self, "_engine_pool_block", None) + if block is not None: + block.note_request(variable, self.year) + + def _refuse_pool_share_substitution( + self, names: Iterable[str | None], route: str + ) -> None: + """Refuse a sibling that would change what the block's correction read.""" + + block = getattr(self, "_engine_pool_block", None) + if block is not None: + block.refuse_substitution(names, route) def use_current_income_for_annual_measures(self, *, reason: str) -> None: """Let a frame without annual income descriptors read current values. @@ -1388,6 +1504,7 @@ def _refuse_dated_measure_after_window(self, variable: str) -> None: def compute(self, entity: str, variable: str) -> tuple[np.ndarray, str]: self._refuse_dated_measure_after_window(variable) + self._note_engine_request(variable) if variable in UK_ANNUAL_INCOME_MEASURES: return self._annual_income_measure(entity, variable) if variable in UK_HOUSEHOLD_ENERGY_MEASURES: @@ -1445,6 +1562,7 @@ def counterfactual_delta( ): raise ValueError(f"unsupported output_delta {direction!r}") metric = str(binding.get("metric_name") or output) + self._note_engine_request(output) allocation = binding.get("allocation") if allocation is not None: rate_class = _declared_rate_class(allocation, output, direction, entity) @@ -1689,6 +1807,11 @@ def _new_simulation(self) -> Any: that would. A direct H5 keeps the engine's whole projection, and is built where no cut of that loop is in force (:func:`~microcosm.frame.adapters.policyengine_uk.uk_whole_rebalancing_years`). + + Once the resolver is an engine block, ``self._factory`` is the block's + wrapped factory (``EngineBlock.wrap``), so each sibling built here, + the annual-income and counterfactual simulations, reads the pool's + shares and is built inside the same cut (microcosm#931). """ if self._simulation_end_year is None: @@ -1715,6 +1838,15 @@ def _annual_income_simulation(self) -> Any: if self._annual_income_simulation_cache is not None: return self._annual_income_simulation_cache + self._refuse_pool_share_substitution( + ( + *(name for name, _share, _other in _ANNUAL_INCOME_INPUTS), + "state_pension", + _COMPANY_CAR_AVAILABILITY, + *_ANNUAL_INCOME_HELD_VARIABLES, + ), + "the annual-income simulation", + ) person = self.frame.table("person") simulation = self._new_simulation() annual_inputs: dict[str, np.ndarray] = {} @@ -1772,32 +1904,53 @@ def _annual_income_simulation(self) -> Any: def _engine_projection_factor(self, name: str) -> float: """The factor the engine carries ``name`` by from the dataset's year. - Read off the engine's own projection, so the other spells grow by the - series the current amount grows by; a projection that is not one - factor for every row refuses, since no single factor then carries the - other spells as the engine carries pay. + The product, over each year after the dataset's up to this one, of + one plus the engine's own growth rate in the series its dataset + projection carries ``name`` by (``_ANNUAL_INCOME_OTHER_SPELLS_GROWTH``), + read from the simulation's parameters as the projection reads them. + A parameter, not a statistic of the rows, so every engine block and + one engine over the pool carry the other spells alike (microcosm#931). + The engine's own arrays must agree with it, within 1e-5, on every row + that carries ``name``; otherwise no single factor carries the other + spells as the engine carries pay, and it refuses. """ + series = _ANNUAL_INCOME_OTHER_SPELLS_GROWTH.get(name) + if series is None: + raise ValueError( + f"No engine growth series is declared for {name}, so its other " + "spells cannot be carried to the measurement year." + ) dataset_year = self._dataset_year() + if self.year < dataset_year: + raise ValueError( + f"The engine projects the dataset from {dataset_year} forward, so " + f"the other spells of {name} cannot be carried back to {self.year}." + ) + growth = self.simulation.tax_benefit_system.parameters.get_child(series) + factor = 1.0 + for year in range(dataset_year + 1, self.year + 1): + factor *= 1.0 + float(growth(str(year))) then = _values(self.simulation.calculate(name, dataset_year)).astype(float) now = _values(self.simulation.calculate(name, self.year)).astype(float) carried = np.abs(then) > 0 if not carried.any(): raise ValueError( f"No row carries {name} in {dataset_year}, so the engine's factor " - f"to {self.year} for the other spells cannot be read." + f"to {self.year} for the other spells cannot be checked." ) factors = now[carried] / then[carried] - factor = float(np.median(factors)) if (np.abs(factors - factor) > 1e-5 * max(abs(factor), 1.0)).any(): raise ValueError( f"The engine does not carry {name} from {dataset_year} to " - f"{self.year} by one factor, so the other spells cannot follow it." + f"{self.year} by one factor, its growth series {series}, so the " + "other spells cannot follow it." ) self._annual_income_factors[name] = factor return factor def _counterfactual_simulation(self, zeroed: str, folded: str | None) -> Any: + self._refuse_pool_share_substitution((zeroed, folded), "a counterfactual") key = (zeroed, folded) if key in self._counterfactuals: return self._counterfactuals[key] @@ -1904,6 +2057,12 @@ def receipt(self) -> dict[str, Any]: } if getattr(self, "_annual_income_measures_used", None): receipt["annual_income_measures"] = self._annual_income_receipt() + block = getattr(self, "_engine_pool_block", None) + if block is not None: + # Only an engine block of the dense measure step carries these + # (engine_pool_shares, and engine_population_reach once judged), + # so a single-block receipt keeps its bytes (microcosm#931). + receipt.update(block.receipt_entries()) return receipt def _annual_income_receipt(self) -> dict[str, Any]: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py index 1c39b2fcbc..e20dafcc62 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py @@ -55,6 +55,7 @@ "UK_EXPORTED_WEIGHT_COLUMNS", "UK_HOUSEHOLD_WEIGHT_KIND_ATTR", "UK_MASS_LOG_ATTR", + "UK_NATIONAL_EXPORT_DROPPED_COLUMNS", "UK_NATIONAL_H5_TABLES", "UK_NATIONAL_SCHEMA", "UK_TIME_PERIOD_METADATA_KEY", @@ -75,6 +76,7 @@ "uk_time_period", "validate_uk_national_frame", "uk_published_release_frame", + "uk_release_export_dropped_columns", "uk_release_export_frame", "write_uk_national_frame", ] @@ -779,8 +781,10 @@ def _write_uk_single_year_tables( ), "household": ( "property_wealth", - # The value the spine's regional home values replaced (microcosm#1112): - # the dense node ranks on it, and it is no engine input. + # The WAS chain's own draw of the main residence (microcosm#1112, + # microcosm#931): was_wealth ranks England and Wales owners on it to + # localise their homes and keeps it beside the localised value, and + # regional_property_uprating divides by it. It is no engine input. "main_residence_value_rank_basis", # Since policyengine-uk 2.164.0 these add the person cash_isa_balance # and stocks_and_shares_isa_balance, which the isa_subscriptions stage @@ -790,9 +794,80 @@ def _write_uk_single_year_tables( # and the split reads them); the engine sums the person balances. "cash_isa", "stocks_and_shares_isa", + # The root's two area-conditioning carriers + # (frs_spine.FRS_AREA_CONDITIONING_COLUMNS, microcosm#931): each + # household's IMD decile within its nation and whether its council + # tax band is a band the FRS gives it that its region uses. The + # clone-first spine's area assignment reads them; they are no engine + # inputs. + "imd_decile", + "council_tax_band_observed", + # The council-tax localisation's two carriers (microcosm#931): each + # bill's provenance from frs_council_tax + # (frs_council_tax.FRS_COUNCIL_TAX_BILL_PROVENANCE_COLUMN), which + # council_tax_localise reads to choose a household's path, and the + # paid bill council_tax_localise keeps from before it localises + # (spine_council_tax.COUNCIL_TAX_FRS_BILL_COLUMN), which was_wealth's + # council-tax predictor reads. They are no engine inputs. + "council_tax_bill_provenance", + "council_tax_frs_bill", ), } +#: What the national line's release boundary drops beside +#: :data:`UK_RELEASE_EXPORT_DROPPED_COLUMNS` (microcosm#931): each entity's +#: geographic clone index, spelled as +#: ``geographic_clones.ladder_clone_index_column`` spells it (here, so that +#: this module imports nothing more; a test holds the spellings equal). The +#: national line builds on a spine of one geographic clone, so every index is +#: zero there: a constant column the degenerate-surface gate would fail, which +#: tells a consumer nothing. The dense line's export keeps them, as the +#: lineage of its clones. +UK_NATIONAL_EXPORT_DROPPED_COLUMNS: Mapping[str, tuple[str, ...]] = MappingProxyType( + { + "person": ("person_clone_index",), + "benunit": ("benunit_clone_index",), + "household": ("household_clone_index",), + } +) + +#: The release lines an export can be cut for, as a gate context declares its +#: ``release_line`` (``weighted_integrity.UK_RELEASE_LINES``, spelled here so +#: that this module imports nothing more; a test holds the two equal). +_UK_EXPORT_LINES = frozenset({"national", "local"}) + + +def uk_release_export_dropped_columns( + line: str | None = None, +) -> Mapping[str, tuple[str, ...]]: + """What the release boundary drops on ``line``, by entity (microcosm#931). + + On every line the reviewed export exclusions + (:data:`UK_RELEASE_EXPORT_DROPPED_COLUMNS`); on the national line each + entity's geographic clone index as well + (:data:`UK_NATIONAL_EXPORT_DROPPED_COLUMNS`), after them. ``line`` is a + gate context's ``release_line``: ``"national"``, ``"local"`` (the dense + line's K-clone pool) or ``None``, a caller that declares no line, which + the boundary treats as the local line. Any other is refused. + """ + + if line is not None and line not in _UK_EXPORT_LINES: + raise ValueError( + f"A release export's line must be one of {sorted(_UK_EXPORT_LINES)} " + f"or None, got {line!r}." + ) + if line != "national": + return UK_RELEASE_EXPORT_DROPPED_COLUMNS + return MappingProxyType( + { + entity: ( + *UK_RELEASE_EXPORT_DROPPED_COLUMNS.get(entity, ()), + *UK_NATIONAL_EXPORT_DROPPED_COLUMNS.get(entity, ()), + ) + for entity in ("person", "benunit", "household") + } + ) + #: The largest identifier policyengine-uk stores. Its five identifier #: variables (``person_id``, ``benunit_id``, ``household_id``, @@ -1023,36 +1098,40 @@ def uk_build_identifier_frame(frame: Frame) -> Frame: ) -def uk_release_export_frame(frame: Frame) -> Frame: +def uk_release_export_frame(frame: Frame, *, line: str | None = None) -> Frame: """The frame the release boundary writes: reviewed export exclusions dropped. - The three area codes leave under the consumers' names (microcosm#1114, - ``geography_ladder.UK_EXPORT_AREA_CODE_COLUMNS``); a national frame - carries none. The identifiers leave inside the 32-bit range the engine - stores, with the build identifiers beside them (microcosm#1194, - :func:`uk_release_identifier_tables`). A frame that already left passes - unchanged. + ``line`` is a gate context's ``release_line``; the boundary drops what + :func:`uk_release_export_dropped_columns` lists for it, on the national + line each entity's geographic clone index too (microcosm#931). The three + area codes leave under the consumers' names on every line (microcosm#1114, + ``geography_ladder.UK_EXPORT_AREA_CODE_COLUMNS``). The identifiers leave + inside the 32-bit range the engine stores, with the build identifiers + beside them (microcosm#1194, :func:`uk_release_identifier_tables`). A + frame that already left on ``line`` passes unchanged: a household table + with a consumer name and no ladder name is not renamed again, since the + release cut's gates re-export the candidate they load (microcosm#931), + and one with both is refused. """ - from .geography_ladder import export_area_code_columns + from .geography_ladder import UK_EXPORT_AREA_CODE_COLUMNS, export_area_code_columns + drops = uk_release_export_dropped_columns(line) tables: dict[str, pd.DataFrame] = {} changed = False for entity in ("person", "benunit", "household"): table = frame.table(entity) dropped = [ - column - for column in UK_RELEASE_EXPORT_DROPPED_COLUMNS.get(entity, ()) - if column in table.columns + column for column in drops.get(entity, ()) if column in table.columns ] if dropped: table = table.drop(columns=dropped) changed = True - if entity == "household": - renamed = export_area_code_columns(table) - if renamed is not table: - table = renamed - changed = True + if entity == "household" and any( + ladder in table.columns for ladder in UK_EXPORT_AREA_CODE_COLUMNS + ): + table = export_area_code_columns(table) + changed = True tables[entity] = table if not _uk_build_identifier_state(tables): tables = uk_release_identifier_tables(tables) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_sampling.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_sampling.py index 1b130f25d9..889e001e11 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_sampling.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_sampling.py @@ -1,63 +1,35 @@ -"""Scale-ladder sampling policy for the UK national build (#627). - -The UK samples a mid-pipeline artifact: the certified compact already carries -geography/CG clone families (``clone_index`` reverses to the canonical -``clone_index == 0`` surface by ID arithmetic), SPI-synthetic and -capital-gains derivative rows that reverse **to the raw FRS surface** by -max-derived offsets (``max(canonical raw ids) + 1``), and the zero-weight SPI -dead channel whose rebuild enforces a per-stratum ``#base >= #dead`` quota. -A uniform household draw would break all three, so the UK policy over the -shared :mod:`microcosm.build.frame_sampling` core is: - -- **The sampling unit is the source FRS family**: the raw canonical - household plus every SPI/CG derivative and every geography clone that - reverses onto it through both ID arithmetics. A drawn family therefore - keeps clone reversal *and* the SPI/CG source reversal intact — a derived - row never survives without the raw row it reverses to. -- **The reversal constants are pinned by forced retention.** The stage - fence re-derives the clone multiplier (``10 ** len(str(max canonical - id))``) and the SPI/CG offsets (``max(canonical raw id) + 1``, - ``max(canonical pre-CG id) + 1``) from surviving data, so the families - carrying each argmax id — canonical, raw, and pre-CG, household and - person — are always retained and every constant is stable. -- **The draw is stratified by the raw canonical household's region.** - Channel flags vary *within* a source family (raw + SPI + CG rows travel - together), so they are not strata; the per-cell SPI quota is preserved - structurally instead — a dead row's in-cell base source is in its own - family — and a post-sample check re-asserts the full per-cell quota, - failing closed on any irregular input. -- **Sampled mass is renormalized to the full-source household total** (the - US stacked semantics #627 names), so the anchor population and the SPI - 50/50 mass-share allocation behave identically at every rung. +"""Scale-ladder sampling for the UK spine (#627). + +A spine built below f100 samples its FRS root household by household +(:func:`sample_uk_spine_root`) through :class:`UKSampledSpineRootTransform`, +which the root node hashes with this module, and below f100 with the shared +:mod:`microcosm.build.frame_sampling` core too. Sampled mass is renormalized +to the root's household total (the US stacked semantics #627 names). The +module also holds the rung tokens and the default sampling seed the UK builds +share. + +The dense line samples nothing of its own: it takes the bound spine's +sampling with its clones and areas (microcosm#931), so its sampler, which drew +whole source families from a spine, is retired. """ from __future__ import annotations +import sys from collections.abc import Mapping -import numpy as np -import pandas as pd - +from microcosm.build import frame_sampling from microcosm.build.frame_sampling import ( normalize_sampled_household_mass, sample_frame_households, - validate_sample_fraction, - validate_sample_seed, -) -from microcosm.build.uk_runtime.national_frame import validate_uk_national_frame -from microcosm.build.uk_runtime.spi_support import ( - HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN, - SPI_REPLACEMENT_STRATA_COLUMNS, ) from microcosm.frame import Frame __all__ = [ "UK_SAMPLE_RUNG_TOKENS", "UK_SAMPLE_SEED_DEFAULT", - "sample_uk_national_frame", - "sample_uk_spine_frame", - "uk_spine_source_family_units", - "uk_source_family_units", + "UKSampledSpineRootTransform", + "sample_uk_spine_root", ] #: The #624 scale-ladder rungs and their identity tokens, shared with the US @@ -69,485 +41,124 @@ } #: Default survey-sampling seed, matching the US stacked pipeline's. Distinct -#: from the build ``--seed`` (SPI replacement draw, donor bootstrap, QRF) so -#: dev-scale seed sweeps vary one draw at a time. +#: from the build ``--seed`` so dev-scale seed sweeps vary one draw at a time. UK_SAMPLE_SEED_DEFAULT = 578 -( - _CLONE_INDEX_COLUMN, - _CAPITAL_GAINS_CLONE_COLUMN, - _REGION_COLUMN, -) = SPI_REPLACEMENT_STRATA_COLUMNS - - -def _required_household_columns() -> tuple[str, ...]: - return ( - "household_id", - HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN, - *SPI_REPLACEMENT_STRATA_COLUMNS, - ) - - -def _strict_bool(values: pd.Series, *, label: str) -> np.ndarray: - if values.isna().any(): - raise ValueError(f"UK sample: {label} contains missing values.") - if pd.api.types.is_bool_dtype(values): - return values.to_numpy(dtype=bool) - if pd.api.types.is_numeric_dtype(values): - numeric = pd.to_numeric(values, errors="coerce") - if not numeric.isin([0, 1]).all(): - raise ValueError( - f"UK sample: {label} must contain only boolean/0/1 values." - ) - return numeric.to_numpy(dtype=float) != 0.0 - raise ValueError(f"UK sample: {label} must be a boolean column.") - - -def _int_column(values: pd.Series, *, label: str) -> np.ndarray: - if values.isna().any(): - raise ValueError(f"UK sample: {label} contains missing values.") - if not pd.api.types.is_integer_dtype(values.dtype): - raise ValueError( - f"UK sample: {label} must be integer-typed; got dtype {values.dtype}." - ) - return values.to_numpy(dtype=np.int64) - - -def _str_column(values: pd.Series, *, label: str) -> np.ndarray: - if values.isna().any(): - raise ValueError(f"UK sample: {label} contains missing values.") - array = values.to_numpy(dtype=object) - if not all(isinstance(value, str) and value for value in array): - raise ValueError( - f"UK sample: {label} must contain non-empty strings; numeric " - "codes are refused because equal codes of different dtypes " - "(1 vs 1.0) would silently split one stratum into two." - ) - return array - - -_SPINE_SOURCE_HOUSEHOLD_ID_COLUMN = "source_household_id" -_SPINE_SUPPORT_CLONE_INDEX_COLUMN = "household_support_clone_index" -_SPINE_CGT_SUPPORT_COPY_COLUMN = "household_is_cgt_support_copy" -_SPINE_CGT_RESIDENTIAL_ARM_COLUMN = "household_is_cgt_residential_clone" - - -def uk_spine_source_family_units(frame: Frame) -> tuple[np.ndarray, np.ndarray]: - """Return the UK spine's explicit source-family keys and raw-row strata. - - Every derivative row travels with the raw FRS household named by its - ``source_household_id``. The raw household owns the family's region, so - derivative flags and incidental derivative regions never split one - source family across sampling strata. - """ - - household = frame.table("household") - household_id_column = "household_id" - lineage_columns = { - _SPINE_SOURCE_HOUSEHOLD_ID_COLUMN, - _SPINE_SUPPORT_CLONE_INDEX_COLUMN, - HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN, - _CAPITAL_GAINS_CLONE_COLUMN, - _SPINE_CGT_SUPPORT_COPY_COLUMN, - _SPINE_CGT_RESIDENTIAL_ARM_COLUMN, - } - required = {household_id_column, _REGION_COLUMN, *lineage_columns} - missing = sorted(required - set(household.columns)) - present_lineage = lineage_columns & set(household.columns) - if not present_lineage: - raw_required = {household_id_column, _REGION_COLUMN} - raw_missing = sorted(raw_required - set(household.columns)) - if raw_missing: - raise ValueError( - "UK spine sample requires household identity and region columns; " - f"household is missing {raw_missing}." - ) - household_ids = _int_column( - household[household_id_column], label=household_id_column - ) - regions = _str_column(household[_REGION_COLUMN], label=_REGION_COLUMN) - return household_ids, np.asarray( - [f"region={region}" for region in regions], dtype=object - ) - if missing: - raise ValueError( - "UK spine sample has a partial lineage surface; " - f"household is missing {missing}." - ) - - household_ids = _int_column(household["household_id"], label="household_id") - units = _int_column( - household[_SPINE_SOURCE_HOUSEHOLD_ID_COLUMN], - label=_SPINE_SOURCE_HOUSEHOLD_ID_COLUMN, - ) - support_clone = _int_column( - household[_SPINE_SUPPORT_CLONE_INDEX_COLUMN], - label=_SPINE_SUPPORT_CLONE_INDEX_COLUMN, - ) - if (support_clone < 0).any(): - raise ValueError( - "UK sample: household_support_clone_index must be non-negative." - ) - spi = _strict_bool( - household[HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN], - label=HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN, - ) - capital_gains = _strict_bool( - household[_CAPITAL_GAINS_CLONE_COLUMN], - label=_CAPITAL_GAINS_CLONE_COLUMN, - ) - # A support split's root keeps its raw-row role; its created copies join - # the family through source_household_id (microcosm#1045). - support_copy = _strict_bool( - household[_SPINE_CGT_SUPPORT_COPY_COLUMN], - label=_SPINE_CGT_SUPPORT_COPY_COLUMN, - ) - # A residential split's incumbent arm keeps its raw-row role and ids; - # the residential arms join the family the same way (microcosm#1063). - residential_arm = _strict_bool( - household[_SPINE_CGT_RESIDENTIAL_ARM_COLUMN], - label=_SPINE_CGT_RESIDENTIAL_ARM_COLUMN, - ) - regions = _str_column(household[_REGION_COLUMN], label=_REGION_COLUMN) - raw = ( - (support_clone == 0) & ~spi & ~capital_gains & ~support_copy & ~residential_arm - ) - source_rows = raw & (household_ids == units) - source_keys = set(int(value) for value in units[source_rows]) - missing_sources = sorted(set(int(value) for value in units) - source_keys) - if missing_sources: - raise ValueError( - "UK spine sample: source_household_id family key(s) lack a " - "household_id-bearing raw row: " - f"{missing_sources[:5]}." - ) - - raw_region_by_unit: dict[int, str] = {} - multi_region: list[int] = [] - for unit in sorted(set(int(value) for value in units)): - family_regions = set(str(value) for value in regions[raw & (units == unit)]) - if len(family_regions) > 1: - multi_region.append(unit) - else: - raw_region_by_unit[unit] = next(iter(family_regions)) - if multi_region: - raise ValueError( - "UK spine sample: source family spans more than one raw-row " - f"region for key(s) {multi_region[:5]}." - ) - - strata = np.asarray( - [f"region={raw_region_by_unit[int(unit)]}" for unit in units], - dtype=object, - ) - return units, strata - - -def sample_uk_spine_frame( +def sample_uk_spine_root( frame: Frame, *, fraction: float, seed: int, -) -> tuple[Frame, dict[str, object]]: - """Sample complete explicit source families from a UK Microcosm spine.""" +) -> tuple[Frame, dict[str, object] | None]: + """Sample the spine build's FRS root at a rung below f100. - validate_sample_fraction(fraction, label="UK spine sample") - validate_sample_seed(seed, label="UK spine sample") - validate_uk_national_frame(frame) + Households are drawn by the shared core and their mass is renormalized + to the root's household total. At f100 the frame is returned as it is, + with no receipt. + """ - household = frame.table("household") - units, strata = uk_spine_source_family_units(frame) - pre_counts = pd.Series(units).value_counts(sort=False) - full_mass = float(frame.weights_for("household").total) - sampled, core_receipt = sample_frame_households( + if fraction == 1.0: + return frame, None + household_weight = frame.weights_for("household") + pre_households = int(len(frame.table("household"))) + sampled, receipt = sample_frame_households( frame, fraction=fraction, seed=seed, - source_name="UK spine", - unit_ids=units, - unit_strata=strata, - unit_noun="source family", - floor_context="the UK rowwise candidate", + source_name="UK FRS spine", ) normalized, factor = normalize_sampled_household_mass( sampled, - target_mass=full_mass, - source_name="UK spine", - ) - - post_units, _ = uk_spine_source_family_units(normalized) - post_counts = pd.Series(post_units).value_counts(sort=False) - incomplete = sorted( - int(unit) - for unit, count in post_counts.items() - if int(count) != int(pre_counts.loc[unit]) + target_mass=float(household_weight.total), + source_name="UK FRS spine", ) - if incomplete: - raise ValueError( - "UK spine sample retained incomplete source family key(s): " - f"{incomplete[:5]}." - ) - validate_uk_national_frame(normalized) return normalized, { "fraction": float(fraction), "seed": int(seed), - "rung_token": UK_SAMPLE_RUNG_TOKENS.get(float(fraction)), - "pre_household_count": int(len(household)), + "rung_token": UK_SAMPLE_RUNG_TOKENS[fraction], + "pre_household_count": pre_households, "post_household_count": int(len(normalized.table("household"))), - "pre_family_count": int(len(pre_counts)), - "post_family_count": int(len(post_counts)), "normalization_factor": float(factor), - "strata_count": int(len(np.unique(strata))), - "receipt": dict(core_receipt), + "receipt": dict(receipt), } -def uk_source_family_units( - frame: Frame, -) -> tuple[np.ndarray, tuple[int, ...], int]: - """Per-household-row source-FRS-family key, mirroring the stage fence. - - Reproduces ``_resolve_candidate_lineage``'s two-layer arithmetic: the - clone multiplier is ``10 ** max(1, len(str(canonical_max)))`` over the - ``clone_index == 0`` household **and** person ids; the SPI and CG - household offsets are ``max(canonical raw household id) + 1`` and - ``max(canonical pre-CG household id) + 1``; and every row's family key is - - ``household_id - clone_index*multiplier - spi*spi_offset - cg*cg_offset`` - - which must land on an existing canonical raw FRS household. Fails closed - before any draw on household-level rows the stage fence would refuse; - person-level lineage defects are outside this pre-check and surface at - stage one, where the fence stays strict. +class UKSampledSpineRootTransform: + """CREATE-stage adapter applying the declared sampling rung at ingest. - Returns: - The per-household-row family keys, the forced family keys (the - families carrying every argmax id the fence's constants are derived - from — retaining them pins the multiplier's digit count and both - offsets exactly), and the clone multiplier. + The spine build wraps the FRS root's transform in it. Its sampling code + lives here and in the shared core, not in the root stage's module, so it + declares them for the root node to hash (microcosm#931): this module at + every rung, since it resets the root's indexes at f100 too, and the shared + core only below f100, since :func:`sample_uk_spine_root` returns before + calling it at f100. """ - household = frame.table("household") - missing = sorted(set(_required_household_columns()) - set(household.columns)) - if missing: - raise ValueError( - "UK sample requires the certified compact's lineage and channel " - f"columns; household is missing {missing}." - ) - person = frame.table("person") - household_ids = _int_column(household["household_id"], label="household_id") - clone_index = _int_column(household[_CLONE_INDEX_COLUMN], label=_CLONE_INDEX_COLUMN) - if (clone_index < 0).any(): - raise ValueError("UK sample: clone_index must be non-negative.") - spi_flag = _strict_bool( - household[HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN], - label=HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN, - ) - cg_flag = _strict_bool( - household[_CAPITAL_GAINS_CLONE_COLUMN], - label=_CAPITAL_GAINS_CLONE_COLUMN, - ) - canonical_households = clone_index == 0 - if not canonical_households.any(): - raise ValueError("UK sample requires clone_index=0 canonical rows.") - - person_ids = _int_column(person["person_id"], label="person_id") - person_household = _int_column( - person["person_household_id"], label="person_household_id" - ) - household_position = pd.Series(np.arange(len(household_ids)), index=household_ids) - person_household_position = household_position.reindex(person_household) - if person_household_position.isna().any(): - raise ValueError( - "UK sample: person_household_id cannot map every person to a household row." - ) - person_positions = person_household_position.to_numpy(dtype=np.int64) - person_clone_index = clone_index[person_positions] - person_spi = spi_flag[person_positions] - person_cg = cg_flag[person_positions] - canonical_people = person_clone_index == 0 - if not canonical_people.any(): - raise ValueError("UK sample requires clone_index=0 person rows.") - - canonical_max = max( - int(household_ids[canonical_households].max()), - int(person_ids[canonical_people].max()), - ) - multiplier = 10 ** max(1, len(str(canonical_max))) - clone_reversed = household_ids - clone_index * multiplier - if (clone_reversed <= 0).any(): - raise ValueError( - "UK sample: clone reversal produced non-positive canonical ids; " - "the input's clone lineage does not match the stage fence's " - "arithmetic." - ) - canonical_set = household_ids[canonical_households] - unknown_clones = np.setdiff1d(np.unique(clone_reversed), canonical_set) - if len(unknown_clones): - raise ValueError( - f"UK sample: {len(unknown_clones)} clone household(s) do not " - "reverse to a clone_index=0 canonical row; refusing to sample an " - "input the stage fence would reject." - ) - - raw_households = canonical_households & ~spi_flag & ~cg_flag - pre_cg_households = canonical_households & ~cg_flag - if not raw_households.any(): - raise ValueError("UK sample requires canonical raw FRS households.") - spi_offset = int(household_ids[raw_households].max()) + 1 - cg_offset = int(household_ids[pre_cg_households].max()) + 1 - units = ( - clone_reversed - - spi_flag.astype(np.int64) * spi_offset - - cg_flag.astype(np.int64) * cg_offset - ) - raw_set = frozenset(int(value) for value in household_ids[raw_households]) - if (units <= 0).any() or not set(int(v) for v in np.unique(units)) <= raw_set: - bad = len(set(int(v) for v in np.unique(units)) - raw_set) - raise ValueError( - f"UK sample: {bad} household(s) do not reverse to the raw FRS " - "surface; refusing to sample an input the stage fence would " - "reject." - ) - - def _unit_of_max_household(mask: np.ndarray) -> int: - position = int(np.argmax(np.where(mask, household_ids, -1))) - return int(units[position]) - - def _unit_of_max_person(mask: np.ndarray) -> int: - position = int(np.argmax(np.where(mask, person_ids, -1))) - return int(units[person_positions[position]]) - - raw_people = canonical_people & ~person_spi & ~person_cg - pre_cg_people = canonical_people & ~person_cg - if not raw_people.any(): - raise ValueError("UK sample requires canonical raw FRS person rows.") - forced = tuple( - sorted( + def __init__( + self, + transform, + *, + fraction: float, + seed: int, + ) -> None: + self.transform = transform + self.fraction = fraction + self.seed = seed + self.sampling: dict[str, object] | None = None + + def graph_implementation_dependencies(self) -> tuple[object, ...]: + """The modules the root node hashes for this adapter's sampling.""" + + if self.fraction == 1.0: + return (sys.modules[__name__],) + return (sys.modules[__name__], frame_sampling) + + def _sample(self, assembled): + sampled, self.sampling = sample_uk_spine_root( + assembled, + fraction=self.fraction, + seed=self.seed, + ) + # Graph populations use row positions as their internal alignment + # index. Frame sampling preserves source DataFrame indexes by design, + # so normalize those indexes at this adapter boundary. + tables = { + entity: sampled.table(entity).reset_index(drop=True) + for entity in sampled.entities + } + tables.update( + {name: sampled.link(name).reset_index(drop=True) for name in sampled.links} + ) + return Frame( + tables, + sampled.schema, { - # Clone-multiplier pins (digit count of the canonical max). - _unit_of_max_household(canonical_households), - _unit_of_max_person(canonical_people), - # SPI-offset pins (max canonical raw id, exactly). - _unit_of_max_household(raw_households), - _unit_of_max_person(raw_people), - # CG-offset pins (max canonical pre-CG id, exactly). - _unit_of_max_household(pre_cg_households), - _unit_of_max_person(pre_cg_people), - } + entity: sampled.weights_for(entity) + for entity in sampled.weighted_entities + }, + sampled.strata.reset_index(drop=True), + mass_log=sampled.mass_log, + metadata=sampled.metadata, ) - ) - return units, forced, multiplier - -def _assert_spi_replacement_quota(household: pd.DataFrame) -> None: - """Re-assert the SPI rebuild's per-cell ``#base >= #dead`` on the sample. + def effective_fraction(self) -> float: + """Return the configured input sampling fraction.""" - Source-family sampling preserves the inequality structurally when every - dead row's source lives in its own cell; this check converts that - assumption into a fail-closed runtime receipt, using the same stratum - cells the rebuild's ``_sample_replacement_household_ids`` groups by. - """ - - synthetic = _strict_bool( - household[HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN], - label=HOUSEHOLD_IS_SPI_SYNTHETIC_COLUMN, - ) - dead = household.loc[synthetic] - if dead.empty: - return - strata = list(SPI_REPLACEMENT_STRATA_COLUMNS) - base = household.loc[~synthetic] - dead_counts = dead.groupby(strata, sort=True, dropna=False).size() - base_counts = base.groupby(strata, sort=True, dropna=False).size() - for key, quota in dead_counts.items(): - capacity = int(base_counts.get(key, 0)) - if capacity < int(quota): - raise ValueError( - "UK sample violates the SPI replacement quota for stratum " - f"{key!r}: need {int(quota)} base household(s), found " - f"{capacity}. Source-family sampling preserves this " - "inequality when dead rows share their source's cell; an " - "irregular structure in the input is the likely cause." - ) - - -def sample_uk_national_frame( - frame: Frame, - *, - fraction: float, - seed: int, -) -> tuple[Frame, dict[str, object]]: - """Draw one seeded UK rung sample and renormalize it to full mass. - - Runs on the loaded certified compact **before** provenance binding, so - the certified-candidate fence attests the frame the stages actually - consume. The result satisfies :func:`validate_uk_national_frame`: the - kernel mints the renormalization :class:`MassChangeRecord`, and the - exported ``household_weight`` column is refreshed from the typed vector. - - Returns: - The sampled frame and the shared sampling receipt extended with the - normalization fields and the UK policy block. - """ + return float(self.fraction) - validate_sample_fraction(fraction, label="UK sample") - validate_sample_seed(seed, label="UK sample") - validate_uk_national_frame(frame) + def __call__(self, frame): + return self._sample(self.transform(frame)) - household = frame.table("household") - units, forced, multiplier = uk_source_family_units(frame) - household_ids = _int_column(household["household_id"], label="household_id") - region_values = _str_column(household[_REGION_COLUMN], label=_REGION_COLUMN) - region_by_household = dict( - zip( - household_ids.tolist(), - region_values.tolist(), - strict=True, + def run_with_sources(self, frame, sources): + runner = getattr(self.transform, "run_with_sources", None) + assembled = ( + runner(frame, sources) if callable(runner) else self.transform(frame) ) - ) - # The stratum is the raw canonical household's region: clone-0 quota - # cells vary by family region, so proportionality must hold within each - # canonical region. Channel flags vary within a source family and are - # deliberately not strata — derived rows travel with their source. - strata = np.asarray( - [f"region={region_by_household[int(unit)]}" for unit in units], - dtype=object, - ) - sampled, receipt = sample_frame_households( - frame, - fraction=fraction, - seed=seed, - source_name="UK national", - unit_ids=units, - unit_strata=strata, - forced_unit_ids=forced, - unit_noun="source family", - floor_context="the UK national build", - ) - if sampled is not frame: - full_mass = float(receipt["incoming_household_mass"]) - sampled, factor = normalize_sampled_household_mass( - sampled, target_mass=full_mass, source_name="UK national" - ) - receipt["normalization_factor"] = factor - receipt["normalized_household_mass"] = float( - sampled.weights_for("household").total - ) - _assert_spi_replacement_quota(sampled.table("household")) - validate_uk_national_frame(sampled) - receipt["uk_policy"] = { - "sampling_unit": "source_frs_family", - "strata_columns": [ - f"{_REGION_COLUMN} (of the raw clone_index=0 canonical row)" - ], - "clone_multiplier": multiplier, - "forced_retention": ( - "families carrying the argmax canonical, raw, and pre-CG " - "household and person ids (clone-multiplier and SPI/CG offset " - "pins)" - ), - "spi_replacement_quota_checked": True, - } - return sampled, receipt + return self._sample(assembled) + + def checkpoint_metadata(self) -> dict[str, object]: + hook = getattr(self.transform, "checkpoint_metadata", None) + if not callable(hook): + raise RuntimeError("FRS root transform exposes no checkpoint metadata.") + return dict(hook()) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/nts_bus_travel.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/nts_bus_travel.py index 598d9ef2d2..47ed196940 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/nts_bus_travel.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/nts_bus_travel.py @@ -47,6 +47,7 @@ nts_band_shares, ) from microcosm.build.uk_runtime.frs_spine import read_pinned_tab +from microcosm.build.uk_runtime.geographic_clones import geographic_clone_count from microcosm.build.uk_runtime.household_income import ( UK_HBAI_GROSS_INCOME_VARIABLES, hbai_gross_income, @@ -2016,6 +2017,9 @@ def __call__(self, frame: Frame) -> Frame: donor_support, columns=UK_NTS_SUPPORT_CLIP_COLUMNS, stage=UK_NTS_BUS_TRAVEL_STAGE_NAME, + clone_count=geographic_clone_count( + frame.table("household"), stage=UK_NTS_BUS_TRAVEL_STAGE_NAME + ), ) clipped = clip.clipped # Outputs. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_contract.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_contract.py index fb893beeb3..7f31002e30 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_contract.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_contract.py @@ -347,13 +347,8 @@ def checkpoint_references(sidecar_path: Path | None) -> tuple[list[dict], str | ( "population", "Population pool", - "Checkpoint admission, sampling, normalisation, expansion into K " - "geographic copies and the pooled problem checkpoint.", - ), - ( - "geography", - "Geography", - "Location draws, derived geographies and the geography integrity gate.", + "Checkpoint admission and the pool checkpoint: the bound spine whole, " + "with its geographic clones, areas and sampling.", ), ( "targets", @@ -381,26 +376,8 @@ def checkpoint_references(sidecar_path: Path | None) -> tuple[list[dict], str | _FULL_BUILD_GROUPS = { "population": ( "uk.full.spine_checkpoint", - "uk.full.sample", - "uk.full.normalize", - "uk.full.expand", "uk.full.pool", ), - "geography": ( - "uk.full.identity", - "uk.full.locations", - "uk.full.geography_mapping", - "uk.full.geography_gate", - "uk.full.geography.assign", - "uk.full.geography.derive", - "uk.full.geography.local_authority", - "uk.full.geography.brma", - "uk.full.geography.rfdr", - "uk.full.private_rent", - "uk.full.property.value", - "uk.full.property.band", - "uk.full.geography.gate", - ), "targets": ( "uk.full.target_compilation", "uk.full.target_selection", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/private_rent_localisation.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/private_rent_localisation.py index 85f74904d7..0cfb01ad03 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/private_rent_localisation.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/private_rent_localisation.py @@ -2,22 +2,23 @@ The UK spine carries each private renter's rent from its source household (the FRS report, or the SPI housing shell's draw from FRS donors), which -describes its FRS region, not the district the household is placed in. After -geography, ``uk.full.private_rent`` keeps each private renter's rank among -the FRS renters of its region and bedroom size and reads that quantile off -the list of rents of the BRMA its assigned area sits in -(``uk/brma_rent_quantiles.json``), scaled per FRS region so that, on the -census private-renter mix of the region's BRMAs, its mean rent is the FRS level -(María's rulings of 2026-10-15/16, kept on 2026-10-17): the lists set where -rents sit within a region and their spread within a BRMA, not the level. -Placement follows all households, not renters, so at design weights a -region's mean rent moves by the gap between the two mixes until -calibration's tenure rows restore the renter mix; the receipts report that -level as a check. +describes its FRS region, not the district the household is placed in. The +clone-first spine's private-rent stage +(``spine_private_rent.UKPrivateRentStageTransform``, G7 of microcosm#931) +keeps each private renter's rank among the FRS renters of its region and +bedroom size and reads that quantile off the list of rents of the BRMA its +drawn area sits in (``uk/brma_rent_quantiles.json``), scaled per FRS region +so that, on the census private-renter mix of the region's BRMAs, its mean +rent is the FRS level (María's rulings of 2026-10-15/16, kept on +2026-10-17): the lists set where rents sit within a region and their spread +within a BRMA, not the level. Placement follows all households, not renters, +so at design weights a region's mean rent moves by the gap between the two +mixes until calibration's tenure rows restore the renter mix; the receipts +report that level as a check. Everything here is pure and depends on the standard library, numpy and -pandas only, so the build tool can load this module by path; the graph -kernel supplies the spine keys and the keyed tie-break uniforms. +pandas only, so the build tool can load this module by path; the spine's +private-rent stage supplies the row keys and the keyed tie-break uniforms. """ from __future__ import annotations @@ -36,12 +37,10 @@ "COUNTS_SOURCE", "GRID", "METHOD_VERSION", - "PRIVATE_RENT_LOCALISATION_MODES", "QUANTILES_KIND", "QUANTILES_METHOD", "QUANTILES_SCHEMA_VERSION", "QUANTILES_SOURCE", - "STREAM_EXPERIMENT", "WEEKS_IN_YEAR", "RentQuantiles", "knot_mean", @@ -55,11 +54,7 @@ "rent_uniforms", ] -#: ``brma_list`` re-draws each Great Britain private renter's rent from its -#: assigned BRMA's list of rents; ``none`` keeps the spine's rents. -PRIVATE_RENT_LOCALISATION_MODES = ("none", "brma_list") METHOD_VERSION = "uk-private-rent-localisation-v1" -STREAM_EXPERIMENT = "uk-post-clone-private-rent-v1" #: Graph source names: the committed rent quantiles and the census counts of #: private-rented households by region, BRMA and bedrooms (microcosm#1087). QUANTILES_SOURCE = "uk_brma_rent_quantiles" diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/private_school_attendance.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/private_school_attendance.py index 72483559b8..5a13e1d0ca 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/private_school_attendance.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/private_school_attendance.py @@ -14,21 +14,37 @@ * the calibration's engine run and the release's read different flags once the weights change. -This stage, the last of the spine, materializes the engine's attendance once, -at the calibration year and the spine's design weights, and stores it, as the -root stage stores ``months_since_last_birthday``. The solve and the release -then read the same flags, and no stage after it copies or re-keys a person. +This stage, the last of the spine, stores attendance once, at the calibration +year and the spine's design weights, as the root stage stores +``months_since_last_birthday``. The solve and the release then read the same +flags, and no stage after it copies or re-keys a person. Stored attendance no longer responds to income changes or to reforms of the attendance parameters; a reform to the fee VAT does not move pupils either way in the pinned engine. + +The rank is over the people a simulation holds, so a simulation over part of +the frame ranks that part alone. Once a spine's households carry more than +one geographic clone, its engine calls run clone by clone +(microcosm#931), so the stage restates the formula's last step itself, over +every person in the frame (:func:`restate_private_school_attendance`): from +engine values that each household's own records decide, the engine's own +parameters, and microdf's weighted percentile rank, the rank the formula +calls. Where the engine computes the frame in one simulation (the national +line, and a spine of one geographic clone) the stage stores the engine's +flags once the restatement equals them bit for bit, so every such build +re-checks the restatement on its own data; where it computes the frame clone +by clone the stage stores the restatement. The receipt names the route and +whether the engine's flags were checked, never a count. """ from __future__ import annotations from collections.abc import Mapping from dataclasses import dataclass, field +from typing import Any import numpy as np +import pandas as pd from microcosm.build.source_manifest import SourceStageSpec from microcosm.build.uk_runtime.frs_release import resolve_uk_year_rule @@ -54,6 +70,40 @@ "The terminal family gate requires a valid mass-conserving MassChangeRecord " "carrying exactly this stage-specific reason." ) +#: How the stored flags were reached: the engine's own, once the restatement +#: equalled them bit for bit (one simulation over the frame), or the +#: restatement over the whole frame (the engine computes it clone by clone). +PRIVATE_SCHOOL_ATTENDANCE_ENGINE_ROUTE = "engine_with_parity" +PRIVATE_SCHOOL_ATTENDANCE_POOLED_ROUTE = "pooled_restatement" +#: What policyengine-uk's formula reads (2.174.1, +#: variables/contrib/labour/attends_private_school.py, a source the UK engine +#: declaration pins), each decided by a household's own records, so the +#: engine returns it alike over the frame and clone by clone. +PRIVATE_SCHOOL_ATTENDANCE_PERSON_INPUTS = ( + "income_tax", + "national_insurance", + "age_under_18", + PRIVATE_SCHOOL_ATTENDANCE_DRAW, +) +PRIVATE_SCHOOL_ATTENDANCE_HOUSEHOLD_INPUTS = ( + "household_market_income", + "household_benefits", + "household_weight", +) +#: The engine parameters the formula reads, the only ones the restatement +#: reads: the attendance rate at every fifth percentile, and the population +#: factor. +PRIVATE_SCHOOL_ATTENDANCE_PARAMETERS = ( + "gov.simulation.private_school_vat.private_school_attendance_rate", + "gov.simulation.private_school_vat.private_school_factor", +) +# The formula's own constants: the percentiles its attendance rates are given +# at, the percentiles it looks rates up for, and the total weight of people +# (a household's weight once per member) below which it imputes no attendance +# (pe-uk#1905). The pinned source and the parity check hold these copies to it. +_RATE_PERCENTILES = tuple(range(0, 101, 5)) +_LOOKUP_PERCENTILES = 101 +_IMPUTED_POPULATION_FLOOR = 1e6 @dataclass(frozen=True) @@ -63,6 +113,8 @@ class UKPrivateSchoolAttendanceResult: frame: Frame year: int weighted_pupils: float + rank_route: str + engine_parity_checked: bool def evidence(self) -> dict[str, object]: return { @@ -70,8 +122,15 @@ def evidence(self) -> dict[str, object]: "year_rule": PRIVATE_SCHOOL_ATTENDANCE_YEAR_RULE, "year": self.year, "weights": PRIVATE_SCHOOL_ATTENDANCE_WEIGHTS, - # A national weighted total over every pupil the engine imputes. + # A national weighted total over every pupil the stage stores. "weighted_pupils": self.weighted_pupils, + # The route's name, and whether the engine's own flags were checked + # against the restatement: True over one simulation; False where + # the engine computes the frame clone by clone and no simulation + # holds the pool's flags. A check that fails raises, so the field + # never records a mismatch. + "rank_route": self.rank_route, + "engine_parity_checked": self.engine_parity_checked, } @@ -107,7 +166,15 @@ def checkpoint_metadata(self) -> dict[str, object]: def store_private_school_attendance( frame: Frame, *, engine: object, calibration_year: int | None = None ) -> UKPrivateSchoolAttendanceResult: - """Store the engine's attendance at the calibration year and these weights.""" + """Store policyengine-uk's attendance at the calibration year and these weights. + + ``engine`` computes the frame's values (``materialize``) and gives its own + parameters (``parameters(period)``). Where it computes the frame in one + simulation, the stage stores the engine's flags once the restatement over + the frame equals them bit for bit, and raises otherwise; where it computes + the frame in more (``simulation_batches``: one per clone; an engine + without the method runs one), the stage stores the restatement. + """ validate_uk_national_frame(frame) assert_rules_engine_country(engine, "uk") @@ -128,21 +195,42 @@ def store_private_school_attendance( if calibration_year is not None else resolve_uk_year_rule(PRIVATE_SCHOOL_ATTENDANCE_YEAR_RULE) ) + parameters = _engine_parameters(engine, year) + batched = _simulation_batches(engine, frame) > 1 + inputs = ( + *PRIVATE_SCHOOL_ATTENDANCE_PERSON_INPUTS, + *PRIVATE_SCHOOL_ATTENDANCE_HOUSEHOLD_INPUTS, + ) + # A call that runs clone by clone refuses the engine's own attendance, which + # would rank each clone alone; one simulation computes it beside the inputs. + requested = inputs if batched else (PRIVATE_SCHOOL_ATTENDANCE_OUTPUT, *inputs) materialized = engine.materialize( - frame, [PRIVATE_SCHOOL_ATTENDANCE_OUTPUT], year, dataset_period=period + frame, list(requested), year, dataset_period=period ) - if PRIVATE_SCHOOL_ATTENDANCE_OUTPUT not in materialized: - raise ValueError( - "Private-school attendance engine output missing: " - f"{[PRIVATE_SCHOOL_ATTENDANCE_OUTPUT]}." - ) - values = np.asarray(materialized[PRIVATE_SCHOOL_ATTENDANCE_OUTPUT]) - if values.shape != (len(person),): - raise ValueError( - f"{PRIVATE_SCHOOL_ATTENDANCE_OUTPUT} has {values.shape[0]} values for " - f"{len(person)} people." - ) - attends = values.astype(bool) + missing = [name for name in requested if name not in materialized] + if missing: + raise ValueError(f"Private-school attendance engine output missing: {missing}.") + restated = restate_private_school_attendance(frame, materialized, parameters) + if batched: + attends = restated + rank_route = PRIVATE_SCHOOL_ATTENDANCE_POOLED_ROUTE + else: + values = np.asarray(materialized[PRIVATE_SCHOOL_ATTENDANCE_OUTPUT]) + if values.shape != (len(person),): + raise ValueError( + f"{PRIVATE_SCHOOL_ATTENDANCE_OUTPUT} has {values.shape[0]} values " + f"for {len(person)} people." + ) + attends = values.astype(bool) + if not np.array_equal(attends, restated): + raise ValueError( + "policyengine-uk's attends_private_school, restated from the " + "engine's own values and parameters, differs from the engine's " + "flags over the same frame. The restatement no longer follows " + "the engine's formula, so a spine whose engine calls run clone by " + "clone would store other flags than one simulation over its pool." + ) + rank_route = PRIVATE_SCHOOL_ATTENDANCE_ENGINE_ROUTE person[PRIVATE_SCHOOL_ATTENDANCE_OUTPUT] = attends household_weights = frame.weights_for("household") weight_by_household = dict( @@ -176,7 +264,154 @@ def store_private_school_attendance( frame=result, year=int(year), weighted_pupils=float(person_weights[attends].sum()), + rank_route=rank_route, + engine_parity_checked=not batched, + ) + + +def restate_private_school_attendance( + frame: Frame, values: Mapping[str, Any], parameters: Any +) -> np.ndarray: + """policyengine-uk's ``attends_private_school`` over every person in ``frame``. + + ``values`` holds the engine's own arrays at one period for + :data:`PRIVATE_SCHOOL_ATTENDANCE_PERSON_INPUTS` and + :data:`PRIVATE_SCHOOL_ATTENDANCE_HOUSEHOLD_INPUTS`: people in the frame's + order and households by ascending id, the frame's own order, as the + adapter returns them whether its call ran in one simulation or clone by + clone. ``parameters`` is the engine's parameter tree at that period, read + at :data:`PRIVATE_SCHOOL_ATTENDANCE_PARAMETERS` only. The steps, their + order and their types are the formula's (policyengine-uk 2.174.1, + variables/contrib/labour/attends_private_school.py), not its code: + + 1. each household's members' income tax plus National Insurance, summed + over the members in the people's order (policyengine-core's group sum, + a ``bincount``) and given to each member; + 2. net income: the household's market income plus its benefits, at each + member, less those taxes; + 3. each person carries the household's weight, so a household counts once + per member; while those weights sum below one million, nobody attends; + 4. each person's percentile of net income among the people of positive + weight, by microdf's weighted percentile rank, the rank the formula + calls; people of no weight keep percentile 0; + 5. the rate at that percentile, interpolated between the engine's rates at + every fifth percentile, times the engine's population factor, for + under-18s only; + 6. a person attends when the stored draw is below that rate. + + The adapter builds every simulation from data, so the formula's branch + for a situation, which imputes nobody, never applies. + """ + + person = frame.table("person") + household_ids = frame.table("household")["household_id"].to_numpy() + member_of = pd.Index(household_ids).get_indexer( + person["person_household_id"].to_numpy() + ) + if (member_of < 0).any(): + raise ValueError("A person's household is not in the frame's household table.") + arrays: dict[str, np.ndarray] = {} + for names, rows, entity in ( + (PRIVATE_SCHOOL_ATTENDANCE_PERSON_INPUTS, len(person), "person"), + (PRIVATE_SCHOOL_ATTENDANCE_HOUSEHOLD_INPUTS, len(household_ids), "household"), + ): + for name in names: + if name not in values: + raise ValueError(f"The restatement needs the engine's {name!r}.") + array = np.asarray(values[name]) + if array.shape != (rows,): + raise ValueError( + f"The engine's {name!r} is not one value per {entity} of the frame." + ) + arrays[name] = array + taxes = np.bincount( + member_of, + weights=arrays["income_tax"] + arrays["national_insurance"], + minlength=len(household_ids), + )[member_of] + net_income = ( + arrays["household_market_income"][member_of] + + arrays["household_benefits"][member_of] + - taxes + ) + weights = arrays["household_weight"][member_of] + if weights.sum() < _IMPUTED_POPULATION_FLOOR: + return np.zeros(len(person), dtype=bool) + percentile = _net_income_percentiles(net_income, weights) + rate_path, factor_path = PRIVATE_SCHOOL_ATTENDANCE_PARAMETERS + rate = _parameter(parameters, rate_path) + rates = np.array([float(rate[str(p)]) for p in _RATE_PERCENTILES]) + rate_by_percentile = np.interp( + np.arange(_LOOKUP_PERCENTILES), list(_RATE_PERCENTILES), rates + ) + probability = ( + rate_by_percentile[percentile] + * _parameter(parameters, factor_path) + * arrays["age_under_18"] ) + return np.asarray(arrays[PRIVATE_SCHOOL_ATTENDANCE_DRAW] < probability, dtype=bool) + + +def _parameter(parameters: Any, path: str) -> Any: + """The node or value at a dotted ``path`` in a parameter tree.""" + + node = parameters + for name in path.split("."): + node = getattr(node, name) + return node + + +def _net_income_percentiles(net_income: np.ndarray, weights: np.ndarray) -> np.ndarray: + """Each person's percentile of net income among the people of positive + weight, in the frame's order: microdf's weighted percentile rank, the + rank policyengine-uk's formula calls, clipped to 0-100. People of no + weight keep 0, as the formula leaves them. microdf comes with + policyengine-uk, so it is imported only here.""" + + try: + from microdf import MicroSeries + except ImportError as exc: + raise ImportError( + "Restating private-school attendance ranks with microdf, which " + "policyengine-uk installs: install microcosm-build's 'uk' extra." + ) from exc + income = MicroSeries(net_income, weights=weights) + percentile = np.zeros(len(net_income), dtype=np.int64) + weighted = weights > 0 + percentile[weighted] = np.asarray( + income[weighted].percentile_rank().clip(0, 100) + ).astype(np.int64) + return percentile + + +def _engine_parameters(engine: object, year: int | str) -> Any: + """The engine's own parameter tree at ``year``, which the restatement + reads its rates and factor from.""" + + parameters = getattr(engine, "parameters", None) + if not callable(parameters): + raise TypeError( + "Private-school attendance restates policyengine-uk's formula from " + "the engine's own parameters, so its engine must give them " + "(parameters(period))." + ) + return parameters(year) + + +def _simulation_batches(engine: object, frame: Frame) -> int: + """How many simulations the engine computes ``frame`` in: one, or one per + clone (the UK adapter's ``simulation_batches``). An engine without the + method computes it in one.""" + + batches = getattr(engine, "simulation_batches", None) + if batches is None: + return 1 + count = batches(frame) + if isinstance(count, bool) or not isinstance(count, int | np.integer) or count < 1: + raise ValueError( + "The engine's simulation_batches must be a positive whole number." + ) + return int(count) def _assert_stage_parameters(stage: SourceStageSpec) -> None: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/property_components.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/property_components.py index 5476eeca2b..ce9f33dc20 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/property_components.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/property_components.py @@ -61,7 +61,7 @@ from __future__ import annotations -from collections.abc import Mapping, Sequence +from collections.abc import Collection, Mapping, Sequence from dataclasses import dataclass, field from pathlib import Path from typing import Any @@ -71,7 +71,11 @@ from microcosm.build.gates import FitWeightRecord from microcosm.build.source_manifest import SourceStageSpec -from microcosm.build.stochastic_assignment import stable_identity_uniforms +from microcosm.build.stochastic_assignment import ( + keyed_tie_order, + stable_identity_uniforms, +) +from microcosm.build.uk_runtime.geographic_clones import geographic_clone_count from microcosm.build.uk_runtime.hmrc_property_rental import ( HMRC_PROPERTY_RENTAL_RECEIPTS_BAND_LOWER_BOUNDS, HMRC_PROPERTY_RENTAL_RECORD_SETS, @@ -259,14 +263,32 @@ def from_parameters(cls, parameters: Mapping[str, Any]) -> FinanceCostModelSpec: @dataclass(frozen=True) class ReceiptsAllocationSpec: - """The ``allocate_property_rental_income`` operation.""" + """The ``allocate_property_rental_income`` operation. + + ``tie_salt`` keys the walk's tie-break: landlords tied on profit take the + order of a uniform keyed on their person_id under ``walk_seed`` and this + salt, then of their ids (microcosm#931). + """ band_lower_bounds: tuple[int, ...] walk_seed: int + tie_salt: str maximum_top_band_multiplier: float @classmethod - def from_parameters(cls, parameters: Mapping[str, Any]) -> ReceiptsAllocationSpec: + def from_parameters( + cls, + parameters: Mapping[str, Any], + *, + salts_in_use: Collection[str] = (), + ) -> ReceiptsAllocationSpec: + """The declared operation, refused where it drifts from the stage's rules. + + ``salts_in_use`` names the salts the stage's other draws key on the + same person ids (the stage passes its finance-cost draws'): a tie + salt equal to one of them would order tied landlords by that draw. + """ + bounds = tuple(int(v) for v in parameters["band_lower_bounds"]) if bounds != HMRC_PROPERTY_RENTAL_RECEIPTS_BAND_LOWER_BOUNDS: raise PropertyComponentsError( @@ -286,9 +308,20 @@ def from_parameters(cls, parameters: Mapping[str, Any]) -> ReceiptsAllocationSpe raise PropertyComponentsError( "maximum_top_band_multiplier must be finite and above 1." ) + tie_salt = parameters.get("tie_salt") + if not isinstance(tie_salt, str) or not tie_salt: + raise PropertyComponentsError( + "allocate_property_rental_income must declare a non-empty tie_salt." + ) + if tie_salt in set(salts_in_use): + raise PropertyComponentsError( + "the receipts walk's tie_salt must differ from the salts of the " + "stage's other draws on the same person ids." + ) return cls( band_lower_bounds=bounds, walk_seed=int(parameters["walk_seed"]), + tie_salt=tie_salt, maximum_top_band_multiplier=multiplier, ) @@ -671,7 +704,9 @@ def allocate_property_rental_income( ) -> ReceiptsAllocation: """Receipts for every person with ``profit`` above zero (zero elsewhere). - Ranked by profit (person id breaks ties), walked top band first into + Ranked by profit, landlords tied on profit in the order of a uniform + keyed on their person id under the walk seed and the tie salt, then of + their ids (``keyed_tie_order``), walked top band first into Table 13's bands, each filled to its share of the individuals-basis landlords times the landlords' weight here, linear within a bounded band from its upper bound down, never below profit; the open top band takes @@ -691,7 +726,11 @@ def allocate_property_rental_income( top_down = list(reversed(bands)) if not len(landlord): raise PropertyComponentsError("no landlord with profit above zero to allocate.") - order = landlord[np.lexsort((ids[landlord], -profit[landlord]))] + order = landlord[ + keyed_tie_order( + -profit[landlord], ids[landlord], seed=spec.walk_seed, salt=spec.tie_salt + ) + ] ranked_weights = weights[order] landlord_weight = float(ranked_weights.sum()) published_landlords = float(sum(band.landlords for band in bands)) @@ -822,7 +861,11 @@ def mean_at(k: float) -> float: on = profit > 0.0 receipt = { "population": "persons with profit from property other than sub-letting above zero", - "order": "profit descending, person_id ascending", + "order": ( + "profit descending, landlords tied on profit in ascending order of a " + "uniform keyed on person_id under walk_seed and tie_salt, then " + "person_id ascending" + ), "walk": {"seed": spec.walk_seed, "offset": offset}, "band_basis": ( "PRIS 2024-25 Table 13 (all tax entities) x Table 1 individuals' " @@ -873,9 +916,13 @@ class UKPropertyComponentsResult: channels: Mapping[str, Any] residuals: Mapping[str, Any] coherence: Mapping[str, Any] + #: The frame's geographic clone count ``K`` (microcosm#931). The receipt + #: gives it only when it is more than one, and the stage's check then + #: allows ``K`` times its declared row maxima. + clone_count: int = 1 def evidence(self) -> dict[str, object]: - return { + evidence: dict[str, object] = { "stage": UK_PROPERTY_COMPONENTS_STAGE_NAME, "facts": dict(self.facts), "donor": dict(self.donor), @@ -885,6 +932,9 @@ def evidence(self) -> dict[str, object]: "residuals": dict(self.residuals), "coherence": dict(self.coherence), } + if self.clone_count > 1: + evidence["clone_count"] = self.clone_count + return evidence @dataclass @@ -928,7 +978,8 @@ def __call__(self, frame: Frame) -> Frame: _operation(self.stage, IMPUTE_FRS_PROPERTY_FINANCE_COSTS_KIND) ) receipts_spec = ReceiptsAllocationSpec.from_parameters( - _operation(self.stage, ALLOCATE_PROPERTY_RENTAL_INCOME_KIND) + _operation(self.stage, ALLOCATE_PROPERTY_RENTAL_INCOME_KIND), + salts_in_use=(finance_spec.incidence_salt, finance_spec.amount_salt), ) verify = _operation(self.stage, VERIFY_VENDORED_FACT_RESOURCE_KIND) if ( @@ -951,6 +1002,9 @@ def __call__(self, frame: Frame) -> Frame: person = frame.table("person").copy() household = frame.table("household") + clone_count = geographic_clone_count( + household, stage=UK_PROPERTY_COMPONENTS_STAGE_NAME + ) missing = sorted(PROPERTY_COMPONENTS_PERSON_INPUTS - set(person.columns)) if missing: raise PropertyComponentsError(f"the spine's person table lacks {missing}.") @@ -1124,6 +1178,7 @@ def __call__(self, frame: Frame) -> Frame: channels=channels, residuals=residuals, coherence=coherence, + clone_count=clone_count, ) return result diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/property_localisation.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/property_localisation.py index 17b61ca850..a4d9d60f52 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/property_localisation.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/property_localisation.py @@ -63,8 +63,8 @@ "load_scotland_band_mix", "load_value_knots", "localise_bands", + "localise_main_residence_values", "localise_values", - "main_residence_rank_basis", "owner_uniforms", "spine_household_keys", ] @@ -84,8 +84,10 @@ #: Owner households at or above this within-region rank are the high-value #: support stratum (microcosm#1240 fix 1, ruled top 5%). SUPPORT_STRATUM_LOWER = 0.95 -#: The value the spine's regional home values replace (microcosm#1112), kept -#: so this node ranks owners on what it ranked them on before that stage. +#: The WAS chain's segment-2 draw of the main residence, which its area home +#: values keep beside the value they read off the owner's constituency and +#: the regional property uprating divides by (``area_home_values``, +#: microcosm#931). The release boundary drops it. RANK_BASIS_COLUMN = "main_residence_value_rank_basis" COUNCIL_TAX_BANDS: tuple[str, ...] = ("A", "B", "C", "D", "E", "F", "G", "H", "I") KNOTS_KIND = "uk_high_value_home_value_knots" @@ -266,26 +268,6 @@ def spine_household_keys( return np.asarray(keys, dtype=object) -def main_residence_rank_basis(household: pd.DataFrame, *, declared: bool) -> np.ndarray: - """The value owners are ranked on: the one the regional stage replaced. - - ``declared`` says whether the regional home values stage stored - ``RANK_BASIS_COLUMN`` (the dense graph then declares it in the value - node's slice). A declared column must arrive and an undeclared one must - not: ranking on the regional values instead would move every owner's - draw, so either mismatch refuses rather than falling back. - """ - - if declared != (RANK_BASIS_COLUMN in household): - raise ValueError( - f"The value node's rank basis {RANK_BASIS_COLUMN!r} is " - f"{'declared but missing' if declared else 'present but undeclared'}; " - "owners must rank on the value the regional stage replaced." - ) - column = RANK_BASIS_COLUMN if declared else "main_residence_value" - return household[column].to_numpy(dtype=float) - - def owner_mask(tenure_type: np.ndarray) -> np.ndarray: categories = np.array( [ @@ -431,36 +413,67 @@ def _england_wales(codes: np.ndarray) -> np.ndarray: return np.char.startswith(codes, "E") | np.char.startswith(codes, "W") -def localise_values( +def localise_main_residence_values( *, u: np.ndarray, constituency_code: np.ndarray, tenure_type: np.ndarray, value: np.ndarray, - property_wealth: np.ndarray, knots: ValueKnots, -) -> tuple[np.ndarray, np.ndarray, dict[str, object]]: +) -> tuple[np.ndarray, np.ndarray]: """Read owner rows' values off their constituency's knots at ``u``. - Returns the new values, the new property wealth (the old total moved by - the value's change, so the wealth identity holds) and a receipt. Rows - that are not owner-occupied, carry no value or sit outside England and - Wales are untouched; a constituency without knots fails closed. + The value half of :func:`localise_values`, which the spine's WAS chain + reads its home values through (``area_home_values``, microcosm#931). + Returns every row's value and the rows it rewrote: the owner-occupied rows + with a positive value whose constituency is in England or Wales. Every + other row keeps its value bit for bit. A rewritten row's constituency + without knots fails closed, by a text that names no constituency: a row's + constituency is its household's place. """ u = np.asarray(u, dtype=float) value = np.asarray(value, dtype=float) - wealth = np.asarray(property_wealth, dtype=float) codes = np.asarray(constituency_code).astype(str) target = owner_mask(tenure_type) & (value > 0) & _england_wales(codes) new_value = value.copy() - missing = sorted(set(codes[target]) - set(knots.quantiles)) - if missing: - raise ValueError(f"No value knots for constituency {missing[:5]}.") + if not set(codes[target]) <= set(knots.quantiles): + raise ValueError( + "No value knots for a constituency an England or Wales owner row is " + "assigned to: the knots resource carries every constituency the " + "area draw places owners in." + ) for code in sorted(set(codes[target])): rows = np.flatnonzero(target & (codes == code)) new_value[rows] = np.interp(u[rows], knots.grid, knots.quantiles[code]) if np.any(~np.isfinite(new_value)) or np.any(new_value < 0): raise ValueError("Localised home values must be finite and non-negative.") + return new_value, target + + +def localise_values( + *, + u: np.ndarray, + constituency_code: np.ndarray, + tenure_type: np.ndarray, + value: np.ndarray, + property_wealth: np.ndarray, + knots: ValueKnots, +) -> tuple[np.ndarray, np.ndarray, dict[str, object]]: + """Read owner rows' values off their constituency's knots at ``u``. + + Returns the new values (:func:`localise_main_residence_values`'), the new + property wealth (the old total moved by the value's change, so the wealth + identity holds) and a receipt. Rows that are not owner-occupied, carry no + value or sit outside England and Wales are untouched; a constituency + without knots fails closed. + """ + u = np.asarray(u, dtype=float) + value = np.asarray(value, dtype=float) + wealth = np.asarray(property_wealth, dtype=float) + codes = np.asarray(constituency_code).astype(str) + new_value, target = localise_main_residence_values( + u=u, constituency_code=codes, tenure_type=tenure_type, value=value, knots=knots + ) delta = new_value - value new_wealth = wealth + delta if np.any(~np.isfinite(new_wealth)): diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/receipt_counts.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/receipt_counts.py index 32cb1225fa..9d1947989a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/receipt_counts.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/receipt_counts.py @@ -84,10 +84,13 @@ def record_source_households( (``microcosm.build.source_lineage.record_sources``) on the UK's copy lineage. A frame without the column is refused, and so is a null source: it drops out of a distinct count, so a cell would read fewer sources than - it rests on. A stage that runs before the SPI support channel, the first - copying step, says so with ``before_copies``: no row is a copy yet, so - each household is its own source and its id serves, unless the frame - already carries copy lineage. + it rests on. A stage that can run before the first copying step says so + with ``before_copies``: on a frame without the column and without copy + lineage, no row is a copy yet, so each household is its own source and its + id serves. On the clone-first spine the first copying step is + ``geographic_expand``, which writes the column, so a stage after it that + says so, ``frs_take_up`` for one, reads it and counts a household's + geographic clones once (microcosm#931). """ if not isinstance(tables, Mapping): @@ -110,8 +113,9 @@ def own_source_households(person: pd.DataFrame) -> np.ndarray: """Each person's own household as its true source: the rule before copies. For a caller that holds a person table without the frame's lineage, as - every household is its own source before the SPI support channel copies - any; a frame with copies counts through :func:`record_source_households`. + every household is its own source before the first copying step, + ``geographic_expand`` on the clone-first spine (microcosm#931); a frame + with clones or copies counts through :func:`record_source_households`. """ return person[UK_NATIONAL_SCHEMA.membership_column("household")].to_numpy() diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/regional_home_values.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/regional_home_values.py deleted file mode 100644 index c1916410a5..0000000000 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/regional_home_values.py +++ /dev/null @@ -1,324 +0,0 @@ -"""Home values from the region's Land Registry distribution, on the spine -(microcosm#1112, microcosm#1095). - -The dense line reads each England-and-Wales owner clone's home value off its -assigned constituency's uprated latest-sale distribution once it has an area -(``uk.full.property.value``, microcosm#1240). The national line solves the -bound spine, which has no area, so its ``main_residence_value`` stayed the -WAS-imputed value after the regional uprating, not a Land Registry value. - -This stage, after the high-value home support copies, gives every -England-and-Wales owner household with a value the one its region's -distribution holds at the household's weighted mid-rank among the owner -households of its FRS region, at the spine's weights, with the same ranking -the dense node uses (``property_localisation.owner_uniforms``: ties broken by -an identity-keyed draw, support copies spread over the top stratum by their -own draws). The region knots are the dense node's resource's region level -(``uk/high_value_home_value_knots.json``), from the same owner-weighted -dwelling table as the seats. ``property_wealth`` moves by the value's change. -Scotland, Northern Ireland, renters and owners without a value keep theirs, -and the council-tax band is untouched (the national band is the FRS's). - -The value it replaces is kept in ``main_residence_value_rank_basis``, which the -dense node ranks on, so the dense line's values, wealth and bands do not -change; the release boundary drops the column on both lines. -""" - -from __future__ import annotations - -from collections.abc import Mapping -from dataclasses import dataclass, field -from importlib.resources import files - -import numpy as np - -from microcosm.build.source_manifest import SourceStageSpec -from microcosm.build.uk_runtime.cgt_structure import _assert_closed_world_operations -from microcosm.build.uk_runtime.household_lineage import HIGH_VALUE_SUPPORT_COPY_COLUMN -from microcosm.build.uk_runtime.national_frame import ( - uk_household_weight_kind, - uk_national_frame, - uk_time_period, - validate_uk_national_frame, -) -from microcosm.build.uk_runtime.property_localisation import ( - RANK_BASIS_COLUMN, - SUPPORT_STRATUM_LOWER, - ValueKnots, - load_value_knots, - main_residence_rank_basis, - owner_mask, - owner_uniforms, -) -from microcosm.build.uk_runtime.receipt_counts import ( - distinct_count, - rounded_count, - source_household_ids, - support_band, -) -from microcosm.build.uk_runtime.rowwise_geography import FRS_REGION_TO_REGION_CODE -from microcosm.frame import Frame, MassChangeRecord - -__all__ = [ - "FRS_REGION_TO_KNOTS_REGION", - "RANK_BASIS_COLUMN", - "REGIONAL_HOME_VALUES_OPERATION_KIND", - "REGIONAL_HOME_VALUES_STAGE_NAME", - "REGIONAL_HOME_VALUES_STREAM", - "UKRegionalHomeValuesResult", - "UKRegionalHomeValuesStageTransform", - "assign_regional_home_values", - "main_residence_rank_basis", - "regional_home_values_operation_parameters", -] - -REGIONAL_HOME_VALUES_STAGE_NAME = "regional_home_values" -REGIONAL_HOME_VALUES_OPERATION_KIND = "assign_regional_home_values" -KNOTS_RESOURCE = "high_value_home_value_knots.json" -#: The stage's own keyed stream, apart from the dense node's. -REGIONAL_HOME_VALUES_STREAM = ( - "sha256-u53-v1", - "uk-spine-regional-home-values-v1", - 0, - 0, -) -#: The knots' region codes for the FRS regions they cover. Wales is the -#: country code the knots use, not the placeholder region code the area -#: assignment keeps for it (``W99999999``). -FRS_REGION_TO_KNOTS_REGION: Mapping[str, str] = { - **{ - region: code - for region, code in FRS_REGION_TO_REGION_CODE.items() - if code.startswith("E12") - }, - "WALES": "W92000004", -} -REGIONAL_HOME_VALUES_MASS_CHANGE_REASON = ( - "Regional home values assigned on the source spine: household weights pass " - "through unchanged and total household mass is conserved." -) -_MASS_GATE_COUPLING = ( - "The terminal family gate requires a valid mass-conserving MassChangeRecord " - "carrying exactly this stage-specific reason." -) - - -def load_region_knots(knots: bytes | None = None) -> tuple[ValueKnots, dict]: - """The packaged knots and their region level, checked like the seats'.""" - - import json - - payload = ( - knots - if knots is not None - else files("microcosm.build.uk").joinpath(KNOTS_RESOURCE).read_bytes() - ) - parsed = load_value_knots(payload) - data = json.loads(payload.decode("utf-8")) - regions = (data.get("levels") or {}).get("region") or {} - quantiles: dict[str, np.ndarray] = {} - for code in sorted(set(FRS_REGION_TO_KNOTS_REGION.values())): - if code not in regions: - raise ValueError(f"The value knots carry no region {code}.") - values = np.asarray(regions[code]["quantiles"], dtype=float) - if values.shape != parsed.grid.shape: - raise ValueError(f"Region {code} knots do not match the grid.") - if ( - np.any(~np.isfinite(values)) - or values[0] <= 0 - or np.any(values > parsed.cap) - or np.any(np.diff(values) <= 0) - ): - raise ValueError( - f"Region {code} knots must be positive, below the cap and " - "strictly increasing, so a positive value stays positive and " - "the dense node's ranks do not tie anew." - ) - quantiles[code] = values - return parsed, quantiles - - -@dataclass(frozen=True) -class UKRegionalHomeValuesResult: - """The frame with the regional values, and the stage's aggregate receipt.""" - - frame: Frame - receipt: dict[str, object] - - def evidence(self) -> dict[str, object]: - return {"stage": REGIONAL_HOME_VALUES_STAGE_NAME, **self.receipt} - - -def assign_regional_home_values( - frame: Frame, *, knots: bytes | None = None -) -> UKRegionalHomeValuesResult: - """Read England-and-Wales owners' values off their region's knots.""" - - validate_uk_national_frame(frame) - household = frame.table("household").copy() - if RANK_BASIS_COLUMN in household.columns: - raise ValueError( - f"{RANK_BASIS_COLUMN!r} is already stored: this stage is its only writer." - ) - for column in ("region", "tenure_type", "main_residence_value", "property_wealth"): - if column not in household.columns: - raise ValueError(f"Regional home values need {column!r}.") - # The graph node declares the lineage, since the executor hands a stage - # only its declared cells. - source_ids = source_household_ids(household, stage="The regional home values stage") - parsed, region_knots = load_region_knots(knots) - weights = np.asarray(frame.weights_for("household").values, dtype=float) - region = household["region"].astype(str).to_numpy() - tenure = household["tenure_type"].to_numpy() - value = household["main_residence_value"].to_numpy(dtype=float) - wealth = household["property_wealth"].to_numpy(dtype=float) - support = ( - household[HIGH_VALUE_SUPPORT_COPY_COLUMN].to_numpy().astype(bool) - if HIGH_VALUE_SUPPORT_COPY_COLUMN in household - else None - ) - u, uniform_receipt = owner_uniforms( - spine_key=household["household_id"].astype(str).to_numpy(dtype=object), - region=region, - tenure_type=tenure, - value=value, - weight=weights, - stream=REGIONAL_HOME_VALUES_STREAM, - support_copy=support, - support_stratum_lower=SUPPORT_STRATUM_LOWER, - ) - knots_region = np.asarray( - [FRS_REGION_TO_KNOTS_REGION.get(code, "") for code in region], dtype=object - ) - target = owner_mask(tenure) & (value > 0) & (knots_region != "") - new_value = value.copy() - rewritten_by_region: dict[str, int | str] = {} - for code in sorted(set(knots_region[target])): - rows = np.flatnonzero(target & (knots_region == code)) - new_value[rows] = np.interp(u[rows], parsed.grid, region_knots[code]) - rewritten_by_region[str(code)] = rounded_count(int(len(rows))) - if np.any(~np.isfinite(new_value)) or np.any(new_value[target] <= 0): - raise ValueError("Regional home values must be finite and positive.") - new_wealth = np.maximum(wealth + (new_value - value), 0.0) - household[RANK_BASIS_COLUMN] = value - household["main_residence_value"] = new_value - household["property_wealth"] = new_wealth - household_weights = frame.weights_for("household") - mass = MassChangeRecord( - entity="household", - old_total=household_weights.total, - new_total=household_weights.total, - declared_factor=1.0, - reason=REGIONAL_HOME_VALUES_MASS_CHANGE_REASON, - ) - result = uk_national_frame( - person=frame.table("person").copy(), - benunit=frame.table("benunit").copy(), - household=household, - time_period=uk_time_period(frame), - weight_kind=uk_household_weight_kind(frame), - household_weights=household_weights.values, - mass_log=(*frame.mass_log, mass), - ) - validate_uk_national_frame(result) - at_cap = distinct_count(source_ids, target & (new_value >= parsed.cap), weights) - receipt = { - "knots_basis_year": parsed.basis_year, - "knots_inputs_sha256": parsed.inputs_sha256, - "rows": rounded_count(int(len(household))), - "rows_rewritten_by_region": rewritten_by_region, - "sources_at_cap": support_band(at_cap), - "weighted_value_before": float(np.dot(value[target], weights[target])), - "weighted_value_after": float(np.dot(new_value[target], weights[target])), - "ranking": { - key: uniform_receipt[key] for key in ("rank_basis", "support_stratum_lower") - }, - } - return UKRegionalHomeValuesResult(frame=result, receipt=receipt) - - -def regional_home_values_operation_parameters() -> tuple[ - tuple[str, dict[str, object]], ... -]: - """The reviewed operations the manifest must carry verbatim.""" - - return ( - ( - REGIONAL_HOME_VALUES_OPERATION_KIND, - { - "knots": KNOTS_RESOURCE, - "knots_level": "region", - "region_map": dict(sorted(FRS_REGION_TO_KNOTS_REGION.items())), - "rows": ( - "owner households (tenure owned outright or with a mortgage) " - "with main_residence_value > 0 in the mapped regions" - ), - "rank": ( - "weighted mid-rank of main_residence_value among the owner " - "households of the FRS region at the spine's weights, ties " - "broken by a keyed draw; high-value support copies take " - "support_stratum_lower + (1 - support_stratum_lower) * draw" - ), - "support_stratum_lower": SUPPORT_STRATUM_LOWER, - "stream": list(REGIONAL_HOME_VALUES_STREAM), - "value": "np.interp(rank, grid, region knots)", - "property_wealth": "moves by the value's change, floored at 0", - "rank_basis_column": RANK_BASIS_COLUMN, - }, - ), - ( - "record_mass_conservation_receipt", - { - "entity": "household", - "reason": REGIONAL_HOME_VALUES_MASS_CHANGE_REASON, - "declared_factor": 1.0, - "gate_coupling": _MASS_GATE_COUPLING, - }, - ), - ) - - -def _assert_stage_parameters(stage: SourceStageSpec) -> None: - _assert_closed_world_operations(stage, regional_home_values_operation_parameters()) - if stage.stage != REGIONAL_HOME_VALUES_STAGE_NAME: - raise ValueError( - f"Expected stage {REGIONAL_HOME_VALUES_STAGE_NAME!r}, got {stage.stage!r}." - ) - if tuple(stage.outputs) != (RANK_BASIS_COLUMN,) or tuple(stage.rewrites) != ( - "main_residence_value", - "property_wealth", - ): - raise ValueError( - "The regional home values stage writes the rank basis and rewrites " - "main_residence_value and property_wealth only." - ) - if stage.grain != "household": - raise ValueError( - f"The regional home values stage is household grain, got {stage.grain!r}." - ) - - -@dataclass(frozen=True) -class UKRegionalHomeValuesStageTransform: - """Whole-stage callable for the regional home values.""" - - stage: SourceStageSpec - last_result: UKRegionalHomeValuesResult | None = field(default=None, init=False) - - def __post_init__(self) -> None: - _assert_stage_parameters(self.stage) - - def __call__(self, frame: Frame) -> Frame: - _assert_stage_parameters(self.stage) - result = assign_regional_home_values(frame) - object.__setattr__(self, "last_result", result) - return result.frame - - @staticmethod - def output_columns() -> tuple[str, ...]: - return (RANK_BASIS_COLUMN,) - - def checkpoint_metadata(self) -> dict[str, object]: - if self.last_result is None: - raise RuntimeError("checkpoint metadata requires a completed stage run.") - return {"evidence": self.last_result.evidence()} diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/regional_uprating.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/regional_uprating.py index b52c752be5..6d787616f5 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/regional_uprating.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/regional_uprating.py @@ -19,6 +19,7 @@ uk_time_period, validate_uk_national_frame, ) +from microcosm.build.uk_runtime.property_localisation import RANK_BASIS_COLUMN from microcosm.build.uk_runtime.spi_support import ( BASE_FRS_SUPPORT_CHANNEL, support_channel_column, @@ -44,6 +45,24 @@ #: ``record_mass_conservation_receipt`` operation repeats it): the terminal #: family gate requires exactly this reason on a valid mass-conserving record. UK_REGIONAL_PROPERTY_UPRATING_MASS_CONSERVATION_REASON = "Regional property uprating on the source spine: household weights pass through unchanged and total household mass is conserved." +#: The packaged regional reference the factor reads: each region's public +#: mean house price. +UK_REGIONAL_LAND_VALUES_RESOURCE = "regional_land_values.json" +#: The stage's operation, whose optional ``denominator`` names the column the +#: factor's mean is taken over (microcosm#931). +UK_REGIONAL_PROPERTY_UPRATING_OPERATION_KIND = "uprate_to_regional_reference" +#: The column the factor's mean is taken over when the operation declares no +#: denominator: the main residence the stage rewrites, as before +#: microcosm#931. +UK_REGIONAL_PROPERTY_DEFAULT_DENOMINATOR = "main_residence_value" +#: The denominators the operation may declare: the main residence, or the WAS +#: chain's draw the spine's area home values keep beside the value they read +#: off the constituency's knots (``area_home_values``), so that the factor is +#: the one those values were divided by (microcosm#931). +UK_REGIONAL_PROPERTY_DENOMINATORS = ( + UK_REGIONAL_PROPERTY_DEFAULT_DENOMINATOR, + RANK_BASIS_COLUMN, +) @dataclass(frozen=True) @@ -59,9 +78,25 @@ def __call__(self, frame: Frame) -> Frame: if self.resource is not None else load_regional_land_values_resource() ) + denominator = declared_regional_reference_denominator(self.stage) + if ( + RANK_BASIS_COLUMN in frame.table("household").columns + and denominator == UK_REGIONAL_PROPERTY_DEFAULT_DENOMINATOR + ): + # Form F's two halves come together (C-15): homes the WAS chain + # read off their constituency's knots, divided by the factor on + # the draw it keeps, land on the knots only when this stage + # divides by that draw too (microcosm#931). + raise ValueError( + f"The regional property uprating reads households carrying " + f"{RANK_BASIS_COLUMN!r}, the draw the WAS chain's area home " + "values keep, but divides by main_residence_value: a stage " + "after them declares that draw its denominator (microcosm#931)." + ) household = uprate_household_property_by_region( frame.table("household"), resource, + denominator_column=denominator, ) result = uk_national_frame( person=frame.table("person").copy(), @@ -87,63 +122,147 @@ def output_columns() -> tuple[str, ...]: def load_regional_land_values_resource() -> Mapping[str, Any]: return json.loads( - files("microcosm.build.uk").joinpath("regional_land_values.json").read_text() + files("microcosm.build.uk") + .joinpath(UK_REGIONAL_LAND_VALUES_RESOURCE) + .read_text() ) -def uprate_household_property_by_region( - household: pd.DataFrame, - resource: Mapping[str, Any], -) -> pd.DataFrame: - """Scale owner property values to region-level public house-price means. +def declared_regional_reference_denominator(stage: SourceStageSpec) -> str: + """The column the stage's declared factor divides by. - The factor for a region is its public mean house price over the unweighted - mean ``main_residence_value`` of FRS-base owners. When the frame carries - SPI support rows, which are imputed from higher incomes, they are uprated - by the same factor but do not enter the mean. The factor scales the - property total and every component of it the table carries, so an owner's - ``property_wealth`` stays the sum of its components after the uprating. + The ``uprate_to_regional_reference`` operation's ``denominator``, or the + main residence where it declares none. Refused, by name: a denominator + outside :data:`UK_REGIONAL_PROPERTY_DENOMINATORS`, and a stage declaring + the operation more than once. """ - required = {"region", "main_residence_value", "property_wealth"} - missing = sorted(required - set(household.columns)) - if missing: - raise KeyError( - f"household table is missing property-uprating columns: {missing}" + operations = [ + operation + for operation in stage.operations + if operation.kind == UK_REGIONAL_PROPERTY_UPRATING_OPERATION_KIND + ] + if len(operations) > 1: + raise ValueError( + f"The regional property uprating declares " + f"{UK_REGIONAL_PROPERTY_UPRATING_OPERATION_KIND!r} more than once." ) + if not operations: + return UK_REGIONAL_PROPERTY_DEFAULT_DENOMINATOR + denominator = operations[0].parameters.get( + "denominator", UK_REGIONAL_PROPERTY_DEFAULT_DENOMINATOR + ) + if denominator not in UK_REGIONAL_PROPERTY_DENOMINATORS: + raise ValueError( + "The regional property uprating divides by one of " + f"{list(UK_REGIONAL_PROPERTY_DENOMINATORS)}; the stage declares " + "another denominator." + ) + return str(denominator) + + +def _regional_house_prices(resource: Mapping[str, Any]) -> dict[str, float]: values = resource.get("values") if not isinstance(values, list): raise ValueError("regional land values resource must contain a values list.") - hpi_prices = { + return { str(entry["region"]): float(entry["avg_house_price"]) for entry in values if isinstance(entry, Mapping) } - result = household.copy() + + +def _base_rows(household: pd.DataFrame) -> pd.Series: channel = support_channel_column("household") - base_rows = ( - result[channel] == BASE_FRS_SUPPORT_CHANNEL - if channel in result.columns - else pd.Series(True, index=result.index) + return ( + household[channel] == BASE_FRS_SUPPORT_CHANNEL + if channel in household.columns + else pd.Series(True, index=household.index) ) - for region, hpi_price in hpi_prices.items(): - region_mask = result["region"].astype(str) == region - owners_mask = region_mask & ( - pd.to_numeric(result["main_residence_value"], errors="coerce") > 0 + + +def _region_owners(household: pd.DataFrame, region: str) -> pd.Series: + """The region's owner rows: a positive main residence, as declared.""" + + region_mask = household["region"].astype(str) == region + return region_mask & ( + pd.to_numeric(household["main_residence_value"], errors="coerce") > 0 + ) + + +def regional_reference_factors( + household: pd.DataFrame, + resource: Mapping[str, Any], + *, + denominator_column: str = UK_REGIONAL_PROPERTY_DEFAULT_DENOMINATOR, +) -> dict[str, float]: + """Each region's factor: its public mean house price over a mean of its + FRS-base owners. + + The owners are the region's rows with a positive ``main_residence_value``; + the mean is the unweighted mean of their ``denominator_column``, summed in + sorted order so that it does not depend on the table's row order. SPI + support rows do not enter it. A region the resource lists with no FRS-base + owner, or whose mean is not positive, has no factor. Returns the factors in + the resource's order of regions. + """ + + required = {"region", "main_residence_value", denominator_column} + missing = sorted(required - set(household.columns)) + if missing: + raise KeyError( + f"household table is missing property-uprating columns: {missing}" ) - base_owners = owners_mask & base_rows + hpi_prices = _regional_house_prices(resource) + base_rows = _base_rows(household) + factors: dict[str, float] = {} + for region, hpi_price in hpi_prices.items(): + base_owners = _region_owners(household, region) & base_rows if not base_owners.any(): continue # Sort before summing so the unweighted mean is independent of row # order: the stage is receipted bitwise under row permutation, and # float addition is not associative. owner_values = np.sort( - result.loc[base_owners, "main_residence_value"].to_numpy(dtype=float) + household.loc[base_owners, denominator_column].to_numpy(dtype=float) ) imputed_mean = float(owner_values.sum() / len(owner_values)) if imputed_mean <= 0: continue - factor = hpi_price / imputed_mean + factors[region] = hpi_price / imputed_mean + return factors + + +def uprate_household_property_by_region( + household: pd.DataFrame, + resource: Mapping[str, Any], + *, + denominator_column: str = UK_REGIONAL_PROPERTY_DEFAULT_DENOMINATOR, +) -> pd.DataFrame: + """Scale owner property values to region-level public house-price means. + + The factor for a region is its public mean house price over the unweighted + mean ``denominator_column`` of FRS-base owners + (:func:`regional_reference_factors`), the ``main_residence_value`` unless + the stage declares otherwise. When the frame carries SPI support rows, + which are imputed from higher incomes, they are uprated by the same factor + but do not enter the mean. The factor scales the property total and every + component of it the table carries, so an owner's ``property_wealth`` stays + the sum of its components after the uprating. + """ + + required = {"region", "main_residence_value", "property_wealth", denominator_column} + missing = sorted(required - set(household.columns)) + if missing: + raise KeyError( + f"household table is missing property-uprating columns: {missing}" + ) + factors = regional_reference_factors( + household, resource, denominator_column=denominator_column + ) + result = household.copy() + for region, factor in factors.items(): + owners_mask = _region_owners(result, region) for column in UK_REGIONAL_PROPERTY_REWRITES: if column in result.columns: result.loc[owners_mask, column] *= factor diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py index 32035d7a99..96e4678e53 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/release_certification.py @@ -483,12 +483,14 @@ def compose_uk_release_certification( Every check is a refusal: a certification only exists when the parts union to the full declared entry set with no gap, no overlap beyond ``UK_SHARED_GATE_IDS``, full phase coverage, verified signatures, the - committed spec's scoped digests, green release-blocking verdicts, and a + committed spec's scoped digests, green release-blocking verdicts, a closed identity join from the spine report through the sidecar, the - build record, the diagnostics digest, and the candidate bytes, and a + build record, the diagnostics digest, and the candidate bytes, a head-to-head receipt that measured this candidate against the cut's own reference (``reference_record``, the current national build the cut - recorded when it fetched it). + recorded when it fetched it), and a spine of one geographic clone whose + geography binding is a release's (:func:`_verify_spine_geography`, + microcosm#931). """ from microcosm.build.uk_runtime.cut_reference import UKCutReference @@ -567,6 +569,7 @@ def compose_uk_release_certification( spine_sha256=spine_sha256, release_id=release_id, ) + _verify_spine_geography(spine_sidecar) score_receipt_bytes = score_receipt_path.read_bytes() score_receipt = json.loads(score_receipt_bytes) _verify_score_receipt( @@ -964,6 +967,55 @@ def _verify_identity_join( ) +def _verify_spine_geography(spine_sidecar: Mapping[str, Any]) -> None: + """Refuse a spine whose areas a national release cannot certify. + + The certifier's half of the geography binding check (microcosm#931; the + dense release pre-flight's is the other). The national line builds on a + geography-first spine of one geographic clone, so the spine's build + sidecar must record K = 1 (``geographic_clone_count``, the int one; a + sidecar that records none describes one clone, as the bound checkpoint + reads it) and a ``geography`` block that is a release's. The block is + held, with the sidecar's K beside it, to K = 1 by the helper the + pre-flight holds a dense candidate's binding to, + :func:`~microcosm.build.uk_runtime.country_adapter.uk_spine_geography_binding_failures`: + the atomic law, keyed on the geography key, under a definition named by + its SHA-256, on exactly the conditioned supports ``sources.yaml`` + registers, on the spine's area stream. The certifier tool holds the + sidecar to the one the calibration recorded (its digest in the build + record's ``spine_provenance``) and runs this check before its battery as + well. The refusal names each field that fails and K, never a value. + """ + + from microcosm.build.uk_runtime import country_adapter + from microcosm.build.uk_runtime.spine_posture import NATIONAL_CLONE_COUNT + + lead = ( + "a national release certifies only a spine of one geographic clone " + "whose geography binding is a release's (microcosm#931); " + ) + clones = spine_sidecar.get("geographic_clone_count", NATIONAL_CLONE_COUNT) + geography = spine_sidecar.get("geography") + if not (isinstance(geography, Mapping) and geography): + refusal = ( + "the spine sidecar records no geography binding (a spine that " + "runs no area draw writes none)" + ) + if type(clones) is not int or clones != NATIONAL_CLONE_COUNT: + refusal += ( + f", and its geographic_clone_count is not K = {NATIONAL_CLONE_COUNT}" + ) + raise UKReleaseCertificationError(lead + refusal + ".") + failures = country_adapter.uk_spine_geography_binding_failures( + {**geography, "geographic_clone_count": clones}, + clone_count=NATIONAL_CLONE_COUNT, + ) + if failures: + raise UKReleaseCertificationError( + lead + "the spine sidecar's binding fails: " + "; ".join(failures) + "." + ) + + #: The head-to-head receipt a certification reads (candidate_score schema 2). UK_SCORE_RECEIPT_SCHEMA_VERSION = 2 diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/rfdr_area_layer.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/rfdr_area_layer.py index 9ec0a511c4..33462031c1 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/rfdr_area_layer.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/rfdr_area_layer.py @@ -11,11 +11,15 @@ excluded units, beside the notice's own districts (``tools/build_uk_rfdr_area_layer.py``). The tool reads the ONS postcode lookup and flags every 2021 output area of England and Wales and 2022 output area of Scotland whose postcodes mostly lie in the scheme; -``uk/rfdr_area_layer.json`` lists them. The post-geography node -``uk.full.geography.rfdr`` sets the flag on each placed household from its -assigned output area. Northern Ireland has no area in the scheme. The -national line has no areas, so it leaves the engine's default of false -(María's ruling of 2026-10-28). +``uk/rfdr_area_layer.json`` lists them. The clone-first spine's fuel-relief +stage (``spine_geography.UKRFDRStageTransform``, G5 of microcosm#931) sets the +flag on each household from the output area the spine draws it, and holds +the layer to the spine's supports: every output area the layer flags must be +an area of its system's support (:func:`area_flags`' ``roster``). Northern +Ireland has no area in the scheme. A dataset built from a spine without +areas carries no flag and keeps the engine's default of false, as María's +ruling of 2026-10-28 set for the national line while it built on such spines; +it now builds only on a spine that assigns areas (microcosm#931). Everything here is pure and depends on the standard library, numpy and pandas only, so the build tool can load this module by path. @@ -36,7 +40,6 @@ "METHOD_VERSION", "RFDR_AREA_LAYER_SOURCE", "RFDR_COLUMN", - "RFDR_LOCALISATION_MODES", "SCHEMA_VERSION", "SYSTEMS", "RFDRAreaLayer", @@ -52,10 +55,7 @@ METHOD_VERSION = "uk-rfdr-area-layer-v1" #: Graph source name of the committed layer bytes. RFDR_AREA_LAYER_SOURCE = "uk_rfdr_area_layer" -#: ``area`` sets each placed Great Britain household's flag from its assigned -#: output area; ``none`` leaves the engine's default on every clone. -RFDR_LOCALISATION_MODES = ("none", "area") -#: The policyengine-uk household input the node writes. +#: The policyengine-uk household input the spine's fuel-relief stage sets. RFDR_COLUMN = "in_rural_fuel_duty_relief_area" #: The support systems the layer covers, by the output-area code they read. SYSTEMS: Mapping[str, str] = { @@ -245,29 +245,59 @@ def area_flags( area_system: Iterable[object], area_code: Iterable[object], require_flagged: bool = True, + roster: Mapping[str, Iterable[object]] | None = None, ) -> tuple[np.ndarray, dict[str, int]]: """Each household's flag from its assigned output area. A Great Britain household is in the scheme when the layer lists its area; a Northern Ireland household never is. Any other support system refuses, since the layer cannot say where it is. A Great Britain code that is not a - well-formed output-area code of its system refuses too, and so does a - system (England and Wales, or Scotland) with households none of which is - flagged: either would let a change of code format or vintage publish its - households unflagged (Anthony's reviews of microcosm#1420). The zero-flag - refusal holds only with ``require_flagged``, which a full-scale build - sets: a sampled rung places so few England and Wales households (41 of - 188,880 output areas are in the scheme) that it can miss them all by - chance (Vahid's review and the independent review of microcosm#1420). + well-formed output-area code of its system refuses too: it would let a + change of code format publish its households unflagged (Anthony's reviews + of microcosm#1420). Against a change of code vintage, a caller holds the + layer to either check, or both: + + - ``roster`` gives the areas of each Great Britain system's support. + Every output area the layer flags must be one of its system's, so a + layer and a support of different vintages refuse whatever the + households drew. The clone-first spine's fuel-relief stage passes its + supports' areas (microcosm#931, C-6). + - ``require_flagged`` refuses a system (England and Wales, or Scotland) + with households none of which is flagged. It suits a full-scale build + only: a sampled rung places so few England and Wales households (41 of + 188,880 output areas are in the scheme) that it can miss them all by + chance (Vahid's review and the independent review of microcosm#1420). + + The unknown-system, roster and malformed-code refusals name no value of + a household and no code, and count nothing. Returns the flags and exact + row counts by system; callers round counts before any receipt. """ systems = np.asarray(list(area_system), dtype=object) codes = np.asarray(list(area_code), dtype=object) if len(systems) != len(codes): raise ValueError("area_system and area_code must align.") - unknown = sorted({str(system) for system in systems} - {*SYSTEMS, NI_SYSTEM}) - if unknown: - raise ValueError(f"RFDR flags read no support system {unknown}.") + known = {*SYSTEMS, NI_SYSTEM} + if any(str(system) not in known for system in systems): + raise ValueError( + f"RFDR flags read households of the support systems {sorted(known)} " + "only; area_system holds another value." + ) + if roster is not None: + if not isinstance(roster, Mapping) or any( + system not in roster for system in SYSTEMS + ): + raise ValueError( + "RFDR flags hold the layer to a roster of support areas for each " + f"Great Britain system {sorted(SYSTEMS)}." + ) + for system, flagged in layer.areas.items(): + if not flagged <= {str(code) for code in roster[system]}: + raise ValueError( + f"The RFDR layer flags {system} output areas that system's " + "support does not hold: the layer and the support must carry " + "one code vintage (microcosm#931)." + ) flags = np.zeros(len(codes), dtype=bool) counts: dict[str, int] = {} for system, flagged in layer.areas.items(): @@ -280,8 +310,8 @@ def area_flags( ] if malformed: raise ValueError( - f"RFDR flags read {len(malformed)} {system} code(s) that are not " - f"well-formed output-area codes (first: {malformed[0]!r})." + f"RFDR flags read {system} code(s) that are not well-formed " + "output-area codes." ) flags[rows] = np.asarray([code in flagged for code in codes[rows]], dtype=bool) if require_flagged and rows.any() and not flags[rows].any(): diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_cli.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_cli.py index 8ec6c77892..09a4dc9a45 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_cli.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_cli.py @@ -107,6 +107,7 @@ "record_candidate_error", "refuse_dense_role_arguments", "refuse_national_role_arguments", + "refuse_spine_bound_arguments", "release_verdict", "resolve_role_arguments", "size_doctrine_drift", @@ -265,7 +266,12 @@ def _size_l2_parameters(args: argparse.Namespace) -> dict[str, Any]: def rowwise_parameters(args: argparse.Namespace, *, source_year: int) -> dict[str, Any]: - """The run parameters a rowwise manifest, plan and identity digest record.""" + """The run parameters a rowwise manifest, plan and identity digest record. + + On the dense line ``n_clones``, ``sample_fraction`` and ``sample_seed`` + are the bound spine's: the build's preparation sets them from the spine + before anything records them (microcosm#931). + """ posture = posture_of(args) return { @@ -295,7 +301,6 @@ def rowwise_parameters(args: argparse.Namespace, *, source_year: int) -> dict[st if args.resume_size_checkpoint is None else str(args.resume_size_checkpoint.expanduser().resolve()), "source_year": source_year, - "source_lineage_modulus": args.source_lineage_modulus, "sample_fraction": float(args.sample_fraction), "sample_seed": int(args.sample_seed), "engine_blocks": int(args.engine_blocks), @@ -304,11 +309,6 @@ def rowwise_parameters(args: argparse.Namespace, *, source_year: int) -> dict[st "skip_holdout": bool(getattr(args, "skip_holdout", False)), "epochs": int(args.epochs), "learning_rate": float(args.learning_rate), - "expected_constituency_vintage": ( - None - if args.expected_constituency_vintage is None - else str(args.expected_constituency_vintage) - ), "doctrine": doctrine_bounds(posture), "solve_options": { "conserve_mass": CONSERVE_MASS, @@ -429,31 +429,6 @@ def record_candidate_error( #: Solve arguments whose argparse default is ``None`` so an explicit value can #: be told from the role's default: the other role's refusal table keys on #: what was actually given. -#: CLI labels of the three UK atomic-area supports (microcosm#932), in -#: ``atomic_area_support.SYSTEMS`` order; ``full_build_cli`` declares the -#: ``--atomic-support-