From 014a1b8bf3ea8b596010b2a5c4c6a277df482e96 Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Fri, 16 Oct 2026 09:59:35 +0100 Subject: [PATCH 1/8] Refuse parent_of codes that name no area of the parent level For a weight-matrix level, Release._groups checked that every child area had an entry in parent_of, but not that the entry was an area of the parent level. A mapping that spelt the parent codes another way left every group empty, so the nesting check passed with no finding. On the synthetic probe from the issue, a seat withheld because three records carry it went unflagged as soon as the regions were named differently. The check now raises DisclosureError and gives the count of unknown codes, never the codes. It checks the parent codes of the child areas the estimates hold; entries for other areas are ignored, as before. The probe is a test. Part of #625. Co-Authored-By: Claude Opus 5.5 --- docs/disclosure.md | 4 ++ src/policyengine/disclosure/release.py | 11 +++++ tests/test_disclosure_release.py | 64 ++++++++++++++++++++++++++ 3 files changed, 79 insertions(+) diff --git a/docs/disclosure.md b/docs/disclosure.md index 7cbffe84..82d4c9ae 100644 --- a/docs/disclosure.md +++ b/docs/disclosure.md @@ -117,6 +117,10 @@ are a reference: the release does not emit their rows, but it treats the rows that pass as public elsewhere and the withheld ones as secret, and checks them like its own. +The check matches areas by their codes, and it raises on a code that matches +nothing rather than leave areas unchecked. Each value of `parent_of` must be +an area code of the parent level's estimates. + `compute_uk_geography_disclosure` always checks constituencies and local authorities against their region, and regions against the nation, also when you publish one level only. A level given as a weight matrix needs diff --git a/src/policyengine/disclosure/release.py b/src/policyengine/disclosure/release.py index 6e45895c..51a31af1 100644 --- a/src/policyengine/disclosure/release.py +++ b/src/policyengine/disclosure/release.py @@ -107,6 +107,8 @@ def add_nesting( ``parent_of`` (child area code to parent area code) is needed only when the child level is a weight matrix and the parent has several areas. + Its parent codes must be area codes of the parent estimates: the check + raises on one that is not. """ mapping = dict(parent_of) if parent_of is not None else None self._nestings.append((child_level, parent_level, mapping)) @@ -339,6 +341,15 @@ def _groups( raise DisclosureError( f"parent_of lacks {len(missing)} {child.level} areas." ) + # Groups match by code. A code that is no parent area would leave + # its child areas in no group, and the check would pass unseen. + unknown = {parent_of[c] for c in child.areas.area_codes} - set(parents) + if unknown: + raise DisclosureError( + f"Codes in parent_of that are not {parent.level} areas: " + f"{len(unknown)}. Map each {child.level} area to an area " + f"code of the {parent.level} estimates." + ) return { code: ( [c for c in child.areas.area_codes if parent_of[c] == code], diff --git a/tests/test_disclosure_release.py b/tests/test_disclosure_release.py index d90cdbd5..44cb2f96 100644 --- a/tests/test_disclosure_release.py +++ b/tests/test_disclosure_release.py @@ -629,6 +629,70 @@ def test_a_weight_matrix_child_needs_parent_of_under_several_parents(): assert release.check() == [] +def _seat_matrix_release(parent_of): + """400 records in two regions and a 4 x 400 seat matrix. + + Three records carry seat M1, so it is withheld. M2 holds the rest of R1; + M3 and M4 share R2, and a few records there carry weight in both. + """ + n = 400 + regions = np.array(["R1"] * 200 + ["R2"] * 200, dtype=object) + matrix = np.zeros((4, n)) + matrix[0, :3] = 100.0 + matrix[1, 3:200] = 100.0 + matrix[2, 200:300] = 100.0 + matrix[3, 300:] = 100.0 + matrix[2:, 290:300] = 50.0 + baseline = np.random.default_rng(2).uniform(20_000.0, 60_000.0, n) + common = { + "reform": "reform", + "year": 2026, + "source_ids": np.arange(n), + "baseline": baseline, + "reformed": baseline + 200.0, + } + release = Release() + release.add( + estimate_change( + level="region", + areas=AreaWeights.from_labels(regions, np.full(n, 100.0)), + **common, + ) + ) + release.add( + estimate_change( + level="constituency", + areas=AreaWeights.from_matrix(matrix, ["M1", "M2", "M3", "M4"]), + **common, + ) + ) + release.add_nesting("constituency", "region", parent_of) + return release + + +def test_parent_of_must_name_areas_of_the_parent_level(): + """Regions named another way left every group empty, so nothing was found.""" + own = {"M1": "R1", "M2": "R1", "M3": "R2", "M4": "R2"} + + findings = _seat_matrix_release(own).check() + + blocking = _blocking(findings, RECOVERABLE_REMAINDER) + assert len(blocking) == len(findings) == 3 + assert {f.family for f in blocking} == {"change", "share_gaining", "share_losing"} + assert {(f.relation, f.group, tuple(f.members)) for f in blocking} == { + ("nesting", "R1", ("M1",)) + } + + renamed = {"M1": "North", "M2": "North", "M3": "South", "M4": "South"} + with pytest.raises(DisclosureError, match="not region areas: 2") as raised: + _seat_matrix_release(renamed).check() + assert "North" not in str(raised.value) + assert "South" not in str(raised.value) + + with pytest.raises(DisclosureError, match="not region areas: 1"): + _seat_matrix_release({**own, "M4": "South"}).check() + + def test_the_release_refuses_mixed_settings_and_duplicates(): data = _records(STRONG) release = Release() From c7357ecee79806240c64a833405daa2348534a13 Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Fri, 16 Oct 2026 10:02:04 +0100 Subject: [PATCH 2/8] Refuse prior cells that match no area or outcome of the release The release used PriorPublic cells as given. A cell whose area code matched no area of its level was ignored, so a cell that an earlier publication showed, and that this release withholds, still counted as hidden, and the checks that depend on it never ran. Regions published under one spelling and passed to a release that holds them under another matched nothing, without an error. check() now raises DisclosureError for a prior cell at a level the release holds whose area code is not an area of that level, whatever the cell's reform and year. It runs in check(), so the order in which estimates are added does not matter. Release() also refuses a prior cell with an unknown outcome. Cells at levels the release does not hold, or for reforms or years it does not hold, belong to other publications and stay allowed. The messages give the prior's name and the count of cells, never codes or values. Part of #625. Co-Authored-By: Claude Opus 5.5 --- docs/disclosure.md | 7 +++- src/policyengine/disclosure/release.py | 39 ++++++++++++++++++++ src/policyengine/disclosure/schema.py | 6 ++- tests/test_disclosure_release.py | 51 ++++++++++++++++++++++++++ 4 files changed, 101 insertions(+), 2 deletions(-) diff --git a/docs/disclosure.md b/docs/disclosure.md index 82d4c9ae..5a460cec 100644 --- a/docs/disclosure.md +++ b/docs/disclosure.md @@ -119,7 +119,12 @@ them like its own. The check matches areas by their codes, and it raises on a code that matches nothing rather than leave areas unchecked. Each value of `parent_of` must be -an area code of the parent level's estimates. +an area code of the parent level's estimates. Each cell of `prior_public` at +a level the release holds must name an area of that level, whatever its +reform and year, and every cell needs one of the six outcomes. Cells at other +levels, or for other reforms or years, belong to other publications: the +release accepts them, but they take no part in the check. A prior cell counts +only when its level, area code, reform label and year are the release's. `compute_uk_geography_disclosure` always checks constituencies and local authorities against their region, and regions against the nation, also when diff --git a/src/policyengine/disclosure/release.py b/src/policyengine/disclosure/release.py index 51a31af1..d42c69a2 100644 --- a/src/policyengine/disclosure/release.py +++ b/src/policyengine/disclosure/release.py @@ -27,6 +27,7 @@ FAMILY_LOSING, NOISE_REFERENCE_PUBLIC, OUTCOME_FAMILY, + OUTCOMES, PRIVATE_SECONDARY, REASON_DISCLOSURE_CONTROL, REASON_THIN_SUPPORT, @@ -71,6 +72,17 @@ def __init__( if unknown: raise ValueError(f"These findings cannot be acknowledged: {unknown}") self._prior = tuple(prior_public) + for prior in self._prior: + # A cell with an unknown outcome matches no cell of any release. + unknown_outcomes = sum( + outcome not in OUTCOMES for *_, outcome in prior.cells + ) + if unknown_outcomes: + raise DisclosureError( + f"Cells of prior publication {prior.name!r} whose outcome is " + f"unknown: {unknown_outcomes}. Outcomes are " + f"{', '.join(OUTCOMES)}." + ) self._estimates: dict[_Key, AreaEstimates] = {} self._published: dict[_Key, bool] = {} self._nestings: list[tuple[str, str, Optional[dict[str, str]]]] = [] @@ -146,6 +158,7 @@ def findings(self) -> Optional[list[Finding]]: def check(self) -> list[Finding]: """Report what a reader could recover from the rows as they stand.""" + self._require_prior_areas() tiers = { (key, family): self._family_tiers(key, family) for key in self._estimates @@ -268,6 +281,32 @@ def _effective_rows(self, key: _Key) -> Iterable[tuple[dict, str, Optional[str]] else: yield record, record["tier"], reason + def _require_prior_areas(self) -> None: + """Refuse prior cells that name no area of a level the release holds. + + Prior cells match by code. A cell that matched nothing would be + ignored, so a cell an earlier publication showed would still count as + hidden. A level the release holds is checked whatever the reform and + year of the cell; cells at other levels belong to other publications. + """ + areas: dict[str, set[str]] = {} + for (level, _, _), estimates in self._estimates.items(): + areas.setdefault(level, set()).update(estimates.areas.area_codes) + for prior in self._prior: + unmatched = [ + level + for level, area, _, _, _ in prior.cells + if level in areas and area not in areas[level] + ] + if unmatched: + levels = " or ".join(sorted(set(unmatched))) + raise DisclosureError( + f"Cells of prior publication {prior.name!r} that name no " + f"{levels} area of the release: {len(unmatched)}. Prior " + "cells match by code: use the area codes of the release's " + "estimates." + ) + def _known_cells(self) -> set[tuple[str, str, str, int, str]]: """Family cells an earlier publication already made public.""" known = set() diff --git a/src/policyengine/disclosure/schema.py b/src/policyengine/disclosure/schema.py index af8792cf..e66d56ce 100644 --- a/src/policyengine/disclosure/schema.py +++ b/src/policyengine/disclosure/schema.py @@ -252,7 +252,11 @@ class Finding(BaseModel): class PriorPublic(BaseModel): - """Cells an earlier publication already made public.""" + """Cells an earlier publication already made public. + + Each cell is ``(level, area_code, reform, year, outcome)``. A release + matches cells by code, so they must use the area codes of its estimates. + """ model_config = ConfigDict(frozen=True, extra="forbid") diff --git a/tests/test_disclosure_release.py b/tests/test_disclosure_release.py index 44cb2f96..b0e25b0a 100644 --- a/tests/test_disclosure_release.py +++ b/tests/test_disclosure_release.py @@ -601,6 +601,57 @@ def test_a_prior_with_another_noise_setting_blocks_until_acknowledged(): assert [f.blocking for f in accepted.check()] == [False] +@pytest.mark.parametrize( + "order", [("region", "constituency"), ("constituency", "region")] +) +def test_a_prior_cell_must_name_an_area_of_a_level_the_release_holds(order): + """Regions published under other codes would still count as hidden.""" + prior = PriorPublic( + name="regional table", + cells=[ + ("region", "North", "reform", 2026, "average_change"), + ("region", "North", "reform", 2026, "share_gaining"), + ("region", "R2", "reform", 2026, "average_change"), + ], + ) + data = _records(STRONG) + release = Release(prior_public=[prior]) + for level in order: + # The regions are a reference, as in a seat release after a regional one. + release.add(_estimates(data, level), publish=level == "constituency") + release.add_nesting("constituency", "region") + + with pytest.raises( + DisclosureError, match=r"'regional table' that name no region area.*: 2\." + ) as raised: + release.check() + assert "North" not in str(raised.value) + + +def test_a_prior_cell_needs_a_known_outcome(): + cells = [("constituency", "S1", "reform", 2026, "avg_change")] + + with pytest.raises(DisclosureError, match="'typo table' whose outcome is unknown"): + Release(prior_public=[PriorPublic(name="typo table", cells=cells)]) + + +def test_prior_cells_of_other_publications_are_allowed(): + """Cells for other reforms, years or levels take no part in the check.""" + cells = [ + ("constituency", "S1", "other reform", 2026, "average_change"), + ("constituency", "S1", "reform", 2025, "share_gaining"), + ("region", "R1", "reform", 2027, "share_no_change"), + ("local_authority", "L9", "reform", 2026, "average_change"), + ] + prior = PriorPublic(name="other tables", cells=cells) + affected = {**STRONG, "S1": 4, "S2": 30} + + findings = _release(affected, prior_public=[prior]).check() + + assert findings == _release(affected).check() + assert _blocking(findings, RECOVERABLE_REMAINDER) + + def test_a_weight_matrix_child_needs_parent_of_under_several_parents(): data = _records(STRONG) n = len(data["seats"]) From 7056e0b1efb6605269cebcf4f38b1c0d99a0f1bd Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Fri, 16 Oct 2026 10:09:03 +0100 Subject: [PATCH 3/8] Key UK regions on ONS GSS codes in the disclosure adapter The adapter keyed regions on region_code_oa when the household output had it, else on the stored region. On a Microcosm local file that gave GSS codes such as E12000007; on a national dataset, the engine's enum names such as LONDON. Regional results published from the national dataset and passed as prior_public to a seat release from a local file therefore matched nothing. Regions are now always keyed on ONS GSS codes: E12000001 to E12000009 for England's nine regions, and the country codes W92000004, S92000003 and N92000002 for Wales, Scotland and Northern Ireland. The table is one constant, UK_REGION_GSS_CODES. It reads GSS codes, the pseudo codes W99999999, S99999999 and N99999999 that Microcosm writes for the nations, and the enum names, as strings, bytes or enum members. UNKNOWN and missing values are unassigned, so those households count in the nation but in no region. Any other value raises, with a count and never the values. Publishers must use the same GSS codes in parent_of and in the region cells of prior_public; the release check now raises on codes that match nothing. The docs set out the codes. Part of #625. Co-Authored-By: Claude Opus 5.5 --- docs/disclosure.md | 38 ++++- .../outputs/uk_geography_disclosure.py | 84 +++++++++- tests/test_uk_geography_disclosure.py | 152 +++++++++++++++++- 3 files changed, 261 insertions(+), 13 deletions(-) diff --git a/docs/disclosure.md b/docs/disclosure.md index 5a460cec..17ad9a77 100644 --- a/docs/disclosure.md +++ b/docs/disclosure.md @@ -129,7 +129,8 @@ only when its level, area code, reform label and year are the release's. `compute_uk_geography_disclosure` always checks constituencies and local authorities against their region, and regions against the nation, also when you publish one level only. A level given as a weight matrix needs -`parent_of=` with the region of each area; without it the function raises, +`parent_of=` with the GSS code of each area's region (see +[UK region codes](#uk-region-codes)); without it the function raises, because a check against the nation alone nearly always passes. The check reports; it does not withhold cells by itself. A blocking finding @@ -161,6 +162,41 @@ Two findings can be accepted after review, with `acknowledge=[...]`: - `noise_reference_public`: an earlier table shows the same cells under another noise setting. +## UK region codes + +`compute_uk_geography_disclosure` keys regions on their ONS GSS codes, +whichever column the household output carries. It reads `region_code_oa` +when the output has it, as Microcosm's local files do, and the stored +`region` otherwise, as on the national datasets. Both give the same codes: + +| Region | Stored `region` | GSS code | +|---|---|---| +| North East | `NORTH_EAST` | `E12000001` | +| North West | `NORTH_WEST` | `E12000002` | +| Yorkshire and the Humber | `YORKSHIRE` | `E12000003` | +| East Midlands | `EAST_MIDLANDS` | `E12000004` | +| West Midlands | `WEST_MIDLANDS` | `E12000005` | +| East of England | `EAST_OF_ENGLAND` | `E12000006` | +| London | `LONDON` | `E12000007` | +| South East | `SOUTH_EAST` | `E12000008` | +| South West | `SOUTH_WEST` | `E12000009` | +| Wales | `WALES` | `W92000004` | +| Scotland | `SCOTLAND` | `S92000003` | +| Northern Ireland | `NORTHERN_IRELAND` | `N92000002` | + +Wales, Scotland and Northern Ireland are one region each, under their country +code. Microcosm writes the pseudo codes `W99999999`, `S99999999` and +`N99999999` for them in `region_code_oa`, and the adapter reads those as the +country codes. A household whose `region` is `UNKNOWN` counts in the nation +but in no region. Any other value raises. The table is `UK_REGION_GSS_CODES` +in `policyengine.outputs.uk_geography_disclosure`. + +Use the GSS codes wherever you name a region: in `parent_of`, and in the +region cells of `prior_public`. Names such as `London` or `Eastern` are not +codes; map them first. Then a regional release from the national dataset and +a seat release from a local file publish the same codes, and the regional +table can be passed as `prior_public` to the seat release. + ## Noise Noise is off by default. With `DisclosureSettings(noise=True)`, each survey diff --git a/src/policyengine/outputs/uk_geography_disclosure.py b/src/policyengine/outputs/uk_geography_disclosure.py index 511eacfe..64c77d2f 100644 --- a/src/policyengine/outputs/uk_geography_disclosure.py +++ b/src/policyengine/outputs/uk_geography_disclosure.py @@ -7,6 +7,7 @@ """ from collections.abc import Iterable, Mapping, Sequence +from enum import Enum from pathlib import Path from typing import TYPE_CHECKING, Optional, Union @@ -16,6 +17,7 @@ from policyengine.core import Output from policyengine.disclosure import ( + UNASSIGNED, AreaWeights, DisclosureSettings, Finding, @@ -39,6 +41,74 @@ "local_authority": ("la_code_oa",), } UK_SOURCE_HOUSEHOLD_ID_COLUMN = "source_household_id" +# The ONS GSS code of each region, by every spelling a household output may +# carry. Regions are always keyed on these codes, so a release from a national +# dataset and one from a local file publish the same codes, and publishers use +# them in parent_of and prior_public. England has nine regions; Wales, Scotland +# and Northern Ireland are one region each, under their country code. +UK_REGION_GSS_CODES: dict[str, str] = { + # GSS codes, as Microcosm writes them in region_code_oa. + "E12000001": "E12000001", + "E12000002": "E12000002", + "E12000003": "E12000003", + "E12000004": "E12000004", + "E12000005": "E12000005", + "E12000006": "E12000006", + "E12000007": "E12000007", + "E12000008": "E12000008", + "E12000009": "E12000009", + "W92000004": "W92000004", + "S92000003": "S92000003", + "N92000002": "N92000002", + # The pseudo codes Microcosm writes in region_code_oa for the nations. + "W99999999": "W92000004", + "S99999999": "S92000003", + "N99999999": "N92000002", + # The names of the engine's region enum, as the stored region holds them. + "NORTH_EAST": "E12000001", + "NORTH_WEST": "E12000002", + "YORKSHIRE": "E12000003", + "EAST_MIDLANDS": "E12000004", + "WEST_MIDLANDS": "E12000005", + "EAST_OF_ENGLAND": "E12000006", + "LONDON": "E12000007", + "SOUTH_EAST": "E12000008", + "SOUTH_WEST": "E12000009", + "WALES": "W92000004", + "SCOTLAND": "S92000003", + "NORTHERN_IRELAND": "N92000002", + "UNKNOWN": UNASSIGNED, +} + + +def _uk_region_gss_codes(values: object, column: str) -> np.ndarray: + """The GSS code of each region value, blank where it is unassigned. + + Values may be strings, bytes or members of the engine's region enum. A + missing value is unassigned. Any value outside the table raises: a region + under another spelling would match no code that a publisher passes. + """ + index, spellings = pd.factorize(pd.Series(values, dtype=object)) + codes: list[Optional[str]] = [] + unknown = 0 + for spelling in spellings: + if isinstance(spelling, Enum): + spelling = spelling.name + if isinstance(spelling, bytes): + spelling = spelling.decode("utf-8") + spelling = str(spelling).strip() + code = UK_REGION_GSS_CODES.get(spelling) if spelling else UNASSIGNED + if code is None: + unknown += 1 + codes.append(code) + if unknown: + raise ValueError( + f"Values in {column} that name no UK region: {unknown}. A region is " + "a GSS code, a name of the engine's region enum, or a Microcosm " + "pseudo code for a nation; see UK_REGION_GSS_CODES." + ) + # Factorize marks a missing value with -1, which picks the last entry. + return np.asarray(np.array([*codes, UNASSIGNED], dtype=object)[index]) class UKGeographyDisclosure(Output): @@ -47,6 +117,7 @@ class UKGeographyDisclosure(Output): Every number passes the disclosure rules of :class:`~policyengine.disclosure.DisclosureSettings` and carries a 90% interval. ``public`` stays empty while a blocking finding remains. + Regions are keyed on their ONS GSS codes; see ``UK_REGION_GSS_CODES``. """ model_config = ConfigDict(arbitrary_types_allowed=True) @@ -136,7 +207,10 @@ def areas_for(level: str) -> Optional[AreaWeights]: raise ValueError(f"Unknown level {level!r}") for column in columns: if column in baseline_hh.columns: - return AreaWeights.from_labels(baseline_hh[column], weights) + codes = baseline_hh[column].to_numpy(dtype=object) + if level == UK_REGION_LEVEL: + codes = _uk_region_gss_codes(codes, column) + return AreaWeights.from_labels(codes, weights) return None settings = self.settings or DisclosureSettings() @@ -242,11 +316,13 @@ def compute_uk_geography_disclosure( the dataset has no ``source_household_id`` column. area_weights: Weights for a level, in place of the level's area code column, for example an area x household weight matrix. - parent_of: For a level given as a weight matrix, the region code of - each of its areas. Required for such a level when the household - output carries regions. + parent_of: For a level given as a weight matrix, the GSS code of the + region of each of its areas (see ``UK_REGION_GSS_CODES``). + Required for such a level when the household output carries + regions. acknowledge: Findings the caller accepts after review. prior_public: Earlier publications that count as already public. + Their region cells use GSS codes too. audit_dir: Private directory, outside any repository, for the exact values and counts. diff --git a/tests/test_uk_geography_disclosure.py b/tests/test_uk_geography_disclosure.py index 322163b9..743df122 100644 --- a/tests/test_uk_geography_disclosure.py +++ b/tests/test_uk_geography_disclosure.py @@ -3,6 +3,7 @@ All data here is synthetic. """ +from enum import Enum from unittest.mock import MagicMock import numpy as np @@ -11,22 +12,55 @@ from microdf import MicroDataFrame from policyengine import outputs +from policyengine.countries.uk.regions import ENGLISH_REGIONS from policyengine.disclosure import ( NOISE_KEY_ENV, AreaWeights, DisclosureError, DisclosureKeyError, DisclosureSettings, + PriorPublic, PublicTable, validate_public, ) from policyengine.outputs.uk_geography_disclosure import ( + UK_REGION_GSS_CODES, UKGeographyDisclosure, compute_uk_geography_disclosure, ) TEST_KEY = "test-key-for-unit-tests-only" SEATS = {"S1": "NORTH_EAST", "S2": "NORTH_EAST", "S3": "LONDON", "S4": "LONDON"} +NATIONS = { + "S1": "NORTH_EAST", + "S2": "WALES", + "S3": "SCOTLAND", + "S4": "NORTHERN_IRELAND", +} +GSS = { + "NORTH_EAST": "E12000001", + "LONDON": "E12000007", + "WALES": "W92000004", + "SCOTLAND": "S92000003", + "NORTHERN_IRELAND": "N92000002", +} +# What Microcosm writes in region_code_oa: GSS codes, pseudo codes for nations. +MICROCOSM = { + **GSS, + "WALES": "W99999999", + "SCOTLAND": "S99999999", + "NORTHERN_IRELAND": "N99999999", +} + + +class Region(Enum): + """Shaped like the engine's region enum: stored names, display values.""" + + NORTH_EAST = "North East" + WALES = "Wales" + SCOTLAND = "Scotland" + NORTHERN_IRELAND = "Northern Ireland" + UNKNOWN = "Unknown" def _make_sim(household_data: dict) -> MagicMock: @@ -38,11 +72,11 @@ def _make_sim(household_data: dict) -> MagicMock: return sim -def _households(affected=None, *, per_seat=150): - affected = affected or dict.fromkeys(SEATS, 80) +def _households(affected=None, *, per_seat=150, region_of=SEATS): + affected = affected or dict.fromkeys(region_of, 80) rng = np.random.default_rng(0) seats, regions, deltas = [], [], [] - for seat, region in SEATS.items(): + for seat, region in region_of.items(): seats += [seat] * per_seat regions += [region] * per_seat deltas += list(rng.uniform(100.0, 900.0, affected[seat])) @@ -78,7 +112,7 @@ def test_regions_are_published_with_intervals_and_pass_the_validator(): assert isinstance(impact, UKGeographyDisclosure) assert impact.findings == [] assert set(frame["level"]) == {"region"} - assert set(frame["area_code"]) == {"NORTH_EAST", "LONDON"} + assert set(frame["area_code"]) == {"E12000001", "E12000007"} averages = frame[frame["outcome"] == "average_change"] assert (averages["tier"] == "reported").all() assert (averages["lo"] <= averages["value"]).all() @@ -102,17 +136,115 @@ def test_public_rows_round_trip_through_json(): } +def test_the_region_table_maps_every_stored_region_to_its_gss_code(): + stored = (*ENGLISH_REGIONS, "WALES", "SCOTLAND", "NORTHERN_IRELAND") + codes = [f"E1200000{number}" for number in range(1, 10)] + codes += ["W92000004", "S92000003", "N92000002"] + + assert [UK_REGION_GSS_CODES[name] for name in stored] == codes + assert [UK_REGION_GSS_CODES[code] for code in codes] == codes + assert [ + UK_REGION_GSS_CODES[code] for code in ("W99999999", "S99999999", "N99999999") + ] == codes[-3:] + assert UK_REGION_GSS_CODES["UNKNOWN"] == "" + + +def test_regions_are_keyed_on_gss_codes_whatever_the_dataset_stores(): + stored = { + "enum names": ("region", str), + "bytes": ("region", str.encode), + "enum members": ("region", Region.__getitem__), + "GSS codes": ("region_code_oa", GSS.__getitem__), + "Microcosm codes": ("region_code_oa", MICROCOSM.__getitem__), + } + tables = {} + for name, (column, spell) in stored.items(): + baseline, reform = _households(region_of=NATIONS) + values = [spell(region) for region in baseline.pop("region")] + reform.pop("region") + baseline[column] = reform[column] = values + tables[name] = _compute(baseline, reform).public.to_frame() + + assert set(tables["enum names"]["area_code"]) == { + "E12000001", + "W92000004", + "S92000003", + "N92000002", + } + for frame in tables.values(): + pd.testing.assert_frame_equal(frame, tables["enum names"]) + + def test_region_codes_are_preferred_over_the_stored_region(): baseline, reform = _households() codes = [ - "E12000001" if r == "NORTH_EAST" else "E12000007" for r in baseline["region"] + "E12000002" if r == "NORTH_EAST" else "E12000008" for r in baseline["region"] ] baseline["region_code_oa"] = codes reform["region_code_oa"] = codes impact = _compute(baseline, reform) - assert set(impact.public.to_frame()["area_code"]) == {"E12000001", "E12000007"} + assert set(impact.public.to_frame()["area_code"]) == {"E12000002", "E12000008"} + + +def test_an_unknown_region_is_in_the_nation_but_in_no_region(): + baseline, reform = _households() + # The last households of S4 do not change. + regions = [*baseline["region"][:-30], *[Region.UNKNOWN] * 30] + baseline["region"] = reform["region"] = regions + + impact = _compute(baseline, reform, levels=("national", "region")) + + assert impact.findings == [] + frame = impact.public.to_frame() + assert set(frame["area_code"]) == {"UK", "E12000001", "E12000007"} + + +@pytest.mark.parametrize( + "column, spelling", + [ + ("region", "London"), + ("region", "ENGLAND"), + ("region", b"Eastern"), + ("region_code_oa", "E12000010"), + ("region_code_oa", "E92000001"), + ], +) +def test_a_value_that_names_no_region_raises(column, spelling): + baseline, reform = _households() + values = list(baseline["region"]) + if column == "region_code_oa": + values = [GSS[name] for name in values] + values[0] = spelling + baseline[column] = reform[column] = values + + with pytest.raises( + ValueError, match=f"in {column} that name no UK region: 1" + ) as raised: + _compute(baseline, reform) + label = spelling.decode() if isinstance(spelling, bytes) else spelling + assert label not in str(raised.value) + + +def test_a_regional_release_is_a_prior_for_seats_from_another_dataset(): + """Regions from a national dataset first, then seats from a local file.""" + national = _households() + regional = _compute(*national).public.to_frame() + local_baseline, local_reform = _households() + codes = [MICROCOSM[name] for name in local_baseline["region"]] + local_baseline["region_code_oa"] = local_reform["region_code_oa"] = codes + seats = {"levels": ("constituency",)} + + prior = PriorPublic.from_frame("regional release", regional) + impact = _compute(local_baseline, local_reform, prior_public=[prior], **seats) + + assert impact.findings == [] + # The same table under the engine's names would match no region. + names = regional.replace({"area_code": {v: k for k, v in GSS.items()}}) + stale = PriorPublic.from_frame("regional release", names) + with pytest.raises(DisclosureError, match="name no region area of the release"): + _compute(local_baseline, local_reform, prior_public=[stale], **seats) def test_a_dataset_without_a_source_id_is_refused_unless_ids_are_passed(): @@ -197,7 +329,7 @@ def test_seats_and_local_authorities_together_need_an_acknowledgement(): authorities = [ "L1" if s in ("S1", "S3") else "L2" for s in baseline["constituency_code_oa"] ] - regions = ["R"] * len(authorities) + regions = ["LONDON"] * len(authorities) for frame in (baseline, reform): frame["la_code_oa"] = authorities frame["region"] = regions @@ -241,11 +373,15 @@ def test_a_weight_matrix_can_replace_the_area_codes(): with pytest.raises(ValueError, match="parent_of"): _compute(baseline, reform, **options) + # Regions are keyed on GSS codes, so the engine's names match none. + names = {"constituency": {"M1": "NORTH_EAST", "M2": "LONDON"}} + with pytest.raises(DisclosureError, match="not region areas: 2"): + _compute(baseline, reform, parent_of=names, **options) impact = _compute( baseline, reform, - parent_of={"constituency": {"M1": "NORTH_EAST", "M2": "LONDON"}}, + parent_of={"constituency": {"M1": "E12000001", "M2": "E12000007"}}, **options, ) assert set(impact.public.to_frame()["area_code"]) == {"M1", "M2"} From 4eb6dd531214c43bfaf317e8fd504d6584f5851a Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Fri, 16 Oct 2026 10:09:25 +0100 Subject: [PATCH 4/8] Add changelog fragments for #625 A fixed fragment for the release check, which now raises on parent_of codes and prior cells that match nothing instead of passing in silence, and a changed fragment for UK regions, which the adapter now keys on ONS GSS codes whatever the dataset stores. Part of #625. Co-Authored-By: Claude Opus 5.5 --- changelog.d/625.changed.md | 1 + changelog.d/625.fixed.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/625.changed.md create mode 100644 changelog.d/625.fixed.md diff --git a/changelog.d/625.changed.md b/changelog.d/625.changed.md new file mode 100644 index 00000000..2964c6d7 --- /dev/null +++ b/changelog.d/625.changed.md @@ -0,0 +1 @@ +`compute_uk_geography_disclosure` now keys regions on ONS GSS codes (`E12000001` to `E12000009` for England's regions, `W92000004`, `S92000003` and `N92000002` for Wales, Scotland and Northern Ireland), whether the household output carries `region_code_oa` or only the stored `region`, so releases from national and local datasets publish the same codes. Microcosm's pseudo codes for the nations map to the country codes, `UNKNOWN` is unassigned, and any other region value raises. Region codes in `parent_of` and in `prior_public` must be GSS codes; the table is `UK_REGION_GSS_CODES` in `policyengine.outputs.uk_geography_disclosure`. diff --git a/changelog.d/625.fixed.md b/changelog.d/625.fixed.md new file mode 100644 index 00000000..fdd93fcb --- /dev/null +++ b/changelog.d/625.fixed.md @@ -0,0 +1 @@ +The disclosure release check no longer passes in silence when area codes do not match. `Release.check()` now raises `DisclosureError` for a `parent_of` value that is not an area of the parent level, and for a `prior_public` cell whose area code is not an area of a level the release holds; `Release()` refuses a prior cell with an unknown outcome. Such codes used to leave the nesting and earlier-publication checks with nothing to check, so they returned no finding. Prior cells for levels, reforms or years the release does not hold are still accepted. The messages give counts, never codes or values. From 863634c572c8678d479471f18101113c2aab871b Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Fri, 16 Oct 2026 11:33:14 +0100 Subject: [PATCH 5/8] Match a prior cell's area against its own reform and year when held A prior cell at a held (level, reform, year) must name an area of those estimates; only a cell for a reform or year the release does not hold may name an area of any of the level's estimates. Before, an area that only another year held passed and then matched nothing. Co-Authored-By: Claude Opus 5.5 --- src/policyengine/disclosure/release.py | 20 ++++++++++++++++---- tests/test_disclosure_release.py | 26 ++++++++++++++++++++++++-- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/src/policyengine/disclosure/release.py b/src/policyengine/disclosure/release.py index d42c69a2..ce11fc6b 100644 --- a/src/policyengine/disclosure/release.py +++ b/src/policyengine/disclosure/release.py @@ -286,8 +286,10 @@ def _require_prior_areas(self) -> None: Prior cells match by code. A cell that matched nothing would be ignored, so a cell an earlier publication showed would still count as - hidden. A level the release holds is checked whatever the reform and - year of the cell; cells at other levels belong to other publications. + hidden. A cell at a level the release holds must name an area of the + estimates for its reform and year, or, when the release holds none, + of any of the level's estimates. Cells at other levels belong to other + publications. """ areas: dict[str, set[str]] = {} for (level, _, _), estimates in self._estimates.items(): @@ -295,8 +297,9 @@ def _require_prior_areas(self) -> None: for prior in self._prior: unmatched = [ level - for level, area, _, _, _ in prior.cells - if level in areas and area not in areas[level] + for level, area, reform, year, _ in prior.cells + if level in areas + and area not in self._prior_areas(level, reform, year, areas) ] if unmatched: levels = " or ".join(sorted(set(unmatched))) @@ -307,6 +310,15 @@ def _require_prior_areas(self) -> None: "estimates." ) + def _prior_areas( + self, level: str, reform: str, year: int, areas: dict[str, set[str]] + ) -> set[str]: + """The areas a prior cell at a held level may name.""" + estimates = self._estimates.get((level, reform, int(year))) + if estimates is None: + return areas[level] + return set(estimates.areas.area_codes) + def _known_cells(self) -> set[tuple[str, str, str, int, str]]: """Family cells an earlier publication already made public.""" known = set() diff --git a/tests/test_disclosure_release.py b/tests/test_disclosure_release.py index b0e25b0a..6c233812 100644 --- a/tests/test_disclosure_release.py +++ b/tests/test_disclosure_release.py @@ -60,13 +60,15 @@ def _records(affected, *, households=120, change=200.0, loose=0): } -def _estimates(data, level, *, reform="reform", change=None, settings=None, key=None): +def _estimates( + data, level, *, reform="reform", year=2026, change=None, settings=None, key=None +): codes = data["seats"] if level == "constituency" else data[f"{level}s"] delta = data["change"] if change is None else change return estimate_change( level=level, reform=reform, - year=2026, + year=year, areas=AreaWeights.from_labels(codes, data["weights"]), source_ids=data["source_ids"], baseline=data["baseline"], @@ -628,6 +630,26 @@ def test_a_prior_cell_must_name_an_area_of_a_level_the_release_holds(order): assert "North" not in str(raised.value) +def test_a_prior_cell_must_name_an_area_of_its_own_reform_and_year(): + """An area that only another year holds would match nothing.""" + data = _records(STRONG) + later = {**data, "seats": np.char.replace(data["seats"].astype(str), "S", "B")} + cells = [("constituency", "B1", "reform", 2026, "average_change")] + release = Release(prior_public=[PriorPublic(name="seat table", cells=cells)]) + release.add(_estimates(data, "constituency")) + release.add(_estimates(later, "constituency", year=2027)) + + with pytest.raises(DisclosureError, match="name no constituency area.*: 1"): + release.check() + + # For a year the release does not hold, any of the level's areas will do. + cells = [("constituency", "B1", "reform", 2028, "average_change")] + release = Release(prior_public=[PriorPublic(name="seat table", cells=cells)]) + release.add(_estimates(data, "constituency")) + release.add(_estimates(later, "constituency", year=2027)) + assert release.check() == [] + + def test_a_prior_cell_needs_a_known_outcome(): cells = [("constituency", "S1", "reform", 2026, "avg_change")] From d419e4641b21d7fd016d26d9bfcc696102ed5dbb Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Fri, 16 Oct 2026 11:33:14 +0100 Subject: [PATCH 6/8] Refuse prior cells at unknown UK levels, and skip empty area columns - compute_uk_geography_disclosure raises for a prior cell at a level that is neither a UK level nor one given in area_weights=. A release accepts such cells, since they may belong to other publications, so a misspelt level would leave a public cell counted as hidden. - A column that gives no household an area is skipped, so an empty region_code_oa falls back to the stored region. A requested level whose columns give no household an area raises a clear error instead of a KeyError from the check. Co-Authored-By: Claude Opus 5.5 --- .../outputs/uk_geography_disclosure.py | 45 +++++++++++++--- tests/test_uk_geography_disclosure.py | 53 +++++++++++++++++++ 2 files changed, 92 insertions(+), 6 deletions(-) diff --git a/src/policyengine/outputs/uk_geography_disclosure.py b/src/policyengine/outputs/uk_geography_disclosure.py index 64c77d2f..a4869f2c 100644 --- a/src/policyengine/outputs/uk_geography_disclosure.py +++ b/src/policyengine/outputs/uk_geography_disclosure.py @@ -111,6 +111,20 @@ def _uk_region_gss_codes(values: object, column: str) -> np.ndarray: return np.asarray(np.array([*codes, UNASSIGNED], dtype=object)[index]) +def _assigns_an_area(codes: np.ndarray) -> bool: + """Whether any household has an area code, rather than a blank.""" + values = pd.Series(codes, dtype=object) + present = values[values.notna()] + return bool( + present.map( + lambda code: ( + code.decode("utf-8") if isinstance(code, bytes) else str(code) + ).strip() + != UNASSIGNED + ).any() + ) + + class UKGeographyDisclosure(Output): """Income changes by UK area that are safe to publish. @@ -195,6 +209,21 @@ def run( weights = baseline_hh["household_weight"].to_numpy(dtype=float) custom = dict(area_weights or {}) parents = dict(parent_of or {}) + # A release ignores prior cells at levels it does not hold, since they + # may belong to other publications. Every UK level is known here, so a + # cell at any other level is a misspelling that would match nothing. + known = {UK_NATIONAL_LEVEL, *UK_DISCLOSURE_LEVEL_COLUMNS, *custom} + stray = [ + cell[0] + for prior in prior_public + for cell in prior.cells + if cell[0] not in known + ] + if stray: + raise ValueError( + f"Prior cells at levels that are not UK levels: {len(stray)}, " + f"at {sorted(set(stray))}. The levels are {sorted(known)}." + ) def areas_for(level: str) -> Optional[AreaWeights]: if level in custom: @@ -206,10 +235,14 @@ def areas_for(level: str) -> Optional[AreaWeights]: if columns is None: raise ValueError(f"Unknown level {level!r}") for column in columns: - if column in baseline_hh.columns: - codes = baseline_hh[column].to_numpy(dtype=object) - if level == UK_REGION_LEVEL: - codes = _uk_region_gss_codes(codes, column) + if column not in baseline_hh.columns: + continue + codes = baseline_hh[column].to_numpy(dtype=object) + if level == UK_REGION_LEVEL: + codes = _uk_region_gss_codes(codes, column) + # A column that gives no household an area, such as an empty + # region_code_oa, says nothing: read the next one. + if _assigns_an_area(codes): return AreaWeights.from_labels(codes, weights) return None @@ -224,8 +257,8 @@ def areas_for(level: str) -> Optional[AreaWeights]: if level in self.levels: columns = UK_DISCLOSURE_LEVEL_COLUMNS[level] raise ValueError( - f"The household output has none of {list(columns)} for " - f"level {level!r}" + f"The household output gives no household an area in " + f"any of {list(columns)} for level {level!r}" ) continue estimated[level] = areas diff --git a/tests/test_uk_geography_disclosure.py b/tests/test_uk_geography_disclosure.py index 743df122..d4245ab1 100644 --- a/tests/test_uk_geography_disclosure.py +++ b/tests/test_uk_geography_disclosure.py @@ -247,6 +247,59 @@ def test_a_regional_release_is_a_prior_for_seats_from_another_dataset(): _compute(local_baseline, local_reform, prior_public=[stale], **seats) +def test_a_prior_cell_at_a_level_that_is_not_a_uk_level_raises(): + """A misspelt level would match nothing, so the cell would count as hidden.""" + baseline, reform = _households({"S1": 4, "S2": 40, "S3": 80, "S4": 80}) + cells = [ + ("constituencies", "S1", "reform", 2026, "average_change"), + ("constituencies", "S1", "reform", 2026, "share_gaining"), + ("ward", "W1", "reform", 2026, "average_change"), + ] + prior = PriorPublic(name="seat table", cells=cells) + + with pytest.raises( + ValueError, match=r"not UK levels: 3, at \['constituencies', 'ward'\]" + ): + _compute(baseline, reform, levels=("constituency",), prior_public=[prior]) + + # A level given through area_weights is known. + n = len(baseline["household_id"]) + wards = AreaWeights.from_labels( + np.asarray(baseline["constituency_code_oa"], dtype=object), np.full(n, 100.0) + ) + ward = PriorPublic( + name="ward table", cells=[("ward", "S3", "reform", 2026, "average_change")] + ) + impact = _compute( + baseline, + reform, + levels=("ward",), + area_weights={"ward": wards}, + prior_public=[ward], + ) + assert impact.findings is not None + + +@pytest.mark.parametrize("blank", ["", " ", None]) +def test_an_empty_region_code_column_falls_back_to_the_stored_region(blank): + baseline, reform = _households() + empty = [blank] * len(baseline["region"]) + baseline["region_code_oa"] = reform["region_code_oa"] = empty + + impact = _compute(baseline, reform) + + assert set(impact.public.to_frame()["area_code"]) == {"E12000001", "E12000007"} + + +def test_a_level_whose_column_gives_no_household_an_area_raises(): + baseline, reform = _households() + empty = [""] * len(baseline["region"]) + baseline["constituency_code_oa"] = reform["constituency_code_oa"] = empty + + with pytest.raises(ValueError, match="no household an area"): + _compute(baseline, reform, levels=("constituency",)) + + def test_a_dataset_without_a_source_id_is_refused_unless_ids_are_passed(): baseline, reform = _households() ids = baseline.pop("source_household_id") From 79f6b3a8190f06c9b3c9e5378cc7204f87b1ccba Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Fri, 16 Oct 2026 11:33:14 +0100 Subject: [PATCH 7/8] Document the prior-cell rules and region weights; release as a minor The docs and changelog now say a prior on older boundaries raises even for a year the release does not hold, that the adapter refuses unknown levels, and that region weights passed with area_weights= must use GSS codes. An added fragment for UK_REGION_GSS_CODES makes the release a minor one, since region codes change and enum names now raise. Co-Authored-By: Claude Opus 5.5 --- changelog.d/625.added.md | 1 + changelog.d/625.changed.md | 2 +- changelog.d/625.fixed.md | 2 +- docs/disclosure.md | 28 ++++++++++++++++++---------- 4 files changed, 21 insertions(+), 12 deletions(-) create mode 100644 changelog.d/625.added.md diff --git a/changelog.d/625.added.md b/changelog.d/625.added.md new file mode 100644 index 00000000..a5a6803f --- /dev/null +++ b/changelog.d/625.added.md @@ -0,0 +1 @@ +Added `UK_REGION_GSS_CODES` in `policyengine.outputs.uk_geography_disclosure`: the ONS GSS code of each UK region, under every spelling a household output may carry (GSS codes, the engine's region enum names and Microcosm's pseudo codes for the nations). diff --git a/changelog.d/625.changed.md b/changelog.d/625.changed.md index 2964c6d7..4559d019 100644 --- a/changelog.d/625.changed.md +++ b/changelog.d/625.changed.md @@ -1 +1 @@ -`compute_uk_geography_disclosure` now keys regions on ONS GSS codes (`E12000001` to `E12000009` for England's regions, `W92000004`, `S92000003` and `N92000002` for Wales, Scotland and Northern Ireland), whether the household output carries `region_code_oa` or only the stored `region`, so releases from national and local datasets publish the same codes. Microcosm's pseudo codes for the nations map to the country codes, `UNKNOWN` is unassigned, and any other region value raises. Region codes in `parent_of` and in `prior_public` must be GSS codes; the table is `UK_REGION_GSS_CODES` in `policyengine.outputs.uk_geography_disclosure`. +`compute_uk_geography_disclosure` now keys regions on ONS GSS codes (`E12000001` to `E12000009` for England's regions, `W92000004`, `S92000003` and `N92000002` for Wales, Scotland and Northern Ireland), whether the household output carries `region_code_oa` or only the stored `region`, so releases from national and local datasets publish the same codes. Microcosm's pseudo codes for the nations map to the country codes, `UNKNOWN` is unassigned, and any other region value raises. Region codes in `parent_of`, in `prior_public` and in region weights passed with `area_weights=` must be GSS codes. Region rows from national datasets used to carry the engine's enum names, and enum names in `parent_of` or `prior_public` were accepted; they now raise. diff --git a/changelog.d/625.fixed.md b/changelog.d/625.fixed.md index fdd93fcb..6a67c05e 100644 --- a/changelog.d/625.fixed.md +++ b/changelog.d/625.fixed.md @@ -1 +1 @@ -The disclosure release check no longer passes in silence when area codes do not match. `Release.check()` now raises `DisclosureError` for a `parent_of` value that is not an area of the parent level, and for a `prior_public` cell whose area code is not an area of a level the release holds; `Release()` refuses a prior cell with an unknown outcome. Such codes used to leave the nesting and earlier-publication checks with nothing to check, so they returned no finding. Prior cells for levels, reforms or years the release does not hold are still accepted. The messages give counts, never codes or values. +The disclosure release check no longer passes in silence when area codes do not match. `Release.check()` now raises `DisclosureError` for a `parent_of` value that is not an area of the parent level, and for a `prior_public` cell at a level the release holds whose area code is not an area of that level: of the estimates for the cell's reform and year when the release holds them, and of any of the level's estimates otherwise. A prior table on older boundaries therefore raises even for a year the release does not hold, so filter those cells out first. `Release()` refuses a prior cell with an unknown outcome. Such codes used to leave the nesting and earlier-publication checks with nothing to check, so they returned no finding. Prior cells at levels the release does not hold are still accepted, but `compute_uk_geography_disclosure` raises for one at a level that is not a UK level, since that is a misspelling. The adapter also reads the stored `region` when `region_code_oa` gives no household a region, and raises a clear error when a requested level's column gives no household an area, instead of a `KeyError` from the check. The messages give counts, never codes or values. diff --git a/docs/disclosure.md b/docs/disclosure.md index 17ad9a77..b6af7dca 100644 --- a/docs/disclosure.md +++ b/docs/disclosure.md @@ -120,18 +120,24 @@ them like its own. The check matches areas by their codes, and it raises on a code that matches nothing rather than leave areas unchecked. Each value of `parent_of` must be an area code of the parent level's estimates. Each cell of `prior_public` at -a level the release holds must name an area of that level, whatever its -reform and year, and every cell needs one of the six outcomes. Cells at other -levels, or for other reforms or years, belong to other publications: the -release accepts them, but they take no part in the check. A prior cell counts -only when its level, area code, reform label and year are the release's. +a level the release holds must name an area of that level: of the estimates +for the cell's reform and year when the release holds them, and of any of the +level's estimates otherwise. So a prior table on older boundaries raises even +for a year the release does not hold; filter those cells out first. Every +cell needs one of the six outcomes. Cells at other levels belong to other +publications: the release accepts them, but they take no part in the check. +A prior cell counts only when its level, area code, reform label and year are +the release's. `compute_uk_geography_disclosure` always checks constituencies and local authorities against their region, and regions against the nation, also when you publish one level only. A level given as a weight matrix needs `parent_of=` with the GSS code of each area's region (see [UK region codes](#uk-region-codes)); without it the function raises, -because a check against the nation alone nearly always passes. +because a check against the nation alone nearly always passes. The function +also raises for a prior cell at a level that is not a UK level (`national`, +`region`, `constituency`, `local_authority`, or a level given in +`area_weights=`): a release would accept a misspelt level and check nothing. The check reports; it does not withhold cells by itself. A blocking finding names the group and suggests a cell. Withhold it, then check again. When a @@ -166,8 +172,9 @@ Two findings can be accepted after review, with `acknowledge=[...]`: `compute_uk_geography_disclosure` keys regions on their ONS GSS codes, whichever column the household output carries. It reads `region_code_oa` -when the output has it, as Microcosm's local files do, and the stored -`region` otherwise, as on the national datasets. Both give the same codes: +when the output has it and the column gives some household a region, as +Microcosm's local files do, and the stored `region` otherwise, as on the +national datasets. Both give the same codes: | Region | Stored `region` | GSS code | |---|---|---| @@ -191,8 +198,9 @@ country codes. A household whose `region` is `UNKNOWN` counts in the nation but in no region. Any other value raises. The table is `UK_REGION_GSS_CODES` in `policyengine.outputs.uk_geography_disclosure`. -Use the GSS codes wherever you name a region: in `parent_of`, and in the -region cells of `prior_public`. Names such as `London` or `Eastern` are not +Use the GSS codes wherever you name a region: in `parent_of`, in the region +cells of `prior_public`, and in region weights passed with `area_weights=`, +which the function uses as given. Names such as `London` or `Eastern` are not codes; map them first. Then a regional release from the national dataset and a seat release from a local file publish the same codes, and the regional table can be passed as `prior_public` to the seat release. From 5e5e647f84946d8fcba3c3bdc93ae6352dd9058c Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Fri, 16 Oct 2026 11:57:14 +0100 Subject: [PATCH 8/8] Write the empty-column check as a loop, which both ruff versions format alike The lambda's layout differed between the locked ruff and the one CI installs, so the format check failed in CI. Co-Authored-By: Claude Opus 5.5 --- .../outputs/uk_geography_disclosure.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/policyengine/outputs/uk_geography_disclosure.py b/src/policyengine/outputs/uk_geography_disclosure.py index a4869f2c..3e589f4a 100644 --- a/src/policyengine/outputs/uk_geography_disclosure.py +++ b/src/policyengine/outputs/uk_geography_disclosure.py @@ -114,15 +114,11 @@ def _uk_region_gss_codes(values: object, column: str) -> np.ndarray: def _assigns_an_area(codes: np.ndarray) -> bool: """Whether any household has an area code, rather than a blank.""" values = pd.Series(codes, dtype=object) - present = values[values.notna()] - return bool( - present.map( - lambda code: ( - code.decode("utf-8") if isinstance(code, bytes) else str(code) - ).strip() - != UNASSIGNED - ).any() - ) + for code in values[values.notna()].unique(): + text = code.decode("utf-8") if isinstance(code, bytes) else str(code) + if text.strip() != UNASSIGNED: + return True + return False class UKGeographyDisclosure(Output):