diff --git a/changelog.d/625.added.md b/changelog.d/625.added.md new file mode 100644 index 000000000..a5a6803f9 --- /dev/null +++ b/changelog.d/625.added.md @@ -0,0 +1 @@ +Added `UK_REGION_GSS_CODES` in `policyengine.outputs.uk_geography_disclosure`: the ONS GSS code of each UK region, under every spelling a household output may carry (GSS codes, the engine's region enum names and Microcosm's pseudo codes for the nations). diff --git a/changelog.d/625.changed.md b/changelog.d/625.changed.md new file mode 100644 index 000000000..4559d0190 --- /dev/null +++ b/changelog.d/625.changed.md @@ -0,0 +1 @@ +`compute_uk_geography_disclosure` now keys regions on ONS GSS codes (`E12000001` to `E12000009` for England's regions, `W92000004`, `S92000003` and `N92000002` for Wales, Scotland and Northern Ireland), whether the household output carries `region_code_oa` or only the stored `region`, so releases from national and local datasets publish the same codes. Microcosm's pseudo codes for the nations map to the country codes, `UNKNOWN` is unassigned, and any other region value raises. Region codes in `parent_of`, in `prior_public` and in region weights passed with `area_weights=` must be GSS codes. Region rows from national datasets used to carry the engine's enum names, and enum names in `parent_of` or `prior_public` were accepted; they now raise. diff --git a/changelog.d/625.fixed.md b/changelog.d/625.fixed.md new file mode 100644 index 000000000..6a67c05e5 --- /dev/null +++ b/changelog.d/625.fixed.md @@ -0,0 +1 @@ +The disclosure release check no longer passes in silence when area codes do not match. `Release.check()` now raises `DisclosureError` for a `parent_of` value that is not an area of the parent level, and for a `prior_public` cell at a level the release holds whose area code is not an area of that level: of the estimates for the cell's reform and year when the release holds them, and of any of the level's estimates otherwise. A prior table on older boundaries therefore raises even for a year the release does not hold, so filter those cells out first. `Release()` refuses a prior cell with an unknown outcome. Such codes used to leave the nesting and earlier-publication checks with nothing to check, so they returned no finding. Prior cells at levels the release does not hold are still accepted, but `compute_uk_geography_disclosure` raises for one at a level that is not a UK level, since that is a misspelling. The adapter also reads the stored `region` when `region_code_oa` gives no household a region, and raises a clear error when a requested level's column gives no household an area, instead of a `KeyError` from the check. The messages give counts, never codes or values. diff --git a/docs/disclosure.md b/docs/disclosure.md index 7cbffe84f..b6af7dca0 100644 --- a/docs/disclosure.md +++ b/docs/disclosure.md @@ -117,11 +117,27 @@ are a reference: the release does not emit their rows, but it treats the rows that pass as public elsewhere and the withheld ones as secret, and checks them like its own. +The check matches areas by their codes, and it raises on a code that matches +nothing rather than leave areas unchecked. Each value of `parent_of` must be +an area code of the parent level's estimates. Each cell of `prior_public` at +a level the release holds must name an area of that level: of the estimates +for the cell's reform and year when the release holds them, and of any of the +level's estimates otherwise. So a prior table on older boundaries raises even +for a year the release does not hold; filter those cells out first. Every +cell needs one of the six outcomes. Cells at other levels belong to other +publications: the release accepts them, but they take no part in the check. +A prior cell counts only when its level, area code, reform label and year are +the release's. + `compute_uk_geography_disclosure` always checks constituencies and local authorities against their region, and regions against the nation, also when you publish one level only. A level given as a weight matrix needs -`parent_of=` with the region of each area; without it the function raises, -because a check against the nation alone nearly always passes. +`parent_of=` with the GSS code of each area's region (see +[UK region codes](#uk-region-codes)); without it the function raises, +because a check against the nation alone nearly always passes. The function +also raises for a prior cell at a level that is not a UK level (`national`, +`region`, `constituency`, `local_authority`, or a level given in +`area_weights=`): a release would accept a misspelt level and check nothing. The check reports; it does not withhold cells by itself. A blocking finding names the group and suggests a cell. Withhold it, then check again. When a @@ -152,6 +168,43 @@ Two findings can be accepted after review, with `acknowledge=[...]`: - `noise_reference_public`: an earlier table shows the same cells under another noise setting. +## UK region codes + +`compute_uk_geography_disclosure` keys regions on their ONS GSS codes, +whichever column the household output carries. It reads `region_code_oa` +when the output has it and the column gives some household a region, as +Microcosm's local files do, and the stored `region` otherwise, as on the +national datasets. Both give the same codes: + +| Region | Stored `region` | GSS code | +|---|---|---| +| North East | `NORTH_EAST` | `E12000001` | +| North West | `NORTH_WEST` | `E12000002` | +| Yorkshire and the Humber | `YORKSHIRE` | `E12000003` | +| East Midlands | `EAST_MIDLANDS` | `E12000004` | +| West Midlands | `WEST_MIDLANDS` | `E12000005` | +| East of England | `EAST_OF_ENGLAND` | `E12000006` | +| London | `LONDON` | `E12000007` | +| South East | `SOUTH_EAST` | `E12000008` | +| South West | `SOUTH_WEST` | `E12000009` | +| Wales | `WALES` | `W92000004` | +| Scotland | `SCOTLAND` | `S92000003` | +| Northern Ireland | `NORTHERN_IRELAND` | `N92000002` | + +Wales, Scotland and Northern Ireland are one region each, under their country +code. Microcosm writes the pseudo codes `W99999999`, `S99999999` and +`N99999999` for them in `region_code_oa`, and the adapter reads those as the +country codes. A household whose `region` is `UNKNOWN` counts in the nation +but in no region. Any other value raises. The table is `UK_REGION_GSS_CODES` +in `policyengine.outputs.uk_geography_disclosure`. + +Use the GSS codes wherever you name a region: in `parent_of`, in the region +cells of `prior_public`, and in region weights passed with `area_weights=`, +which the function uses as given. Names such as `London` or `Eastern` are not +codes; map them first. Then a regional release from the national dataset and +a seat release from a local file publish the same codes, and the regional +table can be passed as `prior_public` to the seat release. + ## Noise Noise is off by default. With `DisclosureSettings(noise=True)`, each survey diff --git a/src/policyengine/disclosure/release.py b/src/policyengine/disclosure/release.py index 6e45895ce..ce11fc6be 100644 --- a/src/policyengine/disclosure/release.py +++ b/src/policyengine/disclosure/release.py @@ -27,6 +27,7 @@ FAMILY_LOSING, NOISE_REFERENCE_PUBLIC, OUTCOME_FAMILY, + OUTCOMES, PRIVATE_SECONDARY, REASON_DISCLOSURE_CONTROL, REASON_THIN_SUPPORT, @@ -71,6 +72,17 @@ def __init__( if unknown: raise ValueError(f"These findings cannot be acknowledged: {unknown}") self._prior = tuple(prior_public) + for prior in self._prior: + # A cell with an unknown outcome matches no cell of any release. + unknown_outcomes = sum( + outcome not in OUTCOMES for *_, outcome in prior.cells + ) + if unknown_outcomes: + raise DisclosureError( + f"Cells of prior publication {prior.name!r} whose outcome is " + f"unknown: {unknown_outcomes}. Outcomes are " + f"{', '.join(OUTCOMES)}." + ) self._estimates: dict[_Key, AreaEstimates] = {} self._published: dict[_Key, bool] = {} self._nestings: list[tuple[str, str, Optional[dict[str, str]]]] = [] @@ -107,6 +119,8 @@ def add_nesting( ``parent_of`` (child area code to parent area code) is needed only when the child level is a weight matrix and the parent has several areas. + Its parent codes must be area codes of the parent estimates: the check + raises on one that is not. """ mapping = dict(parent_of) if parent_of is not None else None self._nestings.append((child_level, parent_level, mapping)) @@ -144,6 +158,7 @@ def findings(self) -> Optional[list[Finding]]: def check(self) -> list[Finding]: """Report what a reader could recover from the rows as they stand.""" + self._require_prior_areas() tiers = { (key, family): self._family_tiers(key, family) for key in self._estimates @@ -266,6 +281,44 @@ def _effective_rows(self, key: _Key) -> Iterable[tuple[dict, str, Optional[str]] else: yield record, record["tier"], reason + def _require_prior_areas(self) -> None: + """Refuse prior cells that name no area of a level the release holds. + + Prior cells match by code. A cell that matched nothing would be + ignored, so a cell an earlier publication showed would still count as + hidden. A cell at a level the release holds must name an area of the + estimates for its reform and year, or, when the release holds none, + of any of the level's estimates. Cells at other levels belong to other + publications. + """ + areas: dict[str, set[str]] = {} + for (level, _, _), estimates in self._estimates.items(): + areas.setdefault(level, set()).update(estimates.areas.area_codes) + for prior in self._prior: + unmatched = [ + level + for level, area, reform, year, _ in prior.cells + if level in areas + and area not in self._prior_areas(level, reform, year, areas) + ] + if unmatched: + levels = " or ".join(sorted(set(unmatched))) + raise DisclosureError( + f"Cells of prior publication {prior.name!r} that name no " + f"{levels} area of the release: {len(unmatched)}. Prior " + "cells match by code: use the area codes of the release's " + "estimates." + ) + + def _prior_areas( + self, level: str, reform: str, year: int, areas: dict[str, set[str]] + ) -> set[str]: + """The areas a prior cell at a held level may name.""" + estimates = self._estimates.get((level, reform, int(year))) + if estimates is None: + return areas[level] + return set(estimates.areas.area_codes) + def _known_cells(self) -> set[tuple[str, str, str, int, str]]: """Family cells an earlier publication already made public.""" known = set() @@ -339,6 +392,15 @@ def _groups( raise DisclosureError( f"parent_of lacks {len(missing)} {child.level} areas." ) + # Groups match by code. A code that is no parent area would leave + # its child areas in no group, and the check would pass unseen. + unknown = {parent_of[c] for c in child.areas.area_codes} - set(parents) + if unknown: + raise DisclosureError( + f"Codes in parent_of that are not {parent.level} areas: " + f"{len(unknown)}. Map each {child.level} area to an area " + f"code of the {parent.level} estimates." + ) return { code: ( [c for c in child.areas.area_codes if parent_of[c] == code], diff --git a/src/policyengine/disclosure/schema.py b/src/policyengine/disclosure/schema.py index af8792cfb..e66d56cec 100644 --- a/src/policyengine/disclosure/schema.py +++ b/src/policyengine/disclosure/schema.py @@ -252,7 +252,11 @@ class Finding(BaseModel): class PriorPublic(BaseModel): - """Cells an earlier publication already made public.""" + """Cells an earlier publication already made public. + + Each cell is ``(level, area_code, reform, year, outcome)``. A release + matches cells by code, so they must use the area codes of its estimates. + """ model_config = ConfigDict(frozen=True, extra="forbid") diff --git a/src/policyengine/outputs/uk_geography_disclosure.py b/src/policyengine/outputs/uk_geography_disclosure.py index 511eacfe3..3e589f4a6 100644 --- a/src/policyengine/outputs/uk_geography_disclosure.py +++ b/src/policyengine/outputs/uk_geography_disclosure.py @@ -7,6 +7,7 @@ """ from collections.abc import Iterable, Mapping, Sequence +from enum import Enum from pathlib import Path from typing import TYPE_CHECKING, Optional, Union @@ -16,6 +17,7 @@ from policyengine.core import Output from policyengine.disclosure import ( + UNASSIGNED, AreaWeights, DisclosureSettings, Finding, @@ -39,6 +41,84 @@ "local_authority": ("la_code_oa",), } UK_SOURCE_HOUSEHOLD_ID_COLUMN = "source_household_id" +# The ONS GSS code of each region, by every spelling a household output may +# carry. Regions are always keyed on these codes, so a release from a national +# dataset and one from a local file publish the same codes, and publishers use +# them in parent_of and prior_public. England has nine regions; Wales, Scotland +# and Northern Ireland are one region each, under their country code. +UK_REGION_GSS_CODES: dict[str, str] = { + # GSS codes, as Microcosm writes them in region_code_oa. + "E12000001": "E12000001", + "E12000002": "E12000002", + "E12000003": "E12000003", + "E12000004": "E12000004", + "E12000005": "E12000005", + "E12000006": "E12000006", + "E12000007": "E12000007", + "E12000008": "E12000008", + "E12000009": "E12000009", + "W92000004": "W92000004", + "S92000003": "S92000003", + "N92000002": "N92000002", + # The pseudo codes Microcosm writes in region_code_oa for the nations. + "W99999999": "W92000004", + "S99999999": "S92000003", + "N99999999": "N92000002", + # The names of the engine's region enum, as the stored region holds them. + "NORTH_EAST": "E12000001", + "NORTH_WEST": "E12000002", + "YORKSHIRE": "E12000003", + "EAST_MIDLANDS": "E12000004", + "WEST_MIDLANDS": "E12000005", + "EAST_OF_ENGLAND": "E12000006", + "LONDON": "E12000007", + "SOUTH_EAST": "E12000008", + "SOUTH_WEST": "E12000009", + "WALES": "W92000004", + "SCOTLAND": "S92000003", + "NORTHERN_IRELAND": "N92000002", + "UNKNOWN": UNASSIGNED, +} + + +def _uk_region_gss_codes(values: object, column: str) -> np.ndarray: + """The GSS code of each region value, blank where it is unassigned. + + Values may be strings, bytes or members of the engine's region enum. A + missing value is unassigned. Any value outside the table raises: a region + under another spelling would match no code that a publisher passes. + """ + index, spellings = pd.factorize(pd.Series(values, dtype=object)) + codes: list[Optional[str]] = [] + unknown = 0 + for spelling in spellings: + if isinstance(spelling, Enum): + spelling = spelling.name + if isinstance(spelling, bytes): + spelling = spelling.decode("utf-8") + spelling = str(spelling).strip() + code = UK_REGION_GSS_CODES.get(spelling) if spelling else UNASSIGNED + if code is None: + unknown += 1 + codes.append(code) + if unknown: + raise ValueError( + f"Values in {column} that name no UK region: {unknown}. A region is " + "a GSS code, a name of the engine's region enum, or a Microcosm " + "pseudo code for a nation; see UK_REGION_GSS_CODES." + ) + # Factorize marks a missing value with -1, which picks the last entry. + return np.asarray(np.array([*codes, UNASSIGNED], dtype=object)[index]) + + +def _assigns_an_area(codes: np.ndarray) -> bool: + """Whether any household has an area code, rather than a blank.""" + values = pd.Series(codes, dtype=object) + for code in values[values.notna()].unique(): + text = code.decode("utf-8") if isinstance(code, bytes) else str(code) + if text.strip() != UNASSIGNED: + return True + return False class UKGeographyDisclosure(Output): @@ -47,6 +127,7 @@ class UKGeographyDisclosure(Output): Every number passes the disclosure rules of :class:`~policyengine.disclosure.DisclosureSettings` and carries a 90% interval. ``public`` stays empty while a blocking finding remains. + Regions are keyed on their ONS GSS codes; see ``UK_REGION_GSS_CODES``. """ model_config = ConfigDict(arbitrary_types_allowed=True) @@ -124,6 +205,21 @@ def run( weights = baseline_hh["household_weight"].to_numpy(dtype=float) custom = dict(area_weights or {}) parents = dict(parent_of or {}) + # A release ignores prior cells at levels it does not hold, since they + # may belong to other publications. Every UK level is known here, so a + # cell at any other level is a misspelling that would match nothing. + known = {UK_NATIONAL_LEVEL, *UK_DISCLOSURE_LEVEL_COLUMNS, *custom} + stray = [ + cell[0] + for prior in prior_public + for cell in prior.cells + if cell[0] not in known + ] + if stray: + raise ValueError( + f"Prior cells at levels that are not UK levels: {len(stray)}, " + f"at {sorted(set(stray))}. The levels are {sorted(known)}." + ) def areas_for(level: str) -> Optional[AreaWeights]: if level in custom: @@ -135,8 +231,15 @@ def areas_for(level: str) -> Optional[AreaWeights]: if columns is None: raise ValueError(f"Unknown level {level!r}") for column in columns: - if column in baseline_hh.columns: - return AreaWeights.from_labels(baseline_hh[column], weights) + if column not in baseline_hh.columns: + continue + codes = baseline_hh[column].to_numpy(dtype=object) + if level == UK_REGION_LEVEL: + codes = _uk_region_gss_codes(codes, column) + # A column that gives no household an area, such as an empty + # region_code_oa, says nothing: read the next one. + if _assigns_an_area(codes): + return AreaWeights.from_labels(codes, weights) return None settings = self.settings or DisclosureSettings() @@ -150,8 +253,8 @@ def areas_for(level: str) -> Optional[AreaWeights]: if level in self.levels: columns = UK_DISCLOSURE_LEVEL_COLUMNS[level] raise ValueError( - f"The household output has none of {list(columns)} for " - f"level {level!r}" + f"The household output gives no household an area in " + f"any of {list(columns)} for level {level!r}" ) continue estimated[level] = areas @@ -242,11 +345,13 @@ def compute_uk_geography_disclosure( the dataset has no ``source_household_id`` column. area_weights: Weights for a level, in place of the level's area code column, for example an area x household weight matrix. - parent_of: For a level given as a weight matrix, the region code of - each of its areas. Required for such a level when the household - output carries regions. + parent_of: For a level given as a weight matrix, the GSS code of the + region of each of its areas (see ``UK_REGION_GSS_CODES``). + Required for such a level when the household output carries + regions. acknowledge: Findings the caller accepts after review. prior_public: Earlier publications that count as already public. + Their region cells use GSS codes too. audit_dir: Private directory, outside any repository, for the exact values and counts. diff --git a/tests/test_disclosure_release.py b/tests/test_disclosure_release.py index d90cdbd58..6c2338120 100644 --- a/tests/test_disclosure_release.py +++ b/tests/test_disclosure_release.py @@ -60,13 +60,15 @@ def _records(affected, *, households=120, change=200.0, loose=0): } -def _estimates(data, level, *, reform="reform", change=None, settings=None, key=None): +def _estimates( + data, level, *, reform="reform", year=2026, change=None, settings=None, key=None +): codes = data["seats"] if level == "constituency" else data[f"{level}s"] delta = data["change"] if change is None else change return estimate_change( level=level, reform=reform, - year=2026, + year=year, areas=AreaWeights.from_labels(codes, data["weights"]), source_ids=data["source_ids"], baseline=data["baseline"], @@ -601,6 +603,77 @@ def test_a_prior_with_another_noise_setting_blocks_until_acknowledged(): assert [f.blocking for f in accepted.check()] == [False] +@pytest.mark.parametrize( + "order", [("region", "constituency"), ("constituency", "region")] +) +def test_a_prior_cell_must_name_an_area_of_a_level_the_release_holds(order): + """Regions published under other codes would still count as hidden.""" + prior = PriorPublic( + name="regional table", + cells=[ + ("region", "North", "reform", 2026, "average_change"), + ("region", "North", "reform", 2026, "share_gaining"), + ("region", "R2", "reform", 2026, "average_change"), + ], + ) + data = _records(STRONG) + release = Release(prior_public=[prior]) + for level in order: + # The regions are a reference, as in a seat release after a regional one. + release.add(_estimates(data, level), publish=level == "constituency") + release.add_nesting("constituency", "region") + + with pytest.raises( + DisclosureError, match=r"'regional table' that name no region area.*: 2\." + ) as raised: + release.check() + assert "North" not in str(raised.value) + + +def test_a_prior_cell_must_name_an_area_of_its_own_reform_and_year(): + """An area that only another year holds would match nothing.""" + data = _records(STRONG) + later = {**data, "seats": np.char.replace(data["seats"].astype(str), "S", "B")} + cells = [("constituency", "B1", "reform", 2026, "average_change")] + release = Release(prior_public=[PriorPublic(name="seat table", cells=cells)]) + release.add(_estimates(data, "constituency")) + release.add(_estimates(later, "constituency", year=2027)) + + with pytest.raises(DisclosureError, match="name no constituency area.*: 1"): + release.check() + + # For a year the release does not hold, any of the level's areas will do. + cells = [("constituency", "B1", "reform", 2028, "average_change")] + release = Release(prior_public=[PriorPublic(name="seat table", cells=cells)]) + release.add(_estimates(data, "constituency")) + release.add(_estimates(later, "constituency", year=2027)) + assert release.check() == [] + + +def test_a_prior_cell_needs_a_known_outcome(): + cells = [("constituency", "S1", "reform", 2026, "avg_change")] + + with pytest.raises(DisclosureError, match="'typo table' whose outcome is unknown"): + Release(prior_public=[PriorPublic(name="typo table", cells=cells)]) + + +def test_prior_cells_of_other_publications_are_allowed(): + """Cells for other reforms, years or levels take no part in the check.""" + cells = [ + ("constituency", "S1", "other reform", 2026, "average_change"), + ("constituency", "S1", "reform", 2025, "share_gaining"), + ("region", "R1", "reform", 2027, "share_no_change"), + ("local_authority", "L9", "reform", 2026, "average_change"), + ] + prior = PriorPublic(name="other tables", cells=cells) + affected = {**STRONG, "S1": 4, "S2": 30} + + findings = _release(affected, prior_public=[prior]).check() + + assert findings == _release(affected).check() + assert _blocking(findings, RECOVERABLE_REMAINDER) + + def test_a_weight_matrix_child_needs_parent_of_under_several_parents(): data = _records(STRONG) n = len(data["seats"]) @@ -629,6 +702,70 @@ def test_a_weight_matrix_child_needs_parent_of_under_several_parents(): assert release.check() == [] +def _seat_matrix_release(parent_of): + """400 records in two regions and a 4 x 400 seat matrix. + + Three records carry seat M1, so it is withheld. M2 holds the rest of R1; + M3 and M4 share R2, and a few records there carry weight in both. + """ + n = 400 + regions = np.array(["R1"] * 200 + ["R2"] * 200, dtype=object) + matrix = np.zeros((4, n)) + matrix[0, :3] = 100.0 + matrix[1, 3:200] = 100.0 + matrix[2, 200:300] = 100.0 + matrix[3, 300:] = 100.0 + matrix[2:, 290:300] = 50.0 + baseline = np.random.default_rng(2).uniform(20_000.0, 60_000.0, n) + common = { + "reform": "reform", + "year": 2026, + "source_ids": np.arange(n), + "baseline": baseline, + "reformed": baseline + 200.0, + } + release = Release() + release.add( + estimate_change( + level="region", + areas=AreaWeights.from_labels(regions, np.full(n, 100.0)), + **common, + ) + ) + release.add( + estimate_change( + level="constituency", + areas=AreaWeights.from_matrix(matrix, ["M1", "M2", "M3", "M4"]), + **common, + ) + ) + release.add_nesting("constituency", "region", parent_of) + return release + + +def test_parent_of_must_name_areas_of_the_parent_level(): + """Regions named another way left every group empty, so nothing was found.""" + own = {"M1": "R1", "M2": "R1", "M3": "R2", "M4": "R2"} + + findings = _seat_matrix_release(own).check() + + blocking = _blocking(findings, RECOVERABLE_REMAINDER) + assert len(blocking) == len(findings) == 3 + assert {f.family for f in blocking} == {"change", "share_gaining", "share_losing"} + assert {(f.relation, f.group, tuple(f.members)) for f in blocking} == { + ("nesting", "R1", ("M1",)) + } + + renamed = {"M1": "North", "M2": "North", "M3": "South", "M4": "South"} + with pytest.raises(DisclosureError, match="not region areas: 2") as raised: + _seat_matrix_release(renamed).check() + assert "North" not in str(raised.value) + assert "South" not in str(raised.value) + + with pytest.raises(DisclosureError, match="not region areas: 1"): + _seat_matrix_release({**own, "M4": "South"}).check() + + def test_the_release_refuses_mixed_settings_and_duplicates(): data = _records(STRONG) release = Release() diff --git a/tests/test_uk_geography_disclosure.py b/tests/test_uk_geography_disclosure.py index 322163b95..d4245ab10 100644 --- a/tests/test_uk_geography_disclosure.py +++ b/tests/test_uk_geography_disclosure.py @@ -3,6 +3,7 @@ All data here is synthetic. """ +from enum import Enum from unittest.mock import MagicMock import numpy as np @@ -11,22 +12,55 @@ from microdf import MicroDataFrame from policyengine import outputs +from policyengine.countries.uk.regions import ENGLISH_REGIONS from policyengine.disclosure import ( NOISE_KEY_ENV, AreaWeights, DisclosureError, DisclosureKeyError, DisclosureSettings, + PriorPublic, PublicTable, validate_public, ) from policyengine.outputs.uk_geography_disclosure import ( + UK_REGION_GSS_CODES, UKGeographyDisclosure, compute_uk_geography_disclosure, ) TEST_KEY = "test-key-for-unit-tests-only" SEATS = {"S1": "NORTH_EAST", "S2": "NORTH_EAST", "S3": "LONDON", "S4": "LONDON"} +NATIONS = { + "S1": "NORTH_EAST", + "S2": "WALES", + "S3": "SCOTLAND", + "S4": "NORTHERN_IRELAND", +} +GSS = { + "NORTH_EAST": "E12000001", + "LONDON": "E12000007", + "WALES": "W92000004", + "SCOTLAND": "S92000003", + "NORTHERN_IRELAND": "N92000002", +} +# What Microcosm writes in region_code_oa: GSS codes, pseudo codes for nations. +MICROCOSM = { + **GSS, + "WALES": "W99999999", + "SCOTLAND": "S99999999", + "NORTHERN_IRELAND": "N99999999", +} + + +class Region(Enum): + """Shaped like the engine's region enum: stored names, display values.""" + + NORTH_EAST = "North East" + WALES = "Wales" + SCOTLAND = "Scotland" + NORTHERN_IRELAND = "Northern Ireland" + UNKNOWN = "Unknown" def _make_sim(household_data: dict) -> MagicMock: @@ -38,11 +72,11 @@ def _make_sim(household_data: dict) -> MagicMock: return sim -def _households(affected=None, *, per_seat=150): - affected = affected or dict.fromkeys(SEATS, 80) +def _households(affected=None, *, per_seat=150, region_of=SEATS): + affected = affected or dict.fromkeys(region_of, 80) rng = np.random.default_rng(0) seats, regions, deltas = [], [], [] - for seat, region in SEATS.items(): + for seat, region in region_of.items(): seats += [seat] * per_seat regions += [region] * per_seat deltas += list(rng.uniform(100.0, 900.0, affected[seat])) @@ -78,7 +112,7 @@ def test_regions_are_published_with_intervals_and_pass_the_validator(): assert isinstance(impact, UKGeographyDisclosure) assert impact.findings == [] assert set(frame["level"]) == {"region"} - assert set(frame["area_code"]) == {"NORTH_EAST", "LONDON"} + assert set(frame["area_code"]) == {"E12000001", "E12000007"} averages = frame[frame["outcome"] == "average_change"] assert (averages["tier"] == "reported").all() assert (averages["lo"] <= averages["value"]).all() @@ -102,19 +136,170 @@ def test_public_rows_round_trip_through_json(): } +def test_the_region_table_maps_every_stored_region_to_its_gss_code(): + stored = (*ENGLISH_REGIONS, "WALES", "SCOTLAND", "NORTHERN_IRELAND") + codes = [f"E1200000{number}" for number in range(1, 10)] + codes += ["W92000004", "S92000003", "N92000002"] + + assert [UK_REGION_GSS_CODES[name] for name in stored] == codes + assert [UK_REGION_GSS_CODES[code] for code in codes] == codes + assert [ + UK_REGION_GSS_CODES[code] for code in ("W99999999", "S99999999", "N99999999") + ] == codes[-3:] + assert UK_REGION_GSS_CODES["UNKNOWN"] == "" + + +def test_regions_are_keyed_on_gss_codes_whatever_the_dataset_stores(): + stored = { + "enum names": ("region", str), + "bytes": ("region", str.encode), + "enum members": ("region", Region.__getitem__), + "GSS codes": ("region_code_oa", GSS.__getitem__), + "Microcosm codes": ("region_code_oa", MICROCOSM.__getitem__), + } + tables = {} + for name, (column, spell) in stored.items(): + baseline, reform = _households(region_of=NATIONS) + values = [spell(region) for region in baseline.pop("region")] + reform.pop("region") + baseline[column] = reform[column] = values + tables[name] = _compute(baseline, reform).public.to_frame() + + assert set(tables["enum names"]["area_code"]) == { + "E12000001", + "W92000004", + "S92000003", + "N92000002", + } + for frame in tables.values(): + pd.testing.assert_frame_equal(frame, tables["enum names"]) + + def test_region_codes_are_preferred_over_the_stored_region(): baseline, reform = _households() codes = [ - "E12000001" if r == "NORTH_EAST" else "E12000007" for r in baseline["region"] + "E12000002" if r == "NORTH_EAST" else "E12000008" for r in baseline["region"] ] baseline["region_code_oa"] = codes reform["region_code_oa"] = codes impact = _compute(baseline, reform) + assert set(impact.public.to_frame()["area_code"]) == {"E12000002", "E12000008"} + + +def test_an_unknown_region_is_in_the_nation_but_in_no_region(): + baseline, reform = _households() + # The last households of S4 do not change. + regions = [*baseline["region"][:-30], *[Region.UNKNOWN] * 30] + baseline["region"] = reform["region"] = regions + + impact = _compute(baseline, reform, levels=("national", "region")) + + assert impact.findings == [] + frame = impact.public.to_frame() + assert set(frame["area_code"]) == {"UK", "E12000001", "E12000007"} + + +@pytest.mark.parametrize( + "column, spelling", + [ + ("region", "London"), + ("region", "ENGLAND"), + ("region", b"Eastern"), + ("region_code_oa", "E12000010"), + ("region_code_oa", "E92000001"), + ], +) +def test_a_value_that_names_no_region_raises(column, spelling): + baseline, reform = _households() + values = list(baseline["region"]) + if column == "region_code_oa": + values = [GSS[name] for name in values] + values[0] = spelling + baseline[column] = reform[column] = values + + with pytest.raises( + ValueError, match=f"in {column} that name no UK region: 1" + ) as raised: + _compute(baseline, reform) + label = spelling.decode() if isinstance(spelling, bytes) else spelling + assert label not in str(raised.value) + + +def test_a_regional_release_is_a_prior_for_seats_from_another_dataset(): + """Regions from a national dataset first, then seats from a local file.""" + national = _households() + regional = _compute(*national).public.to_frame() + local_baseline, local_reform = _households() + codes = [MICROCOSM[name] for name in local_baseline["region"]] + local_baseline["region_code_oa"] = local_reform["region_code_oa"] = codes + seats = {"levels": ("constituency",)} + + prior = PriorPublic.from_frame("regional release", regional) + impact = _compute(local_baseline, local_reform, prior_public=[prior], **seats) + + assert impact.findings == [] + # The same table under the engine's names would match no region. + names = regional.replace({"area_code": {v: k for k, v in GSS.items()}}) + stale = PriorPublic.from_frame("regional release", names) + with pytest.raises(DisclosureError, match="name no region area of the release"): + _compute(local_baseline, local_reform, prior_public=[stale], **seats) + + +def test_a_prior_cell_at_a_level_that_is_not_a_uk_level_raises(): + """A misspelt level would match nothing, so the cell would count as hidden.""" + baseline, reform = _households({"S1": 4, "S2": 40, "S3": 80, "S4": 80}) + cells = [ + ("constituencies", "S1", "reform", 2026, "average_change"), + ("constituencies", "S1", "reform", 2026, "share_gaining"), + ("ward", "W1", "reform", 2026, "average_change"), + ] + prior = PriorPublic(name="seat table", cells=cells) + + with pytest.raises( + ValueError, match=r"not UK levels: 3, at \['constituencies', 'ward'\]" + ): + _compute(baseline, reform, levels=("constituency",), prior_public=[prior]) + + # A level given through area_weights is known. + n = len(baseline["household_id"]) + wards = AreaWeights.from_labels( + np.asarray(baseline["constituency_code_oa"], dtype=object), np.full(n, 100.0) + ) + ward = PriorPublic( + name="ward table", cells=[("ward", "S3", "reform", 2026, "average_change")] + ) + impact = _compute( + baseline, + reform, + levels=("ward",), + area_weights={"ward": wards}, + prior_public=[ward], + ) + assert impact.findings is not None + + +@pytest.mark.parametrize("blank", ["", " ", None]) +def test_an_empty_region_code_column_falls_back_to_the_stored_region(blank): + baseline, reform = _households() + empty = [blank] * len(baseline["region"]) + baseline["region_code_oa"] = reform["region_code_oa"] = empty + + impact = _compute(baseline, reform) + assert set(impact.public.to_frame()["area_code"]) == {"E12000001", "E12000007"} +def test_a_level_whose_column_gives_no_household_an_area_raises(): + baseline, reform = _households() + empty = [""] * len(baseline["region"]) + baseline["constituency_code_oa"] = reform["constituency_code_oa"] = empty + + with pytest.raises(ValueError, match="no household an area"): + _compute(baseline, reform, levels=("constituency",)) + + def test_a_dataset_without_a_source_id_is_refused_unless_ids_are_passed(): baseline, reform = _households() ids = baseline.pop("source_household_id") @@ -197,7 +382,7 @@ def test_seats_and_local_authorities_together_need_an_acknowledgement(): authorities = [ "L1" if s in ("S1", "S3") else "L2" for s in baseline["constituency_code_oa"] ] - regions = ["R"] * len(authorities) + regions = ["LONDON"] * len(authorities) for frame in (baseline, reform): frame["la_code_oa"] = authorities frame["region"] = regions @@ -241,11 +426,15 @@ def test_a_weight_matrix_can_replace_the_area_codes(): with pytest.raises(ValueError, match="parent_of"): _compute(baseline, reform, **options) + # Regions are keyed on GSS codes, so the engine's names match none. + names = {"constituency": {"M1": "NORTH_EAST", "M2": "LONDON"}} + with pytest.raises(DisclosureError, match="not region areas: 2"): + _compute(baseline, reform, parent_of=names, **options) impact = _compute( baseline, reform, - parent_of={"constituency": {"M1": "NORTH_EAST", "M2": "LONDON"}}, + parent_of={"constituency": {"M1": "E12000001", "M2": "E12000007"}}, **options, ) assert set(impact.public.to_frame()["area_code"]) == {"M1", "M2"}