From bf12d2d890e243a467e970a5dc138df6a9610726 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 21 Oct 2026 03:33:31 +0400 Subject: [PATCH 1/5] Test that each US geography has one label across target sources These tests fail on main (8 failed, 80 passed across the four files): - the ACS local release's population targets label a state with its postal code ("ME") where the ledger targets use the Census name ("Maine"), and label at-large districts differently from the IRS SOI district rows for the same id; - the IRS SOI district rows translated to the 119th Congress plan are not labelled with the Census Bureau's district names, which Chronicle's ACS district facts carry for the same ids; - microcosm.calibrate.geography_constants has no US state FIPS-to-name table; - a US target registry accepts a geography id with two labels, and a registry for another country does not report one; - the ACS local materialize stage reads the staging frame before it checks the admin and population target labels together. Refs #1335 Co-Authored-By: Claude Opus 5.5 --- .../us/test_us_acs_local_geography_labels.py | 256 ++++++++++++++++++ .../test_us_congressional_district_vintage.py | 6 +- .../engine_free/shared/test_diagnostics.py | 62 +++++ .../tests/engine_free/shared/test_registry.py | 79 ++++++ 4 files changed, 401 insertions(+), 2 deletions(-) create mode 100644 packages/microcosm-build/tests/engine_free/us/test_us_acs_local_geography_labels.py diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_geography_labels.py b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_geography_labels.py new file mode 100644 index 0000000000..cf44001000 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_geography_labels.py @@ -0,0 +1,256 @@ +"""The ACS local release gives each geography the label its other sources use. + +The release adds Census population targets for every state and congressional +district to the targets it compiles from the Ledger. A ledger target takes its +state label from the source facts' geography name, the state's Census name +("Maine"). An IRS SOI district row takes its label from the +congressional-district vintage translation, and Chronicle's ACS district facts +carry the Census Bureau's district names. The population targets must label +the same ids the same way, or the release's calibration diagnostics file gives +one geography two labels: populace-us-2024-buildo-acs-local-766b7c7 labelled +each state with its name on 84 targets and with its postal code on its +population target (microcosm#1335). +""" + +from __future__ import annotations + +import importlib.util +from types import SimpleNamespace + +import pytest + +from microcosm.build.us_runtime import ( + SOI_CONGRESSIONAL_DISTRICT_RECORD_SET_ID, + translate_congressional_district_facts_to_current_vintage, +) +from microcosm.calibrate import ( + CalibrationHierarchy, + HierarchyCategory, + HierarchyGeography, + HierarchyNode, + TargetRegistry, + TargetSpec, + geography_constants, +) +from test_support.paths import paths_for + +_TEST_PATHS = paths_for("microcosm-build") + + +def _load_tool_module(): + path = _TEST_PATHS.repository / "tools" / "build_us_acs_local_release.py" + spec = importlib.util.spec_from_file_location( + "build_us_acs_local_release_geography_labels", path + ) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def _ledger_state_spec(name: str, state_fips: str, state_label: str) -> TargetSpec: + """A state target as the ledger compiles it: labelled by the facts' name.""" + + return TargetSpec( + name=name, + entity="household", + measure=name, + value=1.0, + period=2024, + family="usda_snap", + source="fixture ledger feed", + metadata={"state_fips": state_fips, "ledger_geography_level": "state"}, + hierarchy=CalibrationHierarchy( + provider=HierarchyNode(id="usda_snap", label="USDA SNAP"), + category=HierarchyCategory( + id="usda_snap.usda_snap", label="SNAP", provider_id="usda_snap" + ), + geography=HierarchyGeography( + id=f"0400000US{state_fips}", label=state_label, level="state" + ), + dimensions=(), + target=HierarchyNode(id=name, label=f"{state_label} SNAP households"), + ), + ) + + +def _soi_district_fact(district_geoid: str) -> dict[str, object]: + """An IRS SOI district fact on the 117th Congress plan.""" + + source_record_id = ( + f"{SOI_CONGRESSIONAL_DISTRICT_RECORD_SET_ID}.{district_geoid}.return_count" + ) + return { + "label": f"District {district_geoid} returns", + "aggregate_fact_key": f"ledger.aggregate_fact.v2:{district_geoid}", + "lineage": {"source_record_id": source_record_id}, + "value": 1_000.0, + "period": {"type": "tax_year", "value": 2022}, + "geography": { + "level": "congressional_district", + "id": f"5001700US{district_geoid}", + "name": f"District {district_geoid}", + "vintage": "117th_congress", + }, + "entity": {"name": "tax_unit"}, + "aggregation": {"method": "sum"}, + "layout": { + "record_set_id": SOI_CONGRESSIONAL_DISTRICT_RECORD_SET_ID, + "groupby_dimension": "irs_soi.congressional_district", + "groupby_value_id": district_geoid, + "measure_id": "return_count", + }, + "observed_measure": { + "source_name": "irs_soi", + "source_measure_id": "return_count", + }, + "source": {"source_name": "irs_soi"}, + } + + +def _geography_labels(specs) -> dict[str, str]: + return { + spec.hierarchy.geography.id: spec.hierarchy.geography.label for spec in specs + } + + +def test_population_state_targets_carry_the_census_state_name() -> None: + module = _load_tool_module() + names = [ + f"pop_state_{state_fips}" + for state_fips in geography_constants.US_STATE_FIPS_TO_POSTAL + ] + + specs = module.population_target_specs(names, [1.0e6] * len(names)) + + labels = _geography_labels(specs) + assert labels["0400000US23"] == "Maine" + assert labels["0400000US11"] == "District of Columbia" + assert labels == { + f"0400000US{state_fips}": state_name + for state_fips, state_name in geography_constants.US_STATE_FIPS_TO_NAME.items() + } + (maine,) = [spec for spec in specs if spec.name == "pop_state_23"] + assert maine.hierarchy.target.label == "Maine resident population" + + +def test_population_and_ledger_targets_give_a_state_one_label() -> None: + module = _load_tool_module() + ledger_spec = _ledger_state_spec( + "usda_snap.fy2024.state_average_monthly_households.nero.me." + "average_monthly_households", + "23", + "Maine", + ) + + population_specs = module.population_target_specs(["pop_state_23"], [1.4e6]) + registry = TargetRegistry((ledger_spec, *population_specs), country="us") + + assert {spec.hierarchy.geography.label for spec in registry} == {"Maine"} + + +def test_population_and_soi_district_rows_give_a_district_one_label() -> None: + """Numbered, at-large and delegate districts, as the Census Bureau names them. + + Chronicle's ACS S0101 and S2201 district facts carry these names for the + same 119th Congress ids, so a feed with those facts agrees too. + """ + + module = _load_tool_module() + district_geoids = ("0601", "0200", "1100") + translated = translate_congressional_district_facts_to_current_vintage( + [_soi_district_fact(district_geoid) for district_geoid in district_geoids], + [ + { + "source_geography_id": f"5001700US{district_geoid}", + "target_geography_id": f"5001900US{district_geoid}", + "weight": 1.0, + } + for district_geoid in district_geoids + ], + ) + soi_labels = { + fact["geography"]["id"]: fact["geography"]["name"] for fact in translated + } + + population_specs = module.population_target_specs( + [f"pop_cd_{district_geoid}" for district_geoid in district_geoids], + [7.0e5] * len(district_geoids), + ) + + assert _geography_labels(population_specs) == soi_labels + assert soi_labels == { + "5001900US0601": "Congressional District 1 (119th Congress), California", + "5001900US0200": "Congressional District (at Large) (119th Congress), Alaska", + "5001900US1100": ( + "Delegate District (at Large) (119th Congress), District of Columbia" + ), + } + + +def test_materialize_refuses_a_geography_label_conflict_before_the_engine_pass( + monkeypatch, tmp_path +) -> None: + """A feed that names a state another way is refused at the stage's start. + + The admin targets and the ladder's population targets first share a + registry in the lean checkpoint, after the engine pass over every staged + household. The materialize stage checks their labels together before it + reads the staging frame. + """ + + module = _load_tool_module() + admin_registry = TargetRegistry( + [_ledger_state_spec("usda_snap.me.households", "23", "State of Maine")], + country="us", + ) + monkeypatch.setattr( + module, + "state_admin_surface", + lambda feed, families, soi_mode: module.AdminSurface( + admin_registry, (), {"soi_mode": soi_mode} + ), + ) + monkeypatch.setattr( + module, + "ladder_population", + lambda ladder_path, geographies: { + "state": {23: 1_400_000.0}, + "cd": {2301: 700_000.0, 2302: 700_000.0}, + }, + ) + + def engine_pass_started(*_args, **_kwargs): + raise AssertionError( + "the staging frame was read before the geography labels were checked" + ) + + monkeypatch.setattr(module, "_load_staging_frame", engine_pass_started) + monkeypatch.setattr(module, "materialize_chunked", engine_pass_started) + staging_h5 = tmp_path / "staging.h5" + staging_h5.write_bytes(b"staging") + (tmp_path / "staging.summary.json").write_text("{}") + ladder_path = tmp_path / "ladder.npz" + ladder_path.write_bytes(b"ladder") + args = SimpleNamespace( + families="snap,medicaid,soi", + geographies="state,cd", + soi_mode="state", + cd_holdout_fraction=None, + feed=tmp_path / "feed.jsonl", + feed_sha256=None, + staging_h5=staging_h5, + staging_summary=None, + ladder=ladder_path, + checkpoint_dir=tmp_path / "checkpoint", + ) + + with pytest.raises( + ValueError, + match=( + r"'0400000US23' is 'State of Maine' \(first on target " + r"'usda_snap\.me\.households'\) and 'Maine' \(first on target " + r"'pop_state_23'\)" + ), + ): + module.do_materialize(args) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_congressional_district_vintage.py b/packages/microcosm-build/tests/engine_free/us/test_us_congressional_district_vintage.py index 6b3b2b05d3..3107d1587a 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_congressional_district_vintage.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_congressional_district_vintage.py @@ -57,11 +57,13 @@ def test_translate_cd_facts_conserves_values_and_records_lineage() -> None: assert set(by_geography) == {"5001900US0601", "5001900US0602"} assert by_geography["5001900US0601"]["value"] == pytest.approx(75.0) assert by_geography["5001900US0602"]["value"] == pytest.approx(85.0) + # The Census Bureau's district names, which the ACS local release's + # population targets for the same ids also carry (microcosm#1335). assert by_geography["5001900US0601"]["geography"]["name"] == ( - "CA congressional district 1" + "Congressional District 1 (119th Congress), California" ) assert by_geography["5001900US0602"]["layout"]["groupby_value_label"] == ( - "CA congressional district 2" + "Congressional District 2 (119th Congress), California" ) merged_lineage = by_geography["5001900US0602"]["lineage"] assert merged_lineage["target_geography_vintage"] == "119th_congress" diff --git a/packages/microcosm-calibrate/tests/engine_free/shared/test_diagnostics.py b/packages/microcosm-calibrate/tests/engine_free/shared/test_diagnostics.py index fff1c41741..8e289cfa33 100644 --- a/packages/microcosm-calibrate/tests/engine_free/shared/test_diagnostics.py +++ b/packages/microcosm-calibrate/tests/engine_free/shared/test_diagnostics.py @@ -34,6 +34,7 @@ calibrate, calibrate_l0_refit, diagnostics_payload, + geography_constants, past_cap_census, score_targets, write_calibration_diagnostics, @@ -94,6 +95,67 @@ def test_shared_geography_constant_views_are_consistent() -> None: } +def test_us_state_names_are_the_names_the_state_facts_carry() -> None: + # Every state label the ledger targets of the US release + # populace-us-2024-buildo-acs-local-766b7c7 carry, for the 50 states and + # DC: the Census names Chronicle's state facts use (microcosm#1335). + names = geography_constants.US_STATE_FIPS_TO_NAME + assert list(names) == list(US_STATE_FIPS_TO_POSTAL) + assert dict(names) == { + "01": "Alabama", + "02": "Alaska", + "04": "Arizona", + "05": "Arkansas", + "06": "California", + "08": "Colorado", + "09": "Connecticut", + "10": "Delaware", + "11": "District of Columbia", + "12": "Florida", + "13": "Georgia", + "15": "Hawaii", + "16": "Idaho", + "17": "Illinois", + "18": "Indiana", + "19": "Iowa", + "20": "Kansas", + "21": "Kentucky", + "22": "Louisiana", + "23": "Maine", + "24": "Maryland", + "25": "Massachusetts", + "26": "Michigan", + "27": "Minnesota", + "28": "Mississippi", + "29": "Missouri", + "30": "Montana", + "31": "Nebraska", + "32": "Nevada", + "33": "New Hampshire", + "34": "New Jersey", + "35": "New Mexico", + "36": "New York", + "37": "North Carolina", + "38": "North Dakota", + "39": "Ohio", + "40": "Oklahoma", + "41": "Oregon", + "42": "Pennsylvania", + "44": "Rhode Island", + "45": "South Carolina", + "46": "South Dakota", + "47": "Tennessee", + "48": "Texas", + "49": "Utah", + "50": "Vermont", + "51": "Virginia", + "53": "Washington", + "54": "West Virginia", + "55": "Wisconsin", + "56": "Wyoming", + } + + def _hierarchy( name: str, *, diff --git a/packages/microcosm-calibrate/tests/engine_free/shared/test_registry.py b/packages/microcosm-calibrate/tests/engine_free/shared/test_registry.py index f4b6c846d9..d36a2821ae 100644 --- a/packages/microcosm-calibrate/tests/engine_free/shared/test_registry.py +++ b/packages/microcosm-calibrate/tests/engine_free/shared/test_registry.py @@ -156,6 +156,85 @@ def test_families_and_select(self) -> None: assert len(soi) == 1 and next(iter(soi)).name == "irs_soi/agi" +def _geography_spec( + name: str, geography_id: str, label: str, *, level: str = "state" +) -> TargetSpec: + return _spec( + name=name, + hierarchy=CalibrationHierarchy( + provider=HierarchyNode(id="census", label="US Census Bureau"), + category=HierarchyCategory( + id="population", label="Population", provider_id="census" + ), + geography=HierarchyGeography(id=geography_id, label=label, level=level), + dimensions=(), + target=HierarchyNode(id=name, label=f"{label} population"), + ), + ) + + +class TestGeographyLabels: + """One geography id has one label across a registry (microcosm#1335).""" + + def test_a_us_registry_refuses_one_geography_id_with_two_labels(self) -> None: + specs = [ + _geography_spec("usda_snap.me.households", "0400000US23", "Maine"), + _geography_spec("pop_state_23", "0400000US23", "ME"), + ] + + with pytest.raises( + ValueError, + match=( + r"'0400000US23' is 'Maine' \(first on target " + r"'usda_snap\.me\.households'\) and 'ME' \(first on target " + r"'pop_state_23'\)" + ), + ): + TargetRegistry(specs, country="us") + + def test_a_us_registry_accepts_one_label_per_geography_id(self) -> None: + specs = [ + _geography_spec("usda_snap.me.households", "0400000US23", "Maine"), + _geography_spec("pop_state_23", "0400000US23", "Maine"), + _geography_spec("pop_state_33", "0400000US33", "New Hampshire"), + _spec(name="census/unlabelled"), + ] + + registry = TargetRegistry(specs, country="us") + + assert len(registry) == 4 + assert len(registry.select(family="census")) == 4 + + def test_other_countries_report_two_labels_without_refusing(self) -> None: + # UK publishers name some regions and authorities differently; the + # UK registries report the variant until one name per code is chosen. + specs = [ + _geography_spec( + "ons.yorkshire.population", + "E12000003", + "Yorkshire and The Humber", + level="region", + ), + _geography_spec( + "hmrc.yorkshire.income", + "E12000003", + "Yorkshire and the Humber", + level="region", + ), + ] + + with pytest.warns( + UserWarning, + match=( + r"'E12000003' is 'Yorkshire and The Humber' \(first on target " + r"'ons\.yorkshire\.population'\) and 'Yorkshire and the Humber'" + ), + ): + registry = TargetRegistry(specs, country="uk") + + assert len(registry) == 2 + + class TestSpecDictCodec: def test_to_dict_from_dict_round_trip_with_and_without_hierarchy(self) -> None: with_hierarchy = _spec(se=12.5, metadata={"kind": "x"}, hierarchy=_hierarchy()) From afc64901bad762c6f2fb68ed54e44e3d5b74e625 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 21 Oct 2026 03:44:36 +0400 Subject: [PATCH 2/5] Give each US geography one label and refuse a second label in a registry The ACS local release labelled each state's Census population target with its postal code ("ME") where the ledger targets use the state's Census name ("Maine"), so its calibration diagnostics file gave all 51 states two labels. Labels: - Add US_STATE_FIPS_TO_NAME to microcosm.calibrate.geography_constants: the 51 names the Chronicle state facts declare and the release's ledger targets carry. The block ladder's US_STATES table derives its P.L. 94-171 directory names from it (the derived table equals the old literal). - population_target_specs labels a state with that name and a district with congressional_district_label, the Census Bureau's district name ("Congressional District 1 (119th Congress), Alabama"). The IRS SOI district rows translated to the 119th Congress plan use the same function, which replaces the postal-code labels that disagreed with the population targets on at-large districts. Chronicle's ACS district facts carry these names for the same ids. Check: - TargetRegistry refuses a US registry in which one geography id has two labels (GeographyLabelConflictError names the id, the labels and the first target carrying each) and reports it for other countries (GeographyLabelConflictWarning), because UK publishers name some regions, nations and local authorities differently. - The ACS local materialize stage assembles its admin targets and the ladder's population targets into one registry before reading the staging frame, so a conflict stops the build before the engine pass. - Two test fixtures gave one id two labels and now name ids the way production facts do: the US fiscal fixtures ("Test geography 0100000US" and "United States") and the ACS local loss-weights fixture (ids as labels). The tests from the previous commit pass unchanged: 88 passed across the four files. Fixes #1335 Co-Authored-By: Claude Opus 5.5 --- ...35-registry-geography-label-check.added.md | 1 + .../1335-us-geography-one-label.fixed.md | 4 + docs/calibration-target-hierarchy.md | 22 ++++ docs/shared-constants.md | 15 ++- .../build/us_runtime/block_ladder_sources.py | 64 ++-------- .../congressional_district_vintage.py | 46 ++++++-- .../build/us_runtime/target_loss_weights.py | 3 +- .../test_us_acs_local_target_loss_weights.py | 19 ++- .../calibrate/geography_constants.py | 65 +++++++++++ .../src/microcosm/calibrate/registry.py | 109 +++++++++++++++++- .../microcosm_build/us_fiscal_targets.py | 44 ++++++- tools/build_us_acs_local_release.py | 71 +++++++++--- tools/us_acs_local_cd_surface.py | 19 ++- 13 files changed, 391 insertions(+), 91 deletions(-) create mode 100644 changelog.d/1335-registry-geography-label-check.added.md create mode 100644 changelog.d/1335-us-geography-one-label.fixed.md diff --git a/changelog.d/1335-registry-geography-label-check.added.md b/changelog.d/1335-registry-geography-label-check.added.md new file mode 100644 index 0000000000..3b98c49db8 --- /dev/null +++ b/changelog.d/1335-registry-geography-label-check.added.md @@ -0,0 +1 @@ +`TargetRegistry` checks that one geography id has exactly one label across a registry (microcosm#1335). A US registry refuses a second label with `GeographyLabelConflictError`, naming the id, both labels and the first target carrying each; every other country's registry reports it as a `GeographyLabelConflictWarning`, because UK publishers name some regions, nations and local authorities differently. Builds assemble their registry before calibration, and the US ACS local materialize stage assembles its admin and ladder population targets into one registry before the engine pass, so a conflict stops a build at its start. A lean checkpoint written before this change with two labels for one US geography is refused when the calibrate stage reloads it; re-run `--stage materialize`. diff --git a/changelog.d/1335-us-geography-one-label.fixed.md b/changelog.d/1335-us-geography-one-label.fixed.md new file mode 100644 index 0000000000..fbcf4fff3a --- /dev/null +++ b/changelog.d/1335-us-geography-one-label.fixed.md @@ -0,0 +1,4 @@ +The US ACS local release's Census population targets label each state and congressional district the way the release's other targets label the same id (microcosm#1335). The release `populace-us-2024-buildo-acs-local-766b7c7-20261017T030317Z` labelled each of its 51 states with its Census name on the IRS SOI, SNAP and Medicaid targets ("Maine") and with its postal code on the state population target ("ME"). + +- **States.** A population target's state label is the state's Census name from the new shared table `microcosm.calibrate.geography_constants.US_STATE_FIPS_TO_NAME`. Its 51 names equal the release's ledger state labels and the names every Chronicle source package of the pinned US feed declares. The block ladder's `US_STATES` table now derives its P.L. 94-171 directory names from it (the derived table equals the previous literal). +- **Congressional districts.** A district's label is the Census Bureau's name ("Congressional District 1 (119th Congress), Alabama", "Congressional District (at Large) (119th Congress), Alaska", "Delegate District (at Large) (119th Congress), District of Columbia"), from `congressional_district_label` in `microcosm.build.us_runtime.congressional_district_vintage`. The IRS SOI district rows translated to the 119th Congress plan take their label from the same function. The two producers used to disagree on at-large districts ("AK at-large congressional district" against "AK congressional at-large"), so a surface binding both at one at-large id gave it two labels. A feed with Chronicle's ACS district facts, which carry the Census names, now agrees with both. diff --git a/docs/calibration-target-hierarchy.md b/docs/calibration-target-hierarchy.md index 10ceb87281..e2f96929d7 100644 --- a/docs/calibration-target-hierarchy.md +++ b/docs/calibration-target-hierarchy.md @@ -105,6 +105,28 @@ contain `materialization`. If Chronicle does not yet publish the required fact, add the source fact to Chronicle before declaring the Microcosm target. Do not construct an observed target value from a Microcosm runtime input. +## One label per geography + +One geography id has exactly one label across a target registry, so a +calibration diagnostics file names each geography once (microcosm#1335). +`TargetRegistry` checks this whenever a build assembles targets into a +registry, which is before calibration. A US registry refuses a second label +with `GeographyLabelConflictError`, naming the id, the labels and the first +target carrying each. Every other country's registry reports the conflict as a +`GeographyLabelConflictWarning`: UK publishers name some regions, nations and +local authorities differently ("Yorkshire and The Humber" and "Yorkshire and +the Humber", "St Helens" and "St. Helens"), and one label per UK code has not +been chosen yet. + +A target Microcosm declares itself takes the label its other sources give the +same id. The US ACS local release's Census population targets label a state +with `US_STATE_FIPS_TO_NAME` ("Maine", as Chronicle's state facts name it) and +a congressional district with the Census Bureau's district name +("Congressional District 1 (119th Congress), Alabama"), from the same function +that labels the IRS SOI district rows translated to the 119th Congress plan. +That release's materialize stage assembles its admin and population targets +into one registry before the engine pass, so a conflict stops it at the start. + ## Propagation stages The expected data flow is: diff --git a/docs/shared-constants.md b/docs/shared-constants.md index 069fef8aaa..a98e0fda79 100644 --- a/docs/shared-constants.md +++ b/docs/shared-constants.md @@ -39,6 +39,10 @@ and build packages. It currently defines: - `UK_WIDE_GEOGRAPHY_IDS` (the K-prefixed UK-wide aggregates among `UK_GEOGRAPHY_ID_TO_LABEL`, which the nation grain of the UK target-weight rules splits on; microcosm#1124) +- `US_STATE_FIPS_TO_NAME` (each state's Census name, "District of Columbia" + included: the names Chronicle's state facts carry, so a target Microcosm + labels itself gives a state the same label as the ledger targets; + microcosm#1335) - `US_STATE_FIPS_TO_POSTAL` - `US_STATE_NUMERIC_FIPS_TO_POSTAL` - `US_STATE_POSTAL_TO_NUMERIC_FIPS` @@ -51,9 +55,14 @@ from microcosm.calibrate.geography_constants import ( ) ``` -Do not add another state FIPS/postal mapping to a country runtime or diagnostics -module. Extend the shared definition and its consistency test when supported -geographies change. +Do not add another state FIPS/postal or FIPS/name mapping to a country runtime +or diagnostics module. Extend the shared definition and its consistency test +when supported geographies change. The block ladder's `US_STATES` table derives +its P.L. 94-171 directory names from `US_STATE_FIPS_TO_NAME`. A congressional +district's label is the Census Bureau's district name, from +`congressional_district_label` in +`microcosm.build.us_runtime.congressional_district_vintage`; it spells the +state with `US_STATE_FIPS_TO_NAME`. The engine's `local_authority` member names are not a constants table: the 361 April 2023 authority names are published ONS data, held in the sha-pinned diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/block_ladder_sources.py b/packages/microcosm-build/src/microcosm/build/us_runtime/block_ladder_sources.py index 8b4c21baf6..4204f9895f 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/block_ladder_sources.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/block_ladder_sources.py @@ -45,64 +45,22 @@ import numpy as np +from microcosm.calibrate.geography_constants import ( + US_STATE_FIPS_TO_NAME, + US_STATE_FIPS_TO_POSTAL, +) + if TYPE_CHECKING: from microcosm.build.us_runtime.geography_ladder import UsBlockLadder #: The 50 states plus DC: (state FIPS, USPS abbreviation, P.L. 94-171 #: directory name). Puerto Rico and the island territories are outside the -#: US artifact's state spine. -US_STATES: tuple[tuple[str, str, str], ...] = ( - ("01", "AL", "Alabama"), - ("02", "AK", "Alaska"), - ("04", "AZ", "Arizona"), - ("05", "AR", "Arkansas"), - ("06", "CA", "California"), - ("08", "CO", "Colorado"), - ("09", "CT", "Connecticut"), - ("10", "DE", "Delaware"), - ("11", "DC", "District_of_Columbia"), - ("12", "FL", "Florida"), - ("13", "GA", "Georgia"), - ("15", "HI", "Hawaii"), - ("16", "ID", "Idaho"), - ("17", "IL", "Illinois"), - ("18", "IN", "Indiana"), - ("19", "IA", "Iowa"), - ("20", "KS", "Kansas"), - ("21", "KY", "Kentucky"), - ("22", "LA", "Louisiana"), - ("23", "ME", "Maine"), - ("24", "MD", "Maryland"), - ("25", "MA", "Massachusetts"), - ("26", "MI", "Michigan"), - ("27", "MN", "Minnesota"), - ("28", "MS", "Mississippi"), - ("29", "MO", "Missouri"), - ("30", "MT", "Montana"), - ("31", "NE", "Nebraska"), - ("32", "NV", "Nevada"), - ("33", "NH", "New_Hampshire"), - ("34", "NJ", "New_Jersey"), - ("35", "NM", "New_Mexico"), - ("36", "NY", "New_York"), - ("37", "NC", "North_Carolina"), - ("38", "ND", "North_Dakota"), - ("39", "OH", "Ohio"), - ("40", "OK", "Oklahoma"), - ("41", "OR", "Oregon"), - ("42", "PA", "Pennsylvania"), - ("44", "RI", "Rhode_Island"), - ("45", "SC", "South_Carolina"), - ("46", "SD", "South_Dakota"), - ("47", "TN", "Tennessee"), - ("48", "TX", "Texas"), - ("49", "UT", "Utah"), - ("50", "VT", "Vermont"), - ("51", "VA", "Virginia"), - ("53", "WA", "Washington"), - ("54", "WV", "West_Virginia"), - ("55", "WI", "Wisconsin"), - ("56", "WY", "Wyoming"), +#: US artifact's state spine. The directory name is the state's Census name +#: with underscores for spaces ("District_of_Columbia"); the codes and names +#: come from ``microcosm.calibrate.geography_constants``. +US_STATES: tuple[tuple[str, str, str], ...] = tuple( + (state_fips, postal, US_STATE_FIPS_TO_NAME[state_fips].replace(" ", "_")) + for state_fips, postal in US_STATE_FIPS_TO_POSTAL.items() ) _PL_GEO_SUMMARY_LEVEL_FIELD = 2 diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/congressional_district_vintage.py b/packages/microcosm-build/src/microcosm/build/us_runtime/congressional_district_vintage.py index 94b1b03435..b532375abd 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/congressional_district_vintage.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/congressional_district_vintage.py @@ -13,11 +13,17 @@ import numpy as np import pandas as pd -from microcosm.calibrate.geography_constants import US_STATE_FIPS_TO_POSTAL +from microcosm.calibrate.geography_constants import ( + US_STATE_FIPS_TO_NAME, + US_STATE_FIPS_TO_POSTAL, +) SOURCE_CONGRESSIONAL_DISTRICT_PREFIX = "5001700US" CURRENT_CONGRESSIONAL_DISTRICT_PREFIX = "5001900US" CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE = "119th_congress" +#: The Congress whose plan ``CURRENT_CONGRESSIONAL_DISTRICT_PREFIX`` identifies, +#: as the Census Bureau's district names spell it. +CURRENT_CONGRESS_NAME = "119th Congress" #: The packaged default crosswalk, built from Census sources by #: ``tools/build_us_congressional_district_vintage_crosswalk.py`` and shipped #: under ``microcosm.build.us_runtime.data``. See the sibling ``.provenance.json`` @@ -617,7 +623,7 @@ def _translated_fact_shell( target_geoid = target_geography_id.removeprefix( CURRENT_CONGRESSIONAL_DISTRICT_PREFIX ) - target_geography_label = _congressional_district_label(target_geography_id) + target_geography_label = congressional_district_label(target_geography_id) geography = dict(_mapping_at(translated, "geography")) geography["id"] = target_geography_id geography["level"] = "congressional_district" @@ -672,23 +678,45 @@ def _translated_fact_shell( return translated -def _congressional_district_label(geography_id: str) -> str: - """Return a reviewed display label for a Census congressional-district ID.""" +def congressional_district_label(geography_id: str) -> str: + """The Census Bureau's name for a current congressional district. + + The Census Bureau names a 119th Congress district "Congressional District + 1 (119th Congress), Alabama", an at-large seat "Congressional District (at + Large) (119th Congress), Alaska", and the District of Columbia's + non-voting seat (district ``00`` here, Census's ``98``) "Delegate District + (at Large) (119th Congress), District of Columbia". Chronicle's ACS + district facts carry these names for the same ids. The IRS SOI district + rows translated here and the ACS local release's population targets both + take their label from this function, so each district has one label + across target sources (microcosm#1335). + """ geoid = geography_id.removeprefix(CURRENT_CONGRESSIONAL_DISTRICT_PREFIX) - if len(geoid) != 4 or not geoid.isdigit(): + if ( + not geography_id.startswith(CURRENT_CONGRESSIONAL_DISTRICT_PREFIX) + or len(geoid) != 4 + or not geoid.isdigit() + ): raise ValueError( f"Invalid current congressional-district geography id {geography_id!r}." ) state_fips, district = geoid[:2], geoid[2:] - postal = US_STATE_FIPS_TO_POSTAL.get(state_fips) - if postal is None: + state_name = US_STATE_FIPS_TO_NAME.get(state_fips) + if state_name is None: raise ValueError( f"Unknown state FIPS code in congressional-district id {geography_id!r}." ) if district == "00": - return f"{postal} at-large congressional district" - return f"{postal} congressional district {int(district)}" + seat = ( + "Delegate District" + if US_STATE_FIPS_TO_POSTAL[state_fips] == "DC" + else "Congressional District" + ) + district_name = f"{seat} (at Large)" + else: + district_name = f"Congressional District {int(district)}" + return f"{district_name} ({CURRENT_CONGRESS_NAME}), {state_name}" def _append_lineage_contribution( diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/target_loss_weights.py b/packages/microcosm-build/src/microcosm/build/us_runtime/target_loss_weights.py index 602cf719ea..d09c417ba9 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/target_loss_weights.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/target_loss_weights.py @@ -317,7 +317,8 @@ def _ledger_concept_budget_key( #: #: - ``ledger_fact_label`` and ``ledger_layout_groupby_value_label``: the #: human-readable labels the ledger compiler (``ledger_targets.py``) stamps -#: on every row, which name the district ("WV congressional district 1"). +#: on every row, which name the district ("Congressional District 1 (119th +#: Congress), West Virginia"). #: :data:`US_FISCAL_TARGET_CONCEPT_METADATA_EXCLUSIONS` does not list them, #: so under the national mapping every compiled district row is its own #: group; the national mapping is kept as it is here. diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_target_loss_weights.py b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_target_loss_weights.py index a5e7870310..798b87e05e 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_target_loss_weights.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_target_loss_weights.py @@ -21,6 +21,9 @@ import microcosm.calibrate as calibrate_package from microcosm.build.us_runtime import target_loss_weights as lw +from microcosm.build.us_runtime.congressional_district_vintage import ( + congressional_district_label, +) from microcosm.calibrate import ( CalibrationHierarchy, HierarchyCategory, @@ -29,6 +32,7 @@ TargetRegistry, TargetSpec, ) +from microcosm.calibrate.geography_constants import US_STATE_FIPS_TO_NAME from test_support.microcosm_build.us_batched_target_materialization import ( _nested_frame, ) @@ -51,13 +55,26 @@ def _load_tool_module(): return module +def _geography_label(geography_id: str) -> str: + """The label every source gives the id, as the population targets do. + + A US registry refuses one geography id with two labels (microcosm#1335). + """ + + if geography_id.startswith("0400000US"): + return US_STATE_FIPS_TO_NAME[geography_id.removeprefix("0400000US")] + return congressional_district_label(geography_id) + + def _hierarchy(name: str, provider: str, level: str, geography_id: str): return CalibrationHierarchy( provider=HierarchyNode(id=provider, label=provider), category=HierarchyCategory( id=f"{provider}.fixture", label="Fixture", provider_id=provider ), - geography=HierarchyGeography(id=geography_id, label=geography_id, level=level), + geography=HierarchyGeography( + id=geography_id, label=_geography_label(geography_id), level=level + ), dimensions=(), target=HierarchyNode(id=name, label=name), ) diff --git a/packages/microcosm-calibrate/src/microcosm/calibrate/geography_constants.py b/packages/microcosm-calibrate/src/microcosm/calibrate/geography_constants.py index 07a4b3aa04..5e0e82b03f 100644 --- a/packages/microcosm-calibrate/src/microcosm/calibrate/geography_constants.py +++ b/packages/microcosm-calibrate/src/microcosm/calibrate/geography_constants.py @@ -16,6 +16,7 @@ "UK_REGION_TIER", "UK_REGION_TIER_ENUM", "UK_WIDE_GEOGRAPHY_IDS", + "US_STATE_FIPS_TO_NAME", "US_STATE_FIPS_TO_POSTAL", "US_STATE_NUMERIC_FIPS_TO_POSTAL", "US_STATE_POSTAL_TO_NUMERIC_FIPS", @@ -169,6 +170,70 @@ ) +#: Two-character Census state FIPS code -> the state's Census name, for the +#: same 51 codes as ``US_STATE_FIPS_TO_POSTAL``. These are the names the US +#: targets' source facts carry for state geographies: the Chronicle source +#: packages of the pinned US feed declare exactly these, and the ledger +#: targets of the release ``populace-us-2024-buildo-acs-local-766b7c7`` carry +#: exactly these. A target Microcosm labels itself uses them, so each state +#: has one label across target sources (microcosm#1335). +US_STATE_FIPS_TO_NAME: Mapping[str, str] = MappingProxyType( + { + "01": "Alabama", + "02": "Alaska", + "04": "Arizona", + "05": "Arkansas", + "06": "California", + "08": "Colorado", + "09": "Connecticut", + "10": "Delaware", + "11": "District of Columbia", + "12": "Florida", + "13": "Georgia", + "15": "Hawaii", + "16": "Idaho", + "17": "Illinois", + "18": "Indiana", + "19": "Iowa", + "20": "Kansas", + "21": "Kentucky", + "22": "Louisiana", + "23": "Maine", + "24": "Maryland", + "25": "Massachusetts", + "26": "Michigan", + "27": "Minnesota", + "28": "Mississippi", + "29": "Missouri", + "30": "Montana", + "31": "Nebraska", + "32": "Nevada", + "33": "New Hampshire", + "34": "New Jersey", + "35": "New Mexico", + "36": "New York", + "37": "North Carolina", + "38": "North Dakota", + "39": "Ohio", + "40": "Oklahoma", + "41": "Oregon", + "42": "Pennsylvania", + "44": "Rhode Island", + "45": "South Carolina", + "46": "South Dakota", + "47": "Tennessee", + "48": "Texas", + "49": "Utah", + "50": "Vermont", + "51": "Virginia", + "53": "Washington", + "54": "West Virginia", + "55": "Wisconsin", + "56": "Wyoming", + } +) + + # Numeric views are derived from the canonical zero-padded mapping so a caller # that reads integer HDF columns does not maintain another copy of the codes. US_STATE_NUMERIC_FIPS_TO_POSTAL: Mapping[int, str] = MappingProxyType( diff --git a/packages/microcosm-calibrate/src/microcosm/calibrate/registry.py b/packages/microcosm-calibrate/src/microcosm/calibrate/registry.py index bf2e615398..bcc204cf75 100644 --- a/packages/microcosm-calibrate/src/microcosm/calibrate/registry.py +++ b/packages/microcosm-calibrate/src/microcosm/calibrate/registry.py @@ -9,7 +9,7 @@ dependency direction the architecture review demanded (no more harness-extracted ``.npz`` files as the de-facto target store). -Two properties are structural, not stylistic: +Three properties are structural, not stylistic: - **Versioning is content-addressed.** :attr:`TargetRegistry.version` is a hash of the canonical JSON of the specs, so two registries with the same facts have @@ -20,6 +20,11 @@ ``signed=True``. The net-STCG incident — a positive-only target surface that let calibration push short-term capital gains to −$3.9T unnoticed — is the motivating case: sign expectations belong in the fact's declaration. +- **A geography has one label.** A US registry refuses a geography id that two + specs label differently, and other countries' registries report it + (:data:`GEOGRAPHY_LABEL_REFUSING_COUNTRIES`). A US release once labelled + Maine "Maine" on 84 targets and "ME" on its population target, and every + other state the same way (microcosm#1335). Specs are fully declarative (column names, never callables) so the registry serializes; a build that needs a computed measure defines the column on its @@ -31,6 +36,7 @@ import hashlib import json import math +import warnings from collections.abc import Callable, Iterable, Iterator, Mapping from dataclasses import asdict, dataclass, field from pathlib import Path @@ -39,13 +45,41 @@ from microcosm.calibrate.hierarchy import CalibrationHierarchy from microcosm.calibrate.target import Target, TargetSet -__all__ = ["TargetSpec", "TargetRegistry", "specs_from_pe_surface"] +__all__ = [ + "GEOGRAPHY_LABEL_REFUSING_COUNTRIES", + "GeographyLabelConflictError", + "GeographyLabelConflictWarning", + "TargetSpec", + "TargetRegistry", + "geography_label_conflicts", + "specs_from_pe_surface", +] #: Format marker + revision for the JSON artifact. _FORMAT_KEY = "populace_target_registry" _FORMAT_VERSION = 3 _READABLE_FORMAT_VERSIONS = frozenset((2, 3)) +#: Countries whose target registries refuse a geography id that carries more +#: than one label (microcosm#1335). Every US target source labels a state with +#: its Census name and a congressional district with the Census Bureau's +#: district name, so a second label for a US id is a defect. UK publishers name +#: one region, nation or local authority differently ("Yorkshire and The +#: Humber" and "Yorkshire and the Humber", "St Helens" and "St. Helens"), and +#: choosing one label per UK code is separate work, so every other country's +#: registry reports a conflict as a :class:`GeographyLabelConflictWarning`. +GEOGRAPHY_LABEL_REFUSING_COUNTRIES: frozenset[str] = frozenset({"us"}) +#: How many conflicting geography ids a refusal or report names in full. +_GEOGRAPHY_LABEL_CONFLICTS_NAMED = 5 + + +class GeographyLabelConflictError(ValueError): + """A target registry gives one geography id more than one label.""" + + +class GeographyLabelConflictWarning(UserWarning): + """A registry outside the refusing countries gives one id two labels.""" + @dataclass(frozen=True) class TargetSpec: @@ -224,6 +258,14 @@ def to_target(self) -> Target: class TargetRegistry: """An ordered, content-addressed collection of :class:`TargetSpec` facts. + Every spec whose hierarchy names a geography must give that geography id + the same label as every other spec in the registry, so a calibration + diagnostics file names each geography once. A registry for a country in + :data:`GEOGRAPHY_LABEL_REFUSING_COUNTRIES` refuses a second label; any + other country's registry reports it as a + :class:`GeographyLabelConflictWarning`. Builds assemble their registry + before calibration, so a conflict stops a build before its solve. + Args: specs: The facts, in order. Duplicate ``(name, period)`` keys are rejected. @@ -232,6 +274,8 @@ class TargetRegistry: Raises: TypeError: If an entry is not a :class:`TargetSpec`. ValueError: On duplicate ``(name, period)`` keys or an empty country. + GeographyLabelConflictError: If a refusing country's registry gives + one geography id more than one label. """ def __init__(self, specs: Iterable[TargetSpec], *, country: str) -> None: @@ -251,6 +295,7 @@ def __init__(self, specs: Iterable[TargetSpec], *, country: str) -> None: "pair may be declared only once." ) seen.add(spec.key) + _check_geography_labels(materialized, country=country) self._specs = materialized self._country = country @@ -380,6 +425,66 @@ def from_json(cls, path: str | Path) -> TargetRegistry: return registry +def geography_label_conflicts( + specs: Iterable[TargetSpec], +) -> dict[str, dict[str, str]]: + """Each geography id that the specs' hierarchies give more than one label. + + Returns ``{geography id: {label: name of the first spec with it}}`` for + every such id, ids and labels in the order they first appear. A spec + without a hierarchy labels no geography. + """ + + first_spec_by_label: dict[str, dict[str, str]] = {} + for spec in specs: + if spec.hierarchy is None: + continue + geography = spec.hierarchy.geography + first_spec_by_label.setdefault(geography.id, {}).setdefault( + geography.label, spec.name + ) + return { + geography_id: labels + for geography_id, labels in first_spec_by_label.items() + if len(labels) > 1 + } + + +def _check_geography_labels(specs: Iterable[TargetSpec], *, country: str) -> None: + conflicts = geography_label_conflicts(specs) + if not conflicts: + return + message = _geography_label_conflict_message(conflicts, country=country) + if country.lower() in GEOGRAPHY_LABEL_REFUSING_COUNTRIES: + raise GeographyLabelConflictError(message) + # Three frames up is the code that constructed the registry. + warnings.warn(message, GeographyLabelConflictWarning, stacklevel=3) + + +def _geography_label_conflict_message( + conflicts: Mapping[str, Mapping[str, str]], *, country: str +) -> str: + named = list(conflicts.items())[:_GEOGRAPHY_LABEL_CONFLICTS_NAMED] + described = "; ".join( + f"{geography_id!r} is " + + " and ".join( + f"{label!r} (first on target {spec_name!r})" + for label, spec_name in labels.items() + ) + for geography_id, labels in named + ) + unnamed = len(conflicts) - len(named) + if unnamed: + described += f"; and {unnamed} more geography id(s)" + return ( + f"Target registry for {country!r} gives {len(conflicts)} geography " + f"id(s) more than one label: {described}. One geography id must have " + "exactly one label across a registry, so its calibration diagnostics " + "name each geography once; label every source's target for an id the " + "same way (microcosm#1335)." + ) + + def specs_from_pe_surface( names: Iterable[str], values: Iterable[float], diff --git a/test_support/microcosm_build/us_fiscal_targets.py b/test_support/microcosm_build/us_fiscal_targets.py index 6ba190b75b..a76de7b803 100644 --- a/test_support/microcosm_build/us_fiscal_targets.py +++ b/test_support/microcosm_build/us_fiscal_targets.py @@ -28,15 +28,49 @@ fiscal_targets, us_fiscal_target_exclusion_receipt, ) +from microcosm.build.us_runtime.congressional_district_vintage import ( + CURRENT_CONGRESSIONAL_DISTRICT_PREFIX, + congressional_district_label, +) from microcosm.build.us_runtime.fiscal_targets import ( _M_CHIP_STATE_FIPS, US_JCT_TAX_EXPENDITURE_TARGET_REFERENCES, ) -from microcosm.calibrate.geography_constants import US_STATE_FIPS_TO_POSTAL +from microcosm.calibrate.geography_constants import ( + US_STATE_FIPS_TO_NAME, + US_STATE_FIPS_TO_POSTAL, +) from test_support.paths import paths_for _TEST_PATHS = paths_for("microcosm-build") + +def _fixture_geography_name(geography_id: str) -> str: + """The name Chronicle facts carry for a US geography id. + + A US registry refuses one geography id with two labels (microcosm#1335), + and a ledger target takes its label from its facts' geography name, so + every fixture fact names an id the way the production facts do. + """ + + if geography_id == "0100000US": + return "United States" + if geography_id.startswith("0400000US"): + state_fips = geography_id.removeprefix("0400000US") + elif geography_id.startswith(CURRENT_CONGRESSIONAL_DISTRICT_PREFIX): + state_fips = geography_id.removeprefix(CURRENT_CONGRESSIONAL_DISTRICT_PREFIX)[ + :2 + ] + else: + state_fips = None + if state_fips not in US_STATE_FIPS_TO_NAME: + # Territories and other ids outside the 50 states and DC. + return f"Test geography {geography_id}" + if geography_id.startswith("0400000US"): + return US_STATE_FIPS_TO_NAME[state_fips] + return congressional_district_label(geography_id) + + REFERENCE_JCT_TAX_EXPENDITURE_TARGETS = { "salt_deduction": "jct.tax_expenditures.cy2024.salt_deduction.revenue_loss", "medical_expense_deduction": ( @@ -320,7 +354,7 @@ def _usda_snap_caseload_fact( "geography": { "level": geography_level, "id": geography_id, - "name": f"Test geography {geography_id}", + "name": _fixture_geography_name(geography_id), }, "dimensions": {}, "dimension_labels": { @@ -462,7 +496,7 @@ def _ledger_fact_for_reference(reference, *, value: float) -> dict[str, object]: "geography": { "level": geography_level, "id": geography_id, - "name": f"Test geography {geography_id}", + "name": _fixture_geography_name(geography_id), }, "dimensions": dimensions, "layout": { @@ -977,7 +1011,7 @@ def _cms_medicaid_enrollment_fact( "geography": { "level": geography_level, "id": geography_id, - "name": f"Test geography {geography_id}", + "name": _fixture_geography_name(geography_id), }, "dimensions": {}, "dimension_labels": {"program": "Program"}, @@ -1118,7 +1152,7 @@ def _dynamic_ledger_fact( "geography": { "level": geography_level, "id": geography_id, - "name": f"Test geography {geography_id}", + "name": _fixture_geography_name(geography_id), }, "dimensions": fact_dimensions, "dimension_labels": dimension_labels, diff --git a/tools/build_us_acs_local_release.py b/tools/build_us_acs_local_release.py index ebe36a549c..5e9b9d7b9a 100644 --- a/tools/build_us_acs_local_release.py +++ b/tools/build_us_acs_local_release.py @@ -1090,13 +1090,21 @@ def population_targets(frame, ladder_populations, geographies: list[str]): def population_target_specs(names, values): - """Declare ladder population targets with the shared diagnostics hierarchy.""" + """Declare ladder population targets with the shared diagnostics hierarchy. + + Each geography carries the label the release's other targets give the + same id: a state its Census name, the name the ledger targets take from + their facts ("Maine"), and a district the Census Bureau's name, which + the IRS SOI district rows carry too. A registry refuses one US geography + with two labels (microcosm#1335). + """ from microcosm.build.us_runtime.congressional_district_vintage import ( CURRENT_CONGRESSIONAL_DISTRICT_PREFIX, + congressional_district_label, ) from microcosm.calibrate import TargetSpec - from microcosm.calibrate.geography_constants import US_STATE_FIPS_TO_POSTAL + from microcosm.calibrate.geography_constants import US_STATE_FIPS_TO_NAME from microcosm.calibrate.hierarchy import ( CalibrationHierarchy, HierarchyCategory, @@ -1114,21 +1122,19 @@ def population_target_specs(names, values): for name, value in zip(names, values, strict=True): if name.startswith("pop_state_"): fips = name.removeprefix("pop_state_") - label = US_STATE_FIPS_TO_POSTAL.get(fips, f"State {fips}") + label = US_STATE_FIPS_TO_NAME.get(fips) + if label is None: + raise ValueError( + f"Population target {name!r} names no US state FIPS code." + ) geography = HierarchyGeography(f"0400000US{fips}", label, "state") elif name.startswith("pop_cd_"): geoid = name.removeprefix("pop_cd_") - state_fips, district = geoid[:2], geoid[2:] - state = US_STATE_FIPS_TO_POSTAL.get(state_fips, state_fips) - district_label = ( - "at-large" if district == "00" else f"district {int(district)}" - ) - label = f"{state} congressional {district_label}" # The ladder's districts are the 119th Congress plan. + geography_id = f"{CURRENT_CONGRESSIONAL_DISTRICT_PREFIX}{geoid}" + label = congressional_district_label(geography_id) geography = HierarchyGeography( - f"{CURRENT_CONGRESSIONAL_DISTRICT_PREFIX}{geoid}", - label, - "congressional_district", + geography_id, label, "congressional_district" ) else: # pragma: no cover - names originate in population_measure_arrays raise ValueError(f"Unknown population target name {name!r}.") @@ -1154,6 +1160,44 @@ def population_target_specs(names, values): return tuple(specs) +def ladder_population_target_specs(ladder_populations, geographies): + """Every population target the ladder declares, before a household is read. + + :func:`population_targets` later drops a ladder cell no staged household + supports, so its targets are a subset of these with the same labels. + """ + + cells = [ + (cd_surface.population_target_name(geography, code), float(population)) + for geography in geographies + for code, population in sorted(ladder_populations[geography].items()) + ] + return population_target_specs( + [name for name, _population in cells], + [population for _name, population in cells], + ) + + +def require_one_label_per_geography(admin_registry, ladder_populations, geographies): + """Refuse admin and population targets that label one geography two ways. + + The two first share a registry in the lean checkpoint, after the engine + pass over every staged household; assembling them here moves the + registry's geography-label refusal (microcosm#1335) to the start of the + materialize stage. + """ + + from microcosm.calibrate import TargetRegistry + + return TargetRegistry( + ( + *admin_registry.specs, + *ladder_population_target_specs(ladder_populations, geographies), + ), + country="us", + ) + + #: Household columns the lean checkpoint keeps: identity, geography, and the #: origin tags the distinct-household ESS and spine shares need. LEAN_HOUSEHOLD_COLUMNS = ( @@ -1392,6 +1436,8 @@ def do_materialize(args) -> None: f"admin specs: {len(registry)} ({families}, soi_mode={args.soi_mode}); " f"RI substitution records={len(surface.ri_substitutions)}" ) + populations = ladder_population(args.ladder, sorted(set(geographies) | {"cd"})) + require_one_label_per_geography(registry, populations, geographies) summary_path = _staging_summary_path(args) if not summary_path.exists(): raise SystemExit( @@ -1446,7 +1492,6 @@ def do_materialize(args) -> None: ) admin_roles = [cd_surface.target_record(spec) for spec in compiled_specs] - populations = ladder_population(args.ladder, sorted(set(geographies) | {"cd"})) if args.soi_mode == SOI_MODE_STATE_CD: _attach_pro_rata_populations(admin_roles, populations["cd"]) _require_pro_rata_populations(admin_roles) diff --git a/tools/us_acs_local_cd_surface.py b/tools/us_acs_local_cd_surface.py index edd37055fa..c6bc851c75 100644 --- a/tools/us_acs_local_cd_surface.py +++ b/tools/us_acs_local_cd_surface.py @@ -1115,6 +1115,17 @@ def district_row_parents( return parents, unchecked +#: Digits of a ladder population target's geography code, by ladder geography. +POPULATION_TARGET_CODE_DIGITS: Mapping[str, int] = {"state": 2, "cd": 4} + + +def population_target_name(geography: str, code: int) -> str: + """A ladder population target's name: ``pop_state_06``, ``pop_cd_0601``.""" + + digits = POPULATION_TARGET_CODE_DIGITS[geography] + return f"pop_{geography}_{int(code):0{digits}d}" + + def population_rows( households: pd.DataFrame, household_size: np.ndarray, @@ -1128,8 +1139,8 @@ def population_rows( """ column_map = { - "state": ("state_fips", "state", 2, GEOGRAPHY_STATE), - "cd": ("congressional_district_geoid", "cd", 4, GEOGRAPHY_CD), + "state": ("state_fips", "state", GEOGRAPHY_STATE), + "cd": ("congressional_district_geoid", "cd", GEOGRAPHY_CD), } size32 = np.asarray(household_size, dtype=np.float32) records: list[dict] = [] @@ -1138,10 +1149,10 @@ def population_rows( data: list[np.ndarray] = [] dropped: list[str] = [] for geography in geographies: - column, key, width, level = column_map[geography] + column, key, level = column_map[geography] codes = pd.to_numeric(households[column]).to_numpy() for value, population in sorted(ladder_populations[key].items()): - name = f"pop_{geography}_{value:0{width}d}" + name = population_target_name(geography, value) present = np.flatnonzero(codes == value) if not len(present): dropped.append(name) From d1ef7474cf21cedfee305c0e787716207dd70322 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 21 Oct 2026 04:07:54 +0400 Subject: [PATCH 3/5] Test that calibrate names the remedy for an earlier checkpoint A lean checkpoint written by the materialize stage before this branch labels a state's population target with its postal code ("ME") next to ledger targets labelled "Maine". The registry now refuses that target_registry.json, but --stage calibrate stops with the registry's GeographyLabelConflictError, which names the labels and not what the operator should do, although the pull request says the stage tells the operator to re-run --stage materialize. The test writes such a checkpoint and expects load_checkpoint_surface to exit with that remedy. It fails on this branch (1 failed, 4 passed in the module): GeographyLabelConflictError is raised instead of SystemExit. Refs #1335 Co-Authored-By: Claude Opus 5.5 --- .../us/test_us_acs_local_geography_labels.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_geography_labels.py b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_geography_labels.py index cf44001000..d914871777 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_geography_labels.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_geography_labels.py @@ -32,6 +32,7 @@ TargetSpec, geography_constants, ) +from microcosm.calibrate import registry as registry_module from test_support.paths import paths_for _TEST_PATHS = paths_for("microcosm-build") @@ -254,3 +255,38 @@ def engine_pass_started(*_args, **_kwargs): ), ): module.do_materialize(args) + + +def test_calibrate_refuses_an_earlier_checkpoint_and_names_the_remedy( + monkeypatch, tmp_path +) -> None: + """A checkpoint from before the label fix is refused with its remedy. + + The earlier materialize stage labelled a state's population target with + its postal code, so its target_registry.json gives the state two labels. + The calibrate stage must tell the operator to re-run the materialize + stage, not stop at the registry's error, which names only the labels. + """ + + module = _load_tool_module() + checkpoint_dir = tmp_path / "checkpoint" + checkpoint_dir.mkdir() + earlier_specs = [ + _ledger_state_spec("usda_snap.me.households", "23", "Maine"), + _ledger_state_spec("pop_state_23", "23", "ME"), + ] + with monkeypatch.context() as patch: + # The earlier tool wrote its registry without the label check. + patch.setattr( + registry_module, "GEOGRAPHY_LABEL_REFUSING_COUNTRIES", frozenset() + ) + with pytest.warns(registry_module.GeographyLabelConflictWarning): + earlier_registry = TargetRegistry(earlier_specs, country="us") + earlier_registry.to_json(checkpoint_dir / "target_registry.json") + (checkpoint_dir / module.TARGET_MATRIX_FILENAME).write_bytes(b"matrix") + (checkpoint_dir / module.TARGET_ROLES_FILENAME).write_text("[]") + + with pytest.raises(SystemExit, match="Re-run --stage materialize") as refusal: + module.load_checkpoint_surface(checkpoint_dir) + + assert "'0400000US23' is 'Maine'" in str(refusal.value) From fe2359803c41f75308f16e0cf1a8cfb9c3e3bd43 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 21 Oct 2026 04:08:48 +0400 Subject: [PATCH 4/5] Tell the operator to re-run materialize for an earlier checkpoint load_checkpoint_surface (the calibrate stage's checkpoint reader) now turns the registry's GeographyLabelConflictError into the tool's usual SystemExit for an unusable checkpoint: it names the registry file, says that checkpoints written before microcosm#1335 give a geography two labels, tells the operator to re-run --stage materialize with the current tool, and keeps the registry's message with the ids and labels. The changelog fragment says the refusal carries that message. The test from the previous commit passes unchanged: 5 passed in test_us_acs_local_geography_labels.py. Refs #1335 Co-Authored-By: Claude Opus 5.5 --- .../1335-registry-geography-label-check.added.md | 2 +- tools/build_us_acs_local_release.py | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/changelog.d/1335-registry-geography-label-check.added.md b/changelog.d/1335-registry-geography-label-check.added.md index 3b98c49db8..e704c60a6b 100644 --- a/changelog.d/1335-registry-geography-label-check.added.md +++ b/changelog.d/1335-registry-geography-label-check.added.md @@ -1 +1 @@ -`TargetRegistry` checks that one geography id has exactly one label across a registry (microcosm#1335). A US registry refuses a second label with `GeographyLabelConflictError`, naming the id, both labels and the first target carrying each; every other country's registry reports it as a `GeographyLabelConflictWarning`, because UK publishers name some regions, nations and local authorities differently. Builds assemble their registry before calibration, and the US ACS local materialize stage assembles its admin and ladder population targets into one registry before the engine pass, so a conflict stops a build at its start. A lean checkpoint written before this change with two labels for one US geography is refused when the calibrate stage reloads it; re-run `--stage materialize`. +`TargetRegistry` checks that one geography id has exactly one label across a registry (microcosm#1335). A US registry refuses a second label with `GeographyLabelConflictError`, naming the id, both labels and the first target carrying each; every other country's registry reports it as a `GeographyLabelConflictWarning`, because UK publishers name some regions, nations and local authorities differently. Builds assemble their registry before calibration, and the US ACS local materialize stage assembles its admin and ladder population targets into one registry before the engine pass, so a conflict stops a build at its start. A lean checkpoint written before this change with two labels for one US geography is refused when the calibrate stage reloads it, with a message to re-run `--stage materialize`. diff --git a/tools/build_us_acs_local_release.py b/tools/build_us_acs_local_release.py index 5e9b9d7b9a..3b32d8b338 100644 --- a/tools/build_us_acs_local_release.py +++ b/tools/build_us_acs_local_release.py @@ -1832,10 +1832,13 @@ def load_checkpoint_surface(checkpoint_dir: Path, identity: dict | None = None): roles and the matrix must still be the bytes materialize wrote. A checkpoint from before the sparse matrix (dense measure columns in the lean H5) is refused: re-run ``--stage materialize``. + A checkpoint whose registry gives one geography two labels (written + before microcosm#1335) is refused the same way. Returns ``(frame, design_weights, registry, roles, matrix)``. """ from microcosm.calibrate import TargetRegistry + from microcosm.calibrate.registry import GeographyLabelConflictError matrix_path = checkpoint_dir / TARGET_MATRIX_FILENAME roles_path = checkpoint_dir / TARGET_ROLES_FILENAME @@ -1849,7 +1852,17 @@ def load_checkpoint_surface(checkpoint_dir: Path, identity: dict | None = None): ) if identity is not None: _verify_checkpoint_digests(checkpoint_dir, identity) - registry = TargetRegistry.from_json(registry_path) + try: + registry = TargetRegistry.from_json(registry_path) + except GeographyLabelConflictError as error: + # A materialize stage from before microcosm#1335 labelled a state's + # population target with its postal code ("ME") and the ledger + # targets with the state's name ("Maine"). + raise SystemExit( + f"{registry_path} gives one geography more than one label, as " + "checkpoints written before microcosm#1335 do. Re-run --stage " + f"materialize with the current tool. {error}" + ) from error roles = json.loads(roles_path.read_text()) matrix = cd_surface.load_target_matrix(matrix_path) frame, design_weights = load_lean_frame(checkpoint_dir / "target_frame_lean.h5") From bb3f7c8862840870fe1ee3c8da68d900eaf71e94 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 21 Oct 2026 04:09:09 +0400 Subject: [PATCH 5/5] Say which ACS local district label carries the Census name The note on ACS_LOCAL_PER_DISTRICT_METADATA gave the Census district name as the example for both ledger_fact_label and ledger_layout_groupby_value_label. The vintage translation sets only the group value label (and the geography name) to the current district's Census name; it copies the source-plan fact's label unchanged into the translated fact, so ledger_fact_label keeps it. Comment only. Refs #1335 Co-Authored-By: Claude Opus 5.5 --- .../src/microcosm/build/us_runtime/target_loss_weights.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/target_loss_weights.py b/packages/microcosm-build/src/microcosm/build/us_runtime/target_loss_weights.py index d09c417ba9..1ecefe38b7 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/target_loss_weights.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/target_loss_weights.py @@ -317,8 +317,10 @@ def _ledger_concept_budget_key( #: #: - ``ledger_fact_label`` and ``ledger_layout_groupby_value_label``: the #: human-readable labels the ledger compiler (``ledger_targets.py``) stamps -#: on every row, which name the district ("Congressional District 1 (119th -#: Congress), West Virginia"). +#: on every row, which name the district. A translated IRS SOI row's group +#: value label is the current district's Census name ("Congressional +#: District 1 (119th Congress), West Virginia"); its fact label is the +#: label of a source-plan fact, which the translation does not change. #: :data:`US_FISCAL_TARGET_CONCEPT_METADATA_EXCLUSIONS` does not list them, #: so under the national mapping every compiled district row is its own #: group; the national mapping is kept as it is here.