From ab6171978a008a0ba66db3fff1156981f5c4e86b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 19:06:53 -0400 Subject: [PATCH 01/12] Rebase congressional-district rows of the national-control columns as shares On the full surface the congressional-district file's US row was still a second national all-AGI target beside the Table 1.x control for 11 SOI quantities (control / district US row 1.001 to 1.169 on the pinned feed). Its state and district rows of these measures now ship as shares of the file's own US row times #1247's control, naming the denominator in soi_share_total_source_record_id, and the US row retires. The register records each column's IRS id, and a district fact that would be rebased under another column refuses the compile. Co-Authored-By: Claude Opus 5.5 --- .../build/us_runtime/fiscal_targets.py | 263 +++++++++++++++++- 1 file changed, 258 insertions(+), 5 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py b/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py index ffe8714930..06fe36f6e9 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py @@ -47,6 +47,7 @@ __all__ = [ "AmbiguousSoiCapitalGainsControlError", + "UnreviewedSoiNationalControlColumnError", "US_FISCAL_MACRO_REALISM_BANDS", "US_FISCAL_TARGET_REGISTRY", "US_FISCAL_TARGET_SPECS", @@ -1035,6 +1036,9 @@ def compile_us_fiscal_target_registry( ``period_contract_waiver`` metadata in diagnostics. """ materialized_facts = tuple(facts) + _require_reviewed_soi_district_columns( + materialized_facts, target_period=target_period + ) if congressional_district_vintage_crosswalk is not None: materialized_facts = translate_congressional_district_facts_to_current_vintage( materialized_facts, @@ -1073,6 +1077,11 @@ def compile_us_fiscal_target_registry( materialized_facts, target_period=target_period, ) + registry = _rebase_stale_soi_district_distributions( + registry, + materialized_facts, + target_period=target_period, + ) registry = _rebase_soi_state_agi_bands(registry, materialized_facts) registry = apply_ledger_target_profile( registry, @@ -1815,11 +1824,15 @@ class _SoiHt2NationalControlRule: ``concept`` names the Form 1040 line both columns report; the control is the latest ``control_measure_id`` fact of the ``control_family`` table. + ``source_column`` is the IRS column that HT2 and the congressional-district + file both publish under the measure id (``22incddocguide.docx`` defines it + at the same line). """ control_family: str control_measure_id: str concept: str + source_column: str #: Historic Table 2 columns whose national level comes from a Pub 1304 table @@ -1850,66 +1863,99 @@ class _SoiHt2NationalControlRule: #: $11,944.447B). Table 1.1 supplies the return count and AGI; Table 1.2 #: repeats them and stays a national target of equal value. A column absent #: from this register keeps its HT2 level and its HT2 national row. +#: +#: The congressional-district file (``22incd.csv``) publishes the same +#: columns, and ``22incddocguide.docx`` defines each at the same line, so +#: ``_rebase_stale_soi_district_distributions`` takes its rows of these +#: measures as shares of the same controls. Its US row is a narrower count of +#: the same thing: TY2022 returns processed in calendar 2023 and matched to a +#: district by ZIP code, 0.940 (itemized deductions) to 0.997 (IRA +#: distribution returns) of Table 1.4's TY2022 cell. +#: The district package carries no wages or taxable-income rows. _SOI_HT2_NATIONAL_CONTROLS: dict[str, _SoiHt2NationalControlRule] = { "return_count": _SoiHt2NationalControlRule( - "table_1_1", "return_count", "form_1040_returns" + "table_1_1", + "return_count", + "form_1040_returns", + "N1", ), "adjusted_gross_income": _SoiHt2NationalControlRule( - "table_1_1", "adjusted_gross_income", "form_1040_line_11_agi" + "table_1_1", + "adjusted_gross_income", + "form_1040_line_11_agi", + "A00100", ), "wages_salaries_amount": _SoiHt2NationalControlRule( - "table_1_4", "wages_salaries_amount", "form_1040_line_1z_wages" + "table_1_4", + "wages_salaries_amount", + "form_1040_line_1z_wages", + "A00200", ), "wages_salaries_returns": _SoiHt2NationalControlRule( - "table_1_4", "wages_salaries_returns", "form_1040_line_1z_wages" + "table_1_4", + "wages_salaries_returns", + "form_1040_line_1z_wages", + "N00200", ), "taxable_ira_distributions_amount": _SoiHt2NationalControlRule( "table_1_4", "taxable_ira_distributions_amount", "form_1040_line_4b_taxable_ira_distributions", + "A01400", ), "taxable_ira_distributions_returns": _SoiHt2NationalControlRule( "table_1_4", "taxable_ira_distributions_returns", "form_1040_line_4b_taxable_ira_distributions", + "N01400", ), "taxable_pension_income_amount": _SoiHt2NationalControlRule( "table_1_4", "taxable_pension_income_amount", "form_1040_line_5b_taxable_pensions_and_annuities", + "A01700", ), "taxable_pension_income_returns": _SoiHt2NationalControlRule( "table_1_4", "taxable_pension_income_returns", "form_1040_line_5b_taxable_pensions_and_annuities", + "N01700", ), "unemployment_compensation_amount": _SoiHt2NationalControlRule( "table_1_4", "unemployment_compensation_amount", "schedule_1_line_7_unemployment_compensation", + "A02300", ), "unemployment_compensation_returns": _SoiHt2NationalControlRule( "table_1_4", "unemployment_compensation_returns", "schedule_1_line_7_unemployment_compensation", + "N02300", ), "taxable_social_security_amount": _SoiHt2NationalControlRule( "table_1_4", "taxable_social_security_amount", "form_1040_line_6b_taxable_social_security", + "A02500", ), "taxable_social_security_returns": _SoiHt2NationalControlRule( "table_1_4", "taxable_social_security_returns", "form_1040_line_6b_taxable_social_security", + "N02500", ), "itemized_deductions_amount": _SoiHt2NationalControlRule( "table_1_2", "total_itemized_deductions_amount", "form_1040_line_12_itemized_deductions", + "A04470", ), "taxable_income_amount": _SoiHt2NationalControlRule( - "table_1_2", "taxable_income_amount", "form_1040_line_15_taxable_income" + "table_1_2", + "taxable_income_amount", + "form_1040_line_15_taxable_income", + "A04800", ), } _SOI_HT2_MEASURE_BY_CONTROL: dict[tuple[str, str], str] = { @@ -2213,6 +2259,213 @@ def _soi_ht2_lineage_data_year(record_set_id: str) -> str | None: return None +class UnreviewedSoiNationalControlColumnError(ValueError): + """A district fact of a registered measure reads another IRS column.""" + + +@dataclass(frozen=True) +class _SoiDistrictShareTotal: + value: float + source_record_id: str + + +def _rebase_stale_soi_district_distributions( + registry: TargetRegistry, + facts: tuple[object, ...], + *, + target_period: int | str, +) -> TargetRegistry: + """Use congressional-district rows of the registered columns as shares. + + The congressional-district file publishes the columns of + ``_SOI_HT2_NATIONAL_CONTROLS`` at the same Form 1040 lines, for TY2022 + returns processed in calendar 2023 and matched to a district by ZIP code + (the feed stamps it ty2023). Its rows shipped as levels on the ``full`` + surface, and its US row was a second national target beside each Table + 1.x control: on the pinned feed the control was 1.001x (taxable pension + returns) to 1.169x (taxable Social Security) the district US row. + + Each state and district row of such a measure now ships as its share of + the file's own US row at its own period, times the control of + ``_soi_ht2_national_controls`` at or after that period, as microcosm#1229 + does for taxable interest and #1040 for capital gains. Every share row + names its denominator in ``soi_share_total_source_record_id``. One factor + per measure scales the file's states and districts alike, so the + district-to-state reconciliation that runs next keeps factor 1, and + because the file's states and districts each sum exactly to its US row, + each sums to the control. The period stamp cancels in the share: the + value lands at the control's period, and target aging adds the remaining + years on the control's own link. + + The US row retires once its rows are shares, because the control's row + owns the national level. Without an eligible control (none, after the + build period, or older than the row) the rows keep their levels and the + US row stays. With an eligible control but no nonzero US row of the + file's own, a row cannot be a share and is dropped; a stale level beside + the control is the defect this pass removes (#1229 and #1040 do the same). + """ + + controls = _soi_ht2_national_controls(facts, target_period=target_period) + us_totals = _soi_district_us_totals(facts) + specs: list[TargetSpec] = [] + for spec in registry.specs: + measure_id = _soi_district_rebase_measure(spec) + if measure_id is None: + specs.append(spec) + continue + key = _soi_ht2_control_key_from_spec(spec, measure_id) + source_period = spec.metadata.get("source_period", "") + control = controls.get(key) + if control is None or not _period_not_before( + control.period_key, _period_key_from_value(source_period) + ): + specs.append(spec) + continue + if _is_national_all_agi_spec(spec): + continue + us_total = us_totals.get((*key, source_period)) + if us_total is None or us_total.value == 0: + continue + concept = _SOI_HT2_NATIONAL_CONTROLS[measure_id].concept + factor = control.value / us_total.value + specs.append( + replace( + spec, + value=spec.value * factor, + metadata={ + **dict(spec.metadata), + "uprating_index": f"total_{measure_id}", + "uprating_from_period": source_period, + # Lands at the CONTROL's period; target aging completes + # the remaining links (#488). + "uprating_to_period": control.source_period, + "uprating_index_source_period": control.source_period, + "uprating_index_source_record_id": control.source_record_id, + "uprating_factor": _format_float(factor), + "stale_distribution_rebased_to_active_total": "true", + "soi_source_concept": concept, + "soi_control_concept": concept, + "soi_share_total_source_record_id": us_total.source_record_id, + }, + ) + ) + return TargetRegistry(specs, country=registry.country) + + +def _soi_district_rebase_measure(spec: TargetSpec) -> str | None: + """The registered measure of an all-AGI congressional-district spec.""" + if spec.family != "irs_soi": + return None + measure_id = spec.metadata.get("source_measure_id", "") + if measure_id not in _SOI_HT2_NATIONAL_CONTROLS: + return None + record_set_id = spec.metadata.get("ledger_layout_record_set_id", "") + if _soi_record_set_family(record_set_id) != "congressional_district": + return None + if _bounds_from_metadata(spec) != (-float("inf"), float("inf")): + return None + return measure_id + + +def _soi_district_fact_key(fact: object) -> tuple[str, str, str] | None: + """(measure, filing status, universe) of an all-AGI district fact.""" + if _source_name(fact) != "irs_soi": + return None + measure_id = _measure_id(fact) + if measure_id not in _SOI_HT2_NATIONAL_CONTROLS: + return None + record_set_id = _str_at(fact, "layout", "record_set_id") + if _soi_record_set_family(record_set_id) != "congressional_district": + return None + if not _is_all_agi_range_fact(fact): + return None + status = _filing_status_label(_dimensions(fact).get("filing_status")) + if status is None: + return None + return (measure_id, status, _soi_return_universe_from_fact(fact)) + + +def _soi_district_us_totals( + facts: tuple[object, ...], +) -> dict[tuple[str, str, str, str], _SoiDistrictShareTotal]: + """The district file's US totals per (measure, filing status, universe, period). + + A total is a share denominator, so feed order must not pick it: two + different values for one key raise, and equal values under two record + ids name the smaller id (as the #1229 and #1040 denominators do). Two + district files compiled at one period, of different boundary vintages, + share the key, so their US rows must agree. + """ + totals: dict[tuple[str, str, str, str], _SoiDistrictShareTotal] = {} + for fact in facts: + key = _soi_district_fact_key(fact) + if key is None or _geography_level(fact) != "country": + continue + source_record_id = _source_record_id(fact) + if not source_record_id: + continue + total_key = (*key, str(_period_value(fact))) + candidate = _SoiDistrictShareTotal(_numeric_value(fact), source_record_id) + current = totals.get(total_key) + if current is not None and current.value != candidate.value: + raise AmbiguousSoiNationalControlError( + "SOI congressional-district US total is ambiguous: " + f"{total_key} has {current.value!r} ({current.source_record_id}) " + f"and {candidate.value!r} ({candidate.source_record_id}). " + "Deduplicate them upstream; feed order must not pick a rebase " + "denominator." + ) + if current is None or candidate.source_record_id < current.source_record_id: + totals[total_key] = candidate + return totals + + +def _require_reviewed_soi_district_columns( + facts: tuple[object, ...], + *, + target_period: int | str, +) -> None: + """Refuse a district fact the rebase would bind under an unreviewed column. + + ``_rebase_stale_soi_district_distributions`` relies on each registered + measure id reading ``source_column``. A package that maps the measure id + to another column (the CD SALT and PTC measures read the wrong columns, + microcosm#1038), or names none, would be rebased as the wrong concept, so + it refuses the compile. Only a fact with an eligible control is checked; + the others keep their levels as before. This runs on the facts as the + feed carries them, before the crosswalk merges source districts into one + fact with a single contributor's layout. + + Raises: + UnreviewedSoiNationalControlColumnError: A checked fact reads a column + other than its measure's ``source_column``, or names none. + """ + controls: dict[tuple[str, str, str], _SoiTotalControl] | None = None + unreviewed: dict[str, str] = {} + for fact in facts: + key = _soi_district_fact_key(fact) + if key is None: + continue + if controls is None: + controls = _soi_ht2_national_controls(facts, target_period=target_period) + control = controls.get(key) + if control is None or not _period_not_before( + control.period_key, _period_key(fact) + ): + continue + column = _str_at(fact, "layout", "source_column_id") + if column != _SOI_HT2_NATIONAL_CONTROLS[key[0]].source_column: + unreviewed[_source_record_id(fact)] = column or "" + if unreviewed: + sample = dict(sorted(unreviewed.items())[:5]) + raise UnreviewedSoiNationalControlColumnError( + f"{len(unreviewed)} congressional-district fact(s) of registered " + f"measures read an IRS column other than the reviewed one, e.g. " + f"{sample}. Review the column against the district documentation " + "guide and _SOI_HT2_NATIONAL_CONTROLS before rebasing it." + ) + + #: Control universes admitted for the stale-interest rebase. A stale spec's #: own universe always outranks the bridge, and the ONE admitted bridge is #: all_returns -> returns_excluding_dependents: the Pub 1304 excluding- From eb0c9078719f7992f843fe1b812d8e09f0e1991b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 19:12:03 -0400 Subject: [PATCH 02/12] Read rebased bridge siblings at the district file's own level on state_cd The state_cd level bridges lift charitable, interest-paid and QBI district levels by a sibling's Historic Table 2 / district-file ratio. Their siblings (itemized deductions, return count, AGI) are now national shares, so the ratio is read from the sibling's file level (cd_file_level), re-aged like the bridged row. Without this, the bridged amounts fell 9.07% (charitable and interest paid), 5.63% (QBI) and the counts 2.65% on the pinned feed; with it every state_cd value matches the #1247 base within 4.2e-15. Co-Authored-By: Claude Opus 5.5 --- tools/us_acs_local_cd_surface.py | 50 ++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 6 deletions(-) diff --git a/tools/us_acs_local_cd_surface.py b/tools/us_acs_local_cd_surface.py index edd37055fa..11f6377cec 100644 --- a/tools/us_acs_local_cd_surface.py +++ b/tools/us_acs_local_cd_surface.py @@ -86,7 +86,11 @@ #: counts bridge by ``return_count`` (counts are not aged), amounts by the #: closest amount Historic Table 2 carries. The factor band below applies to #: the sibling. The bridge is stamp-invariant: if the stamp is corrected -#: (#1030) both levels move together. +#: (#1030) both levels move together. A sibling the compiler binds as a share +#: of a national control (``return_count``, ``adjusted_gross_income`` and +#: ``itemized_deductions_amount`` are ``_SOI_HT2_NATIONAL_CONTROLS`` columns) +#: is read at the file's own level (:func:`cd_file_level`), so the bridge +#: still measures the file's gap and not the control's. STATE_CD_LEVEL_BRIDGES: Mapping[str, str] = { "charitable_amount": "itemized_deductions_amount", "charitable_returns": "return_count", @@ -196,6 +200,33 @@ def is_cd_file_spec(spec) -> bool: return record_set_spec_id(spec).startswith(CD_FILE_RECORD_SET_SPEC_PREFIX) +def cd_file_level(spec, *, aged_like=None) -> float: + """A district-file spec's value on the district file's own basis. + + The compiler binds some district-file rows as shares of a national + control: such a row names its denominator + (``soi_share_total_source_record_id``) and its ``uprating_factor`` is + control / the file's US row (microcosm#1229, #1040, and the + ``_SOI_HT2_NATIONAL_CONTROLS`` columns). Dividing the factor out gives the + file's own level, which is what a level bridge compares with Historic + Table 2. A share ages from its control's period; with ``aged_like``, the + level is re-aged to that spec's aging factor, so it sits on the same + basis as an unrebased district-file row (the two differ once #1030 ages + the file from its data year). Other specs return their value. + """ + + metadata = _metadata(spec) + value = float(spec.value) + if not metadata.get("soi_share_total_source_record_id"): + return value + value /= float(metadata["uprating_factor"]) + if aged_like is not None: + value *= float(_metadata(aged_like).get("aging_factor", "1")) / float( + metadata.get("aging_factor", "1") + ) + return value + + def soi_materializer_semantics(spec) -> tuple: """Everything the SOI slice reads from a spec, except its geography. @@ -382,11 +413,12 @@ def drop(spec, reason: str) -> None: cd_file_state_by_key[key] = spec # Each state concept both files carry: its Historic Table 2 / district-file - # ratio, judged against the measure's median ratio across states. + # ratio (the file's own level, ``cd_file_level``), judged against the + # measure's median ratio across states. vintage_ratio: dict[tuple, float] = {} for key in ht2_by_key.keys() & cd_file_state_by_key.keys(): ht2_value = float(ht2_by_key[key].value) - cd_value = float(cd_file_state_by_key[key].value) + cd_value = cd_file_level(cd_file_state_by_key[key]) vintage_ratio[key] = ( ht2_value / cd_value if cd_value != 0.0 and (ht2_value > 0) == (cd_value > 0) @@ -480,12 +512,18 @@ def within_band( sibling_key = (state, (sibling, *identity[1:])) ht2_sibling = ht2_by_key.get(sibling_key) cd_sibling = cd_file_state_by_key.get(sibling_key) - if ht2_sibling is None or cd_sibling is None or float(cd_sibling.value) == 0: + # The bridged row is a district-file level, so the sibling must be + # too: a sibling the compiler bound as a national share is read at + # its own file level, aged as the bridged row is. + cd_sibling_level = ( + None if cd_sibling is None else cd_file_level(cd_sibling, aged_like=spec) + ) + if ht2_sibling is None or cd_sibling_level is None or cd_sibling_level == 0: raise ValueError( f"Level bridge {measure} -> {sibling} in state {state} needs " "the sibling's state total in both files." ) - factor = float(ht2_sibling.value) / float(cd_sibling.value) + factor = float(ht2_sibling.value) / cd_sibling_level # A sign conflict is a data defect, not a level disagreement: refuse # it before the band, which only judges finite positive ratios. if not (math.isfinite(factor) and factor > 0.0): @@ -498,7 +536,7 @@ def within_band( # rows) the rebase of its own district block. trusted = within_band( sibling_key, - factor, + vintage_ratio[sibling_key], median_ratio.get(sibling_key[1], math.nan), f"level_bridge:{measure}", measure, From c18aa20716fffcd3400ec344a6c515b8fa1ad40b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 19:21:41 -0400 Subject: [PATCH 03/12] Test the district national-control shares and pin them on the full surface Synthetic tests for the district pass (shares and declarations for all 11 carried columns, own denominators beside HT2, keep-level and drop cases, stamp invariance, the column guard before the crosswalk, denominator ties), a Hypothesis share-conservation property, a differential test against the HT2 pass, and a state_cd property that national-share district rows move no state_cd value or verdict. On the pinned feed: the district rows take their control level, every controlled quantity has one national value on full, and the compiled count re-pins at 33,224. Co-Authored-By: Claude Opus 5.5 --- .../us/test_us_acs_local_cd_properties.py | 124 ++++ .../engine_free/us/test_us_fiscal_targets.py | 142 +++- .../test_us_soi_district_national_controls.py | 696 ++++++++++++++++++ 3 files changed, 959 insertions(+), 3 deletions(-) create mode 100644 packages/microcosm-build/tests/engine_free/us/test_us_soi_district_national_controls.py diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_cd_properties.py b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_cd_properties.py index 243568e5e3..c4e03307b9 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_cd_properties.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_cd_properties.py @@ -850,3 +850,127 @@ def test_a_sign_conflict_or_a_self_contradicting_district_file_is_refused() -> N crosswalk=crosswalk, level_bridges={"charitable_amount": "return_count"}, ) + + +#: Exact binary scales, so a share rebase and its inverse round-trip without +#: rounding and the band's verdicts cannot flip at an edge. +_EXACT_SCALES = st.sampled_from([0.25, 0.5, 1.0, 2.0, 4.0]) + + +def _aged(spec, aging: float): + return dataclasses.replace( + spec, metadata={**spec.metadata, "aging_factor": repr(aging)} + ) + + +def _as_national_share(spec, factor: float, aging: float): + """``spec`` as the compiler binds it as a share of a national control: + its value times control / the file's US row (``factor``), aged from the + control's period (``aging``) instead of its own (#1250, #1229, #1040).""" + + own = float(spec.metadata["aging_factor"]) + value = spec.value * factor * aging / own + metadata = { + **spec.metadata, + "uprating_factor": repr(factor), + "soi_share_total_source_record_id": ( + f"cdfile.us.{spec.metadata['source_measure_id']}" + ), + "aging_factor": repr(aging), + } + return dataclasses.replace(spec, value=value, signed=value < 0, metadata=metadata) + + +def test_cd_file_level_undoes_a_national_share_and_its_aging() -> None: + row = _aged( + _soi_spec( + "cdfile.10_total.return_count", + "return_count", + 200.0, + level="state", + record_set=_CD_FILE, + state="10", + ), + 1.25, + ) + share = _as_national_share(row, 1.5, 1.0) + assert share.value == 240.0 + # The file's level, aged as the share is (from the control's period) ... + assert _CD.cd_file_level(share) == 160.0 + # ... or as an unrebased district-file row is. + assert _CD.cd_file_level(share, aged_like=row) == 200.0 + assert _CD.cd_file_level(row) == _CD.cd_file_level(row, aged_like=share) == 200.0 + + +@_SETTINGS +@given(_state_cd_inputs(), st.data()) +def test_national_share_district_rows_leave_the_state_cd_surface_unchanged( + inputs, data +) -> None: + """The compiler binds the district-file rows of some concepts as shares of + a national control: value x control / the file's US row, aged from the + control's period. ``state_cd`` uses the file for within-state shares, and + for level-bridge siblings through ``cd_file_level``, so the rebase moves + no ``state_cd`` value, drop or band verdict, with or without the band. + + Only concepts Historic Table 2 carries in every state are rebased here, + as the compiler's registered concepts are on the feed: a rebased concept + without an HT2 row in a state has no registered bridge, so it refuses.""" + + specs, crosswalk, _layout = inputs + states = {spec.metadata["state_fips"] for spec in specs} + ht2 = { + (spec.metadata["state_fips"], spec.metadata["source_measure_id"]) + for spec in specs + if spec.metadata["ledger_layout_record_set_spec_id"] == _HT2 + } + carried = sorted( + measure + for measure in {spec.metadata["source_measure_id"] for spec in specs} + if all((state, measure) in ht2 for state in states) + ) + rebased = data.draw(st.lists(st.sampled_from(carried), unique=True)) + factors = {measure: data.draw(_EXACT_SCALES) for measure in rebased} + own_aging, control_aging = data.draw(_EXACT_SCALES), data.draw(_EXACT_SCALES) + band = data.draw(st.sampled_from([math.inf, _CD.STATE_CD_FACTOR_BAND])) + + def is_cd_file(spec) -> bool: + return spec.metadata["ledger_layout_record_set_spec_id"] == _CD_FILE + + base = [_aged(spec, own_aging) if is_cd_file(spec) else spec for spec in specs] + shared = [ + _as_national_share( + spec, factors[spec.metadata["source_measure_id"]], control_aging + ) + if is_cd_file(spec) and spec.metadata["source_measure_id"] in factors + else spec + for spec in base + ] + + def surface(rows): + return _CD.state_cd_soi_surface( + rows, + state_surface_predicate=_TOOL.soi_surface_predicate("state"), + crosswalk=crosswalk, + level_bridges=_BRIDGES, + factor_band=band, + ) + + before, after = surface(base), surface(shared) + assert [spec.name for spec in after.specs] == [spec.name for spec in before.specs] + for old, new in zip(before.specs, after.specs, strict=True): + assert new.value == pytest.approx(old.value, rel=1e-12), new.name + for key in ("counts", "dropped"): + assert after.receipt[key] == before.receipt[key] + + def verdicts(receipt): + return sorted( + (row["state_fips"], row["measure"], row["basis"], row["verdict"]) + for row in receipt["factor_band"]["out_of_band"] + ) + + assert verdicts(after.receipt) == verdicts(before.receipt) + assert ( + after.receipt["level_bridge_factor_by_measure"] + == before.receipt["level_bridge_factor_by_measure"] + ) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_targets.py b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_targets.py index 9f9031f852..8d6bf3230c 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_targets.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_targets.py @@ -893,10 +893,12 @@ def test_pinned_feed_exclusion_receipt(pinned_feed_national_state_surface) -> No def test_pinned_feed_national_state_surface_restores_the_fences( pinned_feed_national_state_surface, ) -> None: - """The release surface on the pinned feed: 33,235 compiled targets after + """The release surface on the pinned feed: 33,224 compiled targets after Medicaid substitution, 6,087 national_state targets at registry 0e899cd7f893, 32 CHIP rows none of them for an M-CHIP state, no - other-income row and no tips return count. microcosm#1243 took it from + other-income row and no tips return count. microcosm#1250 retired the + congressional-district file's 11 US rows of the Table 1.x-controlled + columns (from 33,235; national_state unchanged). microcosm#1243 took it from 33,250 / 6,102 at 882f4764a248 by rebasing 14 Historic Table 2 state families onto their Table 1.x controls and retiring the 14 HT2 US rows and HT2's US-workbook return count. microcosm#1035 moved it from @@ -912,7 +914,7 @@ def test_pinned_feed_national_state_surface_restores_the_fences( ty2020 tips return count it was 32,843 / 5,695 at 386fac439e77 (docs/us-chronicle-feed-repin.md).""" registry, surface, _ = pinned_feed_national_state_surface - assert len(registry.specs) == 33_235 + assert len(registry.specs) == 33_224 assert len(surface.specs) == 6_087 assert surface.version == "0e899cd7f893" chip = [ @@ -1147,6 +1149,140 @@ def test_pinned_feed_ht2_state_rows_take_their_national_control_level( ] == [], measure_id +#: Per congressional-district measure on the pinned feed: the factor +#: control / the district file's US row (22incd.csv, TY2022 stamped ty2023). +#: The control is the measure's ``_PINNED_HT2_CONTROLS`` entry. +_PINNED_CD_FACTORS = { + "adjusted_gross_income": "1.05970317276013", + "itemized_deductions_amount": "1.09974729652659", + "return_count": "1.02724364722741", + "taxable_ira_distributions_amount": "1.0247107152496", + "taxable_ira_distributions_returns": "1.02876142432905", + "taxable_pension_income_amount": "1.05382110367184", + "taxable_pension_income_returns": "1.00136008211203", + "taxable_social_security_amount": "1.1692464093015", + "taxable_social_security_returns": "1.06412503801664", + "unemployment_compensation_amount": "1.0546342976899", + "unemployment_compensation_returns": "1.02721646869479", +} + + +def test_pinned_feed_district_rows_take_their_national_control_level( + pinned_feed_national_state_surface, +) -> None: + """On the full surface each of the 11 Table 1.x-controlled columns the + district file carries enters as 51 state and 436 district shares of the + file's US row times the control (microcosm#1250). The file's states and + its districts each sum to the control after aging, every district + reconciles to its state with factor 1, and the US row is gone.""" + registry, _, _ = pinned_feed_national_state_surface + by_name = {spec.name: spec for spec in registry.specs} + for measure_id, factor in _PINNED_CD_FACTORS.items(): + control_id = _PINNED_HT2_CONTROLS[measure_id][0] + control = by_name[control_id] + concept = fiscal_targets._SOI_HT2_NATIONAL_CONTROLS[measure_id].concept + rows = [ + spec + for spec in registry.specs + if spec.metadata.get("source_measure_id") == measure_id + and ".congressional_district_2022." in spec.name + ] + levels = { + level: [ + spec + for spec in rows + if spec.metadata["ledger_geography_level"] == level + ] + for level in ("country", "state", "congressional_district") + } + assert {level: len(found) for level, found in levels.items()} == { + "country": 0, + "state": 51, + "congressional_district": 436, + }, measure_id + for spec in rows: + metadata = spec.metadata + assert metadata["uprating_index_source_record_id"] == control_id + assert metadata["uprating_factor"] == factor + assert metadata["uprating_to_period"] == "2023" + assert metadata["soi_share_total_source_record_id"] == ( + f"irs_soi.ty2023.congressional_district_2022.all_returns.us.{measure_id}" + ) + assert metadata["aging_factor"] == control.metadata["aging_factor"] + assert metadata["soi_source_concept"] == concept + assert metadata["soi_control_concept"] == concept + for level in ("state", "congressional_district"): + total = math.fsum(spec.value for spec in levels[level]) + assert math.isclose(total, control.value, rel_tol=1e-12), ( + measure_id, + level, + ) + assert { + spec.metadata["hierarchy_reconciliation_factor"] + for spec in levels["congressional_district"] + } == {"1"} + + +def test_pinned_feed_full_surface_has_one_national_value_per_controlled_quantity( + pinned_feed_national_state_surface, +) -> None: + """Invariant on the full surface: every model quantity a + ``_SOI_HT2_NATIONAL_CONTROLS`` control measures has one national all-AGI + value, its control's. Before microcosm#1250 the district file's US row + was a second value for 11 of them, 1.001x (taxable pension returns) to + 1.169x (taxable Social Security) below the control. Table 1.2 repeats + Table 1.1's return count and AGI at the same value, and no state or + district family of these quantities sums past it.""" + registry, _, _ = pinned_feed_national_state_surface + controls = { + spec.metadata["source_measure_id"]: spec + for spec in registry.specs + if spec.name in {entry[0] for entry in _PINNED_HT2_CONTROLS.values()} + } + quantities = { + fiscal_targets._soi_national_quantity(spec): spec for spec in controls.values() + } + assert len(quantities) == 14 + national: dict[tuple[object, ...], list[object]] = {} + families: dict[tuple[object, ...], float] = {} + for spec in registry.specs: + if spec.family != "irs_soi": + continue + quantity = fiscal_targets._soi_national_quantity(spec) + if quantity not in quantities: + continue + if not spec.metadata.get("state_fips"): + national.setdefault(quantity, []).append(spec) + continue + key = ( + quantity, + fiscal_targets._soi_record_set_family( + spec.metadata.get("ledger_layout_record_set_id", "") + ), + spec.metadata["ledger_geography_level"], + ) + families[key] = families.get(key, 0.0) + spec.value + assert set(national) == set(quantities) + for quantity, specs in national.items(): + control = quantities[quantity] + assert { + spec.name: spec.value + for spec in specs + if not math.isclose(spec.value, control.value, rel_tol=1e-12) + } == {}, control.name + assert not [ + spec.name for spec in specs if ".congressional_district_" in spec.name + ] + assert { + key[1:]: total / quantities[key[0]].value + for key, total in families.items() + if total > quantities[key[0]].value * (1 + 1e-12) + } == {} + # Both the district file's states and its districts, and HT2's states, + # are compared for each of the 11 carried quantities. + assert len(families) == 14 + 11 * 2 + 11 + + def test_reviewed_zero_support_facts_are_not_active_targets() -> None: excluded_source_record_id = ( "hhs_acf_tanf.fy2024.cash_assistance.ar." diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_soi_district_national_controls.py b/packages/microcosm-build/tests/engine_free/us/test_us_soi_district_national_controls.py new file mode 100644 index 0000000000..a4d9bb56e3 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/us/test_us_soi_district_national_controls.py @@ -0,0 +1,696 @@ +"""Congressional-district rows rebased onto the national controls (#1250). + +The congressional-district file (``22incd.csv``) publishes the columns of +``_SOI_HT2_NATIONAL_CONTROLS`` at the same Form 1040 lines, so its state and +district rows are shares of the nation and its US row is a second, narrower +national level beside the Table 1.x control. +``_rebase_stale_soi_district_distributions`` takes the rows as shares of the +file's own US row times that control and retires the US row. The synthetic +tests below pin the rule and its invariants; the pinned-feed checks sit +beside the release fixture in ``test_us_fiscal_targets.py``. +""" + +from __future__ import annotations + +import math + +import pytest + +from microcosm.build.us_runtime import compile_us_fiscal_target_registry, fiscal_targets +from test_support.microcosm_build.us_fiscal_targets import ( + _dynamic_ledger_fact, + packaged_reference_facts, +) + +REGISTER = fiscal_targets._SOI_HT2_NATIONAL_CONTROLS +#: The measures the TY2022 district package carries: the register less wages +#: and taxable income, which it does not publish. +CD_MEASURES = sorted( + set(REGISTER) + - {"wages_salaries_amount", "wages_salaries_returns", "taxable_income_amount"} +) +#: States with two districts in the 117th and 119th plans, so a state row +#: and both its districts make a complete hierarchy block. +STATES = ( + ("hi", "15", ("1501", "1502")), + ("nh", "33", ("3301", "3302")), + ("me", "23", ("2301", "2302")), +) +_REVIEWED = object() + + +def _record_set(stamp: int) -> str: + return f"irs_soi.ty{stamp}.congressional_district_2022.all_returns" + + +def _us_id(measure_id: str, stamp: int = 2023, row: str = "us") -> str: + return f"{_record_set(stamp)}.{row}.{measure_id}" + + +def _cd_fact( + measure_id: str, + value: float, + *, + row: str, + level: str, + geography_id: str, + stamp: int = 2023, + column: object = _REVIEWED, +) -> dict[str, object]: + fact = _dynamic_ledger_fact( + source_record_id=f"{_record_set(stamp)}.{row}.{measure_id}", + source_name="irs_soi", + measure_id=measure_id, + value=value, + period_value=stamp, + geography_level=level, + geography_id=geography_id, + dimensions={"income_range": "all", "filing_status": "all"}, + layout_record_set_id=_record_set(stamp), + groupby_dimension="irs_soi.congressional_district", + groupby_value_id=row, + ) + if column is _REVIEWED: + column = REGISTER[measure_id].source_column + if column is not None: + fact["layout"]["source_column_id"] = column + return fact + + +def _cd_facts( + measure_id: str, + *, + stamp: int = 2023, + us: float | None = 1_000.0, + districts: tuple[tuple[float, float], ...] = ((300.0, 200.0), (150.0, 100.0)), + column: object = _REVIEWED, +) -> list[dict[str, object]]: + """The district file's US row and, per state, its total and two districts.""" + facts = [] + if us is not None: + facts.append( + _cd_fact( + measure_id, + us, + row="us", + level="country", + geography_id="0100000US", + stamp=stamp, + column=column, + ) + ) + for (postal, fips, geoids), values in zip(STATES, districts, strict=False): + facts.append( + _cd_fact( + measure_id, + math.fsum(values), + row=f"{postal}_total", + level="state", + geography_id=f"0400000US{fips}", + stamp=stamp, + column=column, + ) + ) + for geoid, value in zip(geoids, values, strict=True): + facts.append( + _cd_fact( + measure_id, + value, + row=f"{postal}_{geoid[2:]}", + level="congressional_district", + geography_id=f"5001700US{geoid}", + stamp=stamp, + column=column, + ) + ) + return facts + + +def _control_fact( + measure_id: str, *, period: int = 2023, value: float = 1_100.0 +) -> dict[str, object]: + rule = REGISTER[measure_id] + suffix = ".all_returns" if rule.control_family == "table_1_2" else "" + record_set_id = f"irs_soi.ty{period}.{rule.control_family}{suffix}" + return _dynamic_ledger_fact( + source_record_id=f"{record_set_id}.all.{rule.control_measure_id}", + source_name="irs_soi", + measure_id=rule.control_measure_id, + value=value, + period_value=period, + dimensions={"income_range": "all", "filing_status": "all"}, + layout_record_set_id=record_set_id, + groupby_dimension="us:statutes/26/62#adjusted_gross_income", + groupby_value_id="all", + ) + + +def _control_id(measure_id: str, period: int = 2023) -> str: + return _control_fact(measure_id, period=period)["lineage"]["source_record_id"] + + +def _ht2_facts( + measure_id: str, *, us: float, states: tuple[float, ...], period: int = 2022 +) -> list[dict[str, object]]: + """An HT2 US row and state rows for the first ``len(states)`` STATES.""" + facts = [ + _dynamic_ledger_fact( + source_record_id=f"irs_soi.ty{period}.historic_table_2.us.all.{measure_id}", + source_name="irs_soi", + measure_id=measure_id, + value=us, + period_value=period, + dimensions={"income_range": "all", "filing_status": "all"}, + layout_record_set_id=f"irs_soi.ty{period}.historic_table_2.us", + groupby_dimension="us:statutes/26/62#adjusted_gross_income", + groupby_value_id="all", + ) + ] + for (postal, fips, _), value in zip(STATES, states, strict=False): + record_set_id = f"irs_soi.ty{period}.historic_table_2.state_broad.{postal}" + facts.append( + _dynamic_ledger_fact( + source_record_id=f"{record_set_id}.all.{measure_id}", + source_name="irs_soi", + measure_id=measure_id, + value=value, + period_value=period, + geography_level="state", + geography_id=f"0400000US{fips}", + dimensions={"income_range": "all", "filing_status": "all"}, + layout_record_set_id=record_set_id, + groupby_dimension="us:statutes/26/62#adjusted_gross_income", + groupby_value_id="all", + ) + ) + return facts + + +def _ht2_state_id(measure_id: str, postal: str, period: int = 2022) -> str: + return f"irs_soi.ty{period}.historic_table_2.state_broad.{postal}.all.{measure_id}" + + +def _compile(facts, *, crosswalk=None): + return compile_us_fiscal_target_registry( + [*packaged_reference_facts(), *facts], + target_period=2024, + congressional_district_vintage_crosswalk=crosswalk, + allow_unaged_dollar_targets=True, + ) + + +def _by_record(registry) -> dict[str, object]: + return {spec.metadata["ledger_source_record_id"]: spec for spec in registry.specs} + + +def _cd_rows(registry, measure_id: str) -> dict[str, object]: + """The district-file rows of one measure, keyed by their row id.""" + return { + spec.metadata["ledger_source_record_id"].split(".")[-2]: spec + for spec in registry.specs + if spec.metadata.get("source_measure_id") == measure_id + and fiscal_targets._soi_record_set_family( + spec.metadata.get("ledger_layout_record_set_id", "") + ) + == "congressional_district" + } + + +def _national_rows(registry, quantity) -> list[object]: + return [ + spec + for spec in registry.specs + if spec.family == "irs_soi" + and not spec.metadata.get("state_fips") + and fiscal_targets._soi_national_quantity(spec) == quantity + ] + + +# -------------------------------------------------------------------------- +# The register + + +#: Each registered measure's IRS column in ``22incddocguide.docx`` (TY2022), +#: with the Form 1040 line it gives; the HT2 guides give the same column at +#: the same line (``docs/us-soi-national-controls.md``). +_DISTRICT_GUIDE_COLUMNS = { + "return_count": "N1", # number of returns + "adjusted_gross_income": "A00100", # 1040:11 + "wages_salaries_amount": "A00200", # 1040:1z + "wages_salaries_returns": "N00200", # 1040:1z + "taxable_ira_distributions_amount": "A01400", # 1040:4b + "taxable_ira_distributions_returns": "N01400", # 1040:4b + "taxable_pension_income_amount": "A01700", # 1040:5b + "taxable_pension_income_returns": "N01700", # 1040:5b + "unemployment_compensation_amount": "A02300", # Schedule 1:7 + "unemployment_compensation_returns": "N02300", # Schedule 1:7 + "taxable_social_security_amount": "A02500", # 1040:6b + "taxable_social_security_returns": "N02500", # 1040:6b + "itemized_deductions_amount": "A04470", # 1040:12 + "taxable_income_amount": "A04800", # 1040:15 +} + + +def test_the_register_names_each_columns_reviewed_irs_id() -> None: + assert { + measure_id: rule.source_column for measure_id, rule in REGISTER.items() + } == _DISTRICT_GUIDE_COLUMNS + assert len(CD_MEASURES) == 11 + + +# -------------------------------------------------------------------------- +# The rebase + + +@pytest.mark.parametrize("measure_id", CD_MEASURES) +def test_district_rows_become_shares_of_their_national_control( + measure_id: str, +) -> None: + """A US row of 1,000 with HI 300 + 200 and NH 150 + 100 under an 1,100 + control: every row gains x1.1, the US row retires, and the control's row + is the quantity's only national level.""" + registry = _compile([*_cd_facts(measure_id), _control_fact(measure_id)]) + rows = _cd_rows(registry, measure_id) + + assert "us" not in rows + expected = { + "hi_total": 550.0, + "hi_01": 330.0, + "hi_02": 220.0, + "nh_total": 275.0, + "nh_01": 165.0, + "nh_02": 110.0, + } + assert {row: spec.value for row, spec in rows.items()} == pytest.approx( + expected, rel=1e-12 + ) + concept = REGISTER[measure_id].concept + for row, spec in rows.items(): + metadata = spec.metadata + assert metadata["uprating_factor"] == "1.1" + assert metadata["uprating_index"] == f"total_{measure_id}" + assert metadata["uprating_from_period"] == "2023" + assert metadata["uprating_to_period"] == "2023" + assert metadata["uprating_index_source_record_id"] == _control_id(measure_id) + assert metadata["soi_share_total_source_record_id"] == _us_id(measure_id) + assert metadata["soi_source_concept"] == concept + assert metadata["soi_control_concept"] == concept + assert metadata["stale_distribution_rebased_to_active_total"] == "true" + if row.endswith(("_01", "_02")): + # One factor scales the states and their districts alike. + assert metadata["hierarchy_reconciliation_factor"] == "1" + assert [ + spec.metadata["ledger_source_record_id"] + for spec in _national_rows( + registry, fiscal_targets._soi_national_quantity(rows["hi_total"]) + ) + ] == [_control_id(measure_id)] + + +def test_each_family_divides_by_its_own_us_row() -> None: + """HT2 rows divide by the HT2 US row and district rows by the district + US row, never by the other's; both US rows retire.""" + measure_id = "taxable_social_security_amount" + registry = _compile( + [ + *_cd_facts(measure_id), + *_ht2_facts(measure_id, us=900.0, states=(400.0, 200.0)), + _control_fact(measure_id), + ] + ) + specs = _by_record(registry) + rows = _cd_rows(registry, measure_id) + + assert specs[_ht2_state_id(measure_id, "hi")].value == pytest.approx( + 400.0 * 1_100.0 / 900.0, rel=1e-12 + ) + assert rows["hi_total"].value == pytest.approx(550.0, rel=1e-12) + assert rows["hi_total"].metadata["soi_share_total_source_record_id"] == _us_id( + measure_id + ) + assert [ + spec.metadata["ledger_source_record_id"] + for spec in _national_rows( + registry, fiscal_targets._soi_national_quantity(rows["hi_total"]) + ) + ] == [_control_id(measure_id)] + + +@pytest.mark.parametrize( + "control", + [ + None, + # Older than the row's (stamped) period: the share would run backwards. + {"period": 2022}, + # After the build period: not yet a level for a 2024 build. + {"period": 2025}, + ], + ids=["no-control", "older-control", "future-control"], +) +def test_district_rows_keep_their_level_without_an_eligible_control(control) -> None: + measure_id = "adjusted_gross_income" + facts = _cd_facts(measure_id) + if control is not None: + facts.append(_control_fact(measure_id, **control)) + rows = _cd_rows(_compile(facts), measure_id) + + assert rows["us"].value == 1_000.0 + assert rows["hi_total"].value == 500.0 + assert rows["hi_01"].value == 300.0 + assert "uprating_factor" not in rows["hi_total"].metadata + + +@pytest.mark.parametrize("us", [None, 0.0], ids=["no-us-row", "zero-us-row"]) +def test_district_rows_without_a_denominator_are_dropped_beside_a_control(us) -> None: + """A row with an eligible control but no nonzero US row of its own file + cannot be a share, and as a level it would be the stale second value the + pass removes, so it is dropped (as #1229 and #1040 do).""" + measure_id = "return_count" + registry = _compile([*_cd_facts(measure_id, us=us), _control_fact(measure_id)]) + assert _cd_rows(registry, measure_id) == {} + assert _control_id(measure_id) in _by_record(registry) + + +def test_the_share_ignores_the_package_stamp() -> None: + """The feed stamps 22incd.csv ty2023; stamped truthfully at 2022 the + shares and their landing period are the same, because the stamp cancels + in row / US row.""" + measure_id = "unemployment_compensation_amount" + stamped, truthful = ( + _cd_rows( + _compile([*_cd_facts(measure_id, stamp=stamp), _control_fact(measure_id)]), + measure_id, + ) + for stamp in (2023, 2022) + ) + assert set(stamped) == set(truthful) + for row, spec in stamped.items(): + assert spec.value == truthful[row].value + assert spec.metadata["uprating_to_period"] == "2023" + assert truthful[row].metadata["uprating_to_period"] == "2023" + assert truthful[row].metadata["uprating_from_period"] == "2022" + assert truthful[row].metadata["soi_share_total_source_record_id"] == _us_id( + measure_id, stamp=2022 + ) + + +# -------------------------------------------------------------------------- +# The column guard + + +@pytest.mark.parametrize("column", ["N2", "A00101", None], ids=str) +def test_a_district_fact_under_an_unreviewed_column_refuses(column) -> None: + """Rebasing a measure id that reads another IRS column (or none) would + bind the wrong concept to the control, so the compile refuses.""" + measure_id = "return_count" + facts = _cd_facts(measure_id) + facts[1]["layout"].pop("source_column_id") + if column is not None: + facts[1]["layout"]["source_column_id"] = column + with pytest.raises( + fiscal_targets.UnreviewedSoiNationalControlColumnError, + match=facts[1]["lineage"]["source_record_id"], + ): + _compile([*facts, _control_fact(measure_id)]) + # Without a control nothing is rebased, so nothing new is claimed. + rows = _cd_rows(_compile(facts), measure_id) + assert rows["hi_total"].value == 500.0 + + +def test_the_column_guard_sees_each_source_district_before_the_crosswalk() -> None: + """The crosswalk merges source districts into one fact that keeps the + first contributor's layout, so a mislabeled second contributor would hide + behind a reviewed column; the guard runs on the facts as fed.""" + measure_id = "taxable_pension_income_amount" + facts = _cd_facts(measure_id, districts=((300.0, 200.0),)) + hi_02 = next( + fact + for fact in facts + if fact["lineage"]["source_record_id"].endswith(f"hi_02.{measure_id}") + ) + hi_02["layout"]["source_column_id"] = "A01750" + crosswalk = [ + { + "source_geography_id": "5001700US1501", + "target_geography_id": "5001900US1501", + "weight": 1.0, + }, + { + "source_geography_id": "5001700US1502", + "target_geography_id": "5001900US1501", + "weight": 1.0, + }, + ] + with pytest.raises(fiscal_targets.UnreviewedSoiNationalControlColumnError): + _compile([*facts, _control_fact(measure_id)], crosswalk=crosswalk) + + +# -------------------------------------------------------------------------- +# The denominator + + +def test_conflicting_district_us_totals_refuse_the_compile() -> None: + """Two US rows of one key with different values would let feed order pick + the denominator.""" + measure_id = "taxable_ira_distributions_amount" + us, *rest = _cd_facts(measure_id) + rival = _cd_fact( + measure_id, 900.0, row="us_v2", level="country", geography_id="0100000US" + ) + for ordered in ((us, rival), (rival, us)): + with pytest.raises( + fiscal_targets.AmbiguousSoiNationalControlError, + match="congressional-district US total", + ): + fiscal_targets._soi_district_us_totals(ordered) + with pytest.raises(fiscal_targets.AmbiguousSoiNationalControlError): + _compile([rival, us, *rest, _control_fact(measure_id)]) + + +def test_equal_district_us_totals_name_the_smaller_record_id() -> None: + """Equal twins are one total; the named denominator is the smaller record + id in either feed order, and both twins retire.""" + measure_id = "taxable_ira_distributions_returns" + us, *rest = _cd_facts(measure_id) + twin = _cd_fact( + measure_id, 1_000.0, row="us_2", level="country", geography_id="0100000US" + ) + named = set() + for ordered in ((us, twin), (twin, us)): + registry = _compile([*ordered, *rest, _control_fact(measure_id)]) + rows = _cd_rows(registry, measure_id) + assert not {"us", "us_2"} & set(rows) + named |= { + spec.metadata["soi_share_total_source_record_id"] for spec in rows.values() + } + assert named == {min(_us_id(measure_id), _us_id(measure_id, row="us_2"))} + + +# -------------------------------------------------------------------------- +# Invariants over compiles + + +def test_district_shares_conserve_the_control_and_leave_one_national_level() -> None: + """Property over compiles, for every carried column, either package + stamp, any control vintage (or none), any states, other-area residual, + HT2 rows alongside or not, an equal twin US row or not, and any feed + order: + + * **Shares conserved.** With an eligible control (not after the build + period, not before the row's period), each district-file row is its + value times control / the file's US row, and names the (smallest) US + row id as its denominator. The states sum to the control times their + share of the US row, so never past the control, and each state's + districts sum to the state with reconciliation factor 1. + * **Own denominator.** HT2 rows divide by the HT2 US row. + * **One national level.** The control's row is the only national + all-AGI row of the quantity: every district US row and the HT2 US row + retire. + * **No eligible control, no change.** The rows and the US row keep + their levels. + * **Order-free.** Shuffling the facts compiles the same registry. + """ + pytest.importorskip("hypothesis") + from hypothesis import given, settings + from hypothesis import strategies as st + + pairs = st.tuples( + st.integers(min_value=1, max_value=5_000_000), + st.integers(min_value=1, max_value=5_000_000), + ) + + @settings(max_examples=40, deadline=None) + @given( + st.sampled_from(CD_MEASURES), + st.sampled_from((2022, 2023)), + st.one_of(st.none(), st.integers(min_value=2020, max_value=2026)), + st.integers(min_value=1, max_value=50_000_000), + st.lists(pairs, min_size=1, max_size=3), + st.integers(min_value=0, max_value=5_000_000), + st.booleans(), + st.sampled_from((None, "aa_us", "zz_us")), + st.randoms(use_true_random=False), + ) + def check( + measure_id, + stamp, + control_period, + control_value, + districts, + other_areas, + with_ht2, + twin_row, + rng, + ) -> None: + state_totals = [math.fsum(pair) for pair in districts] + us_value = float(sum(state_totals) + other_areas) + facts = _cd_facts( + measure_id, + stamp=stamp, + us=us_value, + districts=tuple(tuple(map(float, pair)) for pair in districts), + ) + if twin_row is not None: + facts.append( + _cd_fact( + measure_id, + us_value, + row=twin_row, + level="country", + geography_id="0100000US", + stamp=stamp, + ) + ) + ht2_states = tuple(2.0 * total + 1.0 for total in state_totals) + ht2_us = math.fsum(ht2_states) + 7.0 + if with_ht2: + facts += _ht2_facts(measure_id, us=ht2_us, states=ht2_states) + if control_period is not None: + facts.append( + _control_fact( + measure_id, period=control_period, value=float(control_value) + ) + ) + shuffled = list(facts) + rng.shuffle(shuffled) + registry = _compile(facts) + assert sorted( + (spec.name, spec.value, tuple(sorted(spec.metadata.items()))) + for spec in registry.specs + ) == sorted( + (spec.name, spec.value, tuple(sorted(spec.metadata.items()))) + for spec in _compile(shuffled).specs + ) + + rows = _cd_rows(registry, measure_id) + raw = { + fact["layout"]["groupby_value_id"]: float(fact["value"]) + for fact in facts + if fact["layout"]["record_set_id"] == _record_set(stamp) + } + eligible = control_period is not None and stamp <= control_period <= 2024 + us_rows = {"us", *([twin_row] if twin_row else [])} + if not eligible: + assert {row: spec.value for row, spec in rows.items()} == raw + assert not any("uprating_factor" in s.metadata for s in rows.values()) + return + + factor = control_value / us_value + assert not us_rows & set(rows) + assert set(rows) == set(raw) - us_rows + denominator = min(_us_id(measure_id, stamp, row) for row in us_rows) + for row, spec in rows.items(): + assert math.isclose(spec.value, raw[row] * factor, rel_tol=1e-12) + assert spec.metadata["soi_share_total_source_record_id"] == denominator + assert spec.metadata["uprating_to_period"] == str(control_period) + state_sum = math.fsum( + spec.value for row, spec in rows.items() if row.endswith("_total") + ) + assert math.isclose( + state_sum, + control_value * sum(state_totals) / us_value, + rel_tol=1e-12, + abs_tol=1e-9, + ) + assert state_sum <= control_value * (1 + 1e-12) + for postal, _, _ in STATES[: len(districts)]: + children = [rows[f"{postal}_01"], rows[f"{postal}_02"]] + assert math.isclose( + math.fsum(spec.value for spec in children), + rows[f"{postal}_total"].value, + rel_tol=1e-12, + abs_tol=1e-9, + ) + for spec in children: + assert spec.metadata.get("hierarchy_reconciliation_factor") in ( + None, + "1", + ) + if with_ht2: + specs = _by_record(registry) + for (postal, _, _), value in zip(STATES, ht2_states, strict=False): + assert math.isclose( + specs[_ht2_state_id(measure_id, postal)].value, + value * control_value / ht2_us, + rel_tol=1e-12, + ) + quantity = fiscal_targets._soi_national_quantity(rows[f"{STATES[0][0]}_total"]) + assert [ + spec.metadata["ledger_source_record_id"] + for spec in _national_rows(registry, quantity) + ] == [_control_id(measure_id, control_period)] + + check() + + +def test_the_district_and_hts_passes_agree_on_the_same_shares() -> None: + """Differential: the HT2 pass (#1247) and the district pass implement one + share rule. Fed the same state values and US total under both families, + at the same period, they rebase each state to the same value with the + same factor.""" + pytest.importorskip("hypothesis") + from hypothesis import given, settings + from hypothesis import strategies as st + + @settings(max_examples=40, deadline=None) + @given( + st.sampled_from(CD_MEASURES), + st.sampled_from((2022, 2023)), + st.lists( + st.integers(min_value=1, max_value=10_000_000), min_size=1, max_size=3 + ), + st.integers(min_value=0, max_value=5_000_000), + st.integers(min_value=1, max_value=50_000_000), + ) + def check(measure_id, period, state_values, other_areas, control_value) -> None: + us = float(2 * sum(state_values) + other_areas) + # Each district-file state is two equal districts, so its total is + # the HT2 state value. + facts = [ + *_cd_facts( + measure_id, + stamp=period, + us=us, + districts=tuple((float(value), float(value)) for value in state_values), + ), + *_ht2_facts( + measure_id, + us=us, + states=tuple(2.0 * value for value in state_values), + period=period, + ), + _control_fact(measure_id, period=2023, value=float(control_value)), + ] + registry = _compile(facts) + specs = _by_record(registry) + rows = _cd_rows(registry, measure_id) + for postal, _, _ in STATES[: len(state_values)]: + ht2 = specs[_ht2_state_id(measure_id, postal, period)] + cd = rows[f"{postal}_total"] + assert cd.value == ht2.value + assert cd.metadata["uprating_factor"] == ht2.metadata["uprating_factor"] + assert ( + cd.metadata["uprating_to_period"] == ht2.metadata["uprating_to_period"] + ) + + check() From fc31412387fdb9be91d7d6091c7db43b5b9ecf74 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 20:07:17 -0400 Subject: [PATCH 04/12] Document the district national-control shares and harden the state_cd checks docs/us-soi-national-controls.md gains a congressional-district section: the guide check per column, the district US row's same-year coverage (0.940 to 0.997 of Table 1.4), the rule, and the pinned-feed effect. The ACS-local surface doc explains why level bridges read a rebased sibling at the file's own level and refreshes the bridge figures. The state_cd property now also checks the band's relative gaps and recorded sibling ratios, and an example pins a national-share sibling's bridges and recorded ratios. Co-Authored-By: Claude Opus 5.5 --- .../1250-cd-national-controls.fixed.md | 1 + docs/us-acs-local-soi-target-surface.md | 32 +++- docs/us-chronicle-feed-repin.md | 5 +- docs/us-fact-to-target.md | 9 ++ docs/us-soi-national-controls.md | 146 +++++++++++++++++- .../us/test_us_acs_local_cd_properties.py | 62 ++++++++ 6 files changed, 245 insertions(+), 10 deletions(-) create mode 100644 changelog.d/1250-cd-national-controls.fixed.md diff --git a/changelog.d/1250-cd-national-controls.fixed.md b/changelog.d/1250-cd-national-controls.fixed.md new file mode 100644 index 0000000000..210dcbbc95 --- /dev/null +++ b/changelog.d/1250-cd-national-controls.fixed.md @@ -0,0 +1 @@ +Rebase the congressional-district file's state and district rows of the 11 Table 1.x-controlled SOI columns (return count, AGI, taxable IRA distributions, taxable pensions, unemployment compensation and taxable Social Security, returns and amounts; itemized deductions) as shares of the file's own US row times their #1247 control, and retire the district US row. On the pinned feed's `full` surface no controlled quantity binds two national values any more (control / district US row was 1.001 to 1.169); the district file's 51-state sums move from +0.14% (taxable pension returns) to +16.9% (taxable Social Security) and its states and districts each sum to the control. A district fact that would be rebased under an unreviewed IRS column refuses the compile. `national_state` and every ACS-local `state_cd` value are unchanged; the `state_cd` level bridges read their rebased siblings at the district file's own level (microcosm#1250). diff --git a/docs/us-acs-local-soi-target-surface.md b/docs/us-acs-local-soi-target-surface.md index e072362569..7626e5efd2 100644 --- a/docs/us-acs-local-soi-target-surface.md +++ b/docs/us-acs-local-soi-target-surface.md @@ -201,6 +201,15 @@ district rows, are mostly 1.00 to 1.10. (#1030) would remove the aging part only. - **Two factors reflect known level differences:** taxable interest (2.35 to 2.70, the Table 4.3 rebase) and capital-gains amounts (0.77 to 1.00). +- **The 11 Table 1.x-controlled columns** (return count, AGI, taxable IRA + distributions, pensions, unemployment compensation and Social Security, + itemized deductions) are compiled on both sides as shares of the same + national control: Historic Table 2's since #1247, the district file's since + #1250 ([us-soi-national-controls.md](us-soi-national-controls.md)). Their + factors therefore carry neither the coverage nor the aging gap, only where + the two files disagree about a state's share: 0.975 to 1.063 (medians + 0.995 to 1.009). The two bullets above describe the files before those + rebases. - **The largest single-state factors** were on qualified and ordinary dividends (Hawaii, 2.68 and 2.17), tax-exempt interest (Utah, 1.43) and rental income (New York, 0.72), where the two tables disagree about one @@ -249,8 +258,8 @@ On the pinned feed the band drops eight: | Hawaii | `ordinary_dividends_amount` | rebase | 2.17 | 1.07 | 2.03 | | New York | `rental_royalty_income_amount` | rebase | 0.72 | 1.05 | 0.68 | | Utah | `tax_exempt_interest_amount` | rebase | 1.43 | 1.09 | 1.32 | -| Wyoming | `charitable_amount`, `interest_paid_deduction_amount` | bridge (itemized) | 1.75 | 1.06 | 1.65 | -| South Dakota | `charitable_amount`, `interest_paid_deduction_amount` | bridge (itemized) | 1.44 | 1.06 | 1.36 | +| Wyoming | `charitable_amount`, `interest_paid_deduction_amount` | bridge (itemized) | 1.81 | 1.09 | 1.65 | +| South Dakota | `charitable_amount`, `interest_paid_deduction_amount` | bridge (itemized) | 1.49 | 1.09 | 1.36 | That removes 34 district rows (Hawaii 2 x 2, New York 26, Utah 4) and four bridged state rows (Wyoming and South Dakota are at-large, so they had no @@ -276,14 +285,25 @@ The bridge assumes the measure shares its sibling's Historic Table 2 / district-file ratio in that state: the coverage and aging gap above, plus whatever the two publications disagree about for the sibling. It is an estimate, not an identity, which is why the factor band applies to it too. -Without it these state rows and their 2,562 district rows would sit 2–6% +Without it these state rows and their 2,562 district rows would sit 2–10% below every related concept. On the pinned feed the bridge factors are -1.00–1.03 for counts (median 1.016), 1.03–1.75 for the itemized-deduction -amounts (median 1.062; the two above 1.25x the median are dropped) and -1.03–1.08 for QBI amounts (median 1.049). The bridge is stamp-invariant: if #1030 corrects the +1.006–1.032 for counts (median 1.022), 1.06–1.81 for the itemized-deduction +amounts (median 1.095; the two above 1.25x the median are dropped) and +1.03–1.09 for QBI amounts (median 1.053), as measured with #1247 and #1250 +applied. The bridge is stamp-invariant: if #1030 corrects the stamp, the sibling ratio shrinks with it. Its district rows keep their within-state shares, as every other district row does. +All three siblings are columns the compiler binds as shares of a national +control (#1250), so the district file's sibling row no longer sits at the +file's level. The bridge reads it there anyway (`cd_file_level`: the share's +`uprating_factor` divided out, re-aged as the bridged row is), and the +sibling's band verdict compares the two files' levels. Reading the share as +compiled would lift nothing: on the pinned feed the bridged charitable and +interest-paid amounts would fall 9.07%, QBI amounts 5.63% and the bridged +counts 2.65%. A property test checks that national-share district rows move +no `state_cd` value, drop or verdict. + Every district row records its parent (`state_cd_parent_target_name`), the parent's basis, the district file's own value and the rebase factor; a bridged state row records its sibling and bridge factor. diff --git a/docs/us-chronicle-feed-repin.md b/docs/us-chronicle-feed-repin.md index d0b444fb8d..e574b9a247 100644 --- a/docs/us-chronicle-feed-repin.md +++ b/docs/us-chronicle-feed-repin.md @@ -216,7 +216,10 @@ same 60 return counts move and `national_state` goes from `e1f9906bf0e0` to `882f4764a248` at 6,102 targets. microcosm#1243 then rebases 14 more Historic Table 2 state families onto their Table 1.x controls and retires 15 stale national rows: 33,235 compiled and 6,087 on `national_state` (`0e899cd7f893`) -([us-soi-national-controls.md](us-soi-national-controls.md)). +([us-soi-national-controls.md](us-soi-national-controls.md)). microcosm#1250 +takes the congressional-district rows of 11 of those columns as shares of the +same controls and retires the district file's 11 US rows: 33,224 compiled, +`national_state` unchanged. The labelled feed exposed one latent defect in microcosm, fixed in this change: `apply_us_medicaid_enrollment_substitutions` built Rhode Island's diff --git a/docs/us-fact-to-target.md b/docs/us-fact-to-target.md index 1f9e11b8a1..241a95d0aa 100644 --- a/docs/us-fact-to-target.md +++ b/docs/us-fact-to-target.md @@ -132,6 +132,15 @@ the Table 1.x column of the same Form 1040 line. - `test_pinned_feed_no_soi_state_family_sums_past_its_national_target` fails a re-pin that adds a second national value for a compared quantity, or a state family past its national target. +- **The congressional-district file** carries 11 of the columns. + `_rebase_stale_soi_district_distributions` takes its state and district + rows as shares of its own US row times the same control and retires that + US row (#1250). Register a column only once the district documentation + guide defines it at the same line under the register's `source_column`; + a district fact that reads another column refuses the compile. On `full`, + `test_pinned_feed_full_surface_has_one_national_value_per_controlled_quantity` + fails a re-pin that leaves a second national value for a controlled + quantity, or a state or district family past it. ### State AGI bands bind as shares of the state total (microcosm#940) diff --git a/docs/us-soi-national-controls.md b/docs/us-soi-national-controls.md index ef1841ce23..e473a2dc48 100644 --- a/docs/us-soi-national-controls.md +++ b/docs/us-soi-national-controls.md @@ -4,6 +4,8 @@ Fourteen IRS SOI quantities reach the US target surface from two sources: the Historic Table 2 (HT2) state file and a Pub 1304 Table 1.x national table. This page records why the two count the same thing, which one supplies the national level, and how `fiscal_targets.py` combines them (microcosm#1243). +The congressional-district file carries 11 of the same columns; [its +rows](#congressional-district-rows-1250) take the same controls (#1250). The taxable-interest (#489, #518) and capital-gains (#1035, #1036) families follow the same doctrine through their own passes. @@ -186,6 +188,141 @@ crosswalk, then narrowed to `national_state`. Before is `main` plus #1036. The diff and checks are reproduced by `scripts/diff_compiles.py` in the evidence folder. +## Congressional-district rows (#1250) + +The congressional-district file (`22incd.csv`, packaged as +`irs_soi.ty2023.congressional_district_2022.all_returns`: TY2022 data +stamped ty2023) carries 11 of the 14 columns on the `full` surface. It has +no wage or taxable-income rows. Each column has a US row, 51 state rows +(`_total`), and 428 district rows that the packaged crosswalk maps onto +the 436 districts of the 119th plan; an at-large state enters as its state +total. Before #1250 these rows were TY2022 levels, and the US row was a +second national target beside each control. + +### Same columns, a narrower count + +`22incddocguide.docx` defines each column at the same line as the HT2 +guides, under the same IRS name. Its US row counts fewer returns than Table +1.4 in the same tax year, because the file covers only returns processed in +calendar 2023, drops returns whose ZIP code is missing or does not match the +state, and drops APO/FPO, foreign and territory addresses (guide sections C +and D). It also suppresses items with fewer than 20 returns in a district +and items dominated by one return (section E). It is a narrower count of the +same concept, like HT2 is of Table 1.4, but about 2% narrower again: + +| Column | Line | District US / Table 1.4, TY2022 | District US / HT2 US, TY2022 | Control / district US, pinned feed | +|---|---|---:|---:|---:| +| N1 | returns | 0.9690 | 0.9793 | 1.0272 | +| A00100 | 11 | 0.9724 | 0.9758 | 1.0597 | +| N01400 / A01400 | 4b | 0.9966 / 0.9767 | 0.9877 / 0.9837 | 1.0288 / 1.0247 | +| N01700 / A01700 | 5b | 0.9827 / 0.9702 | 0.9850 / 0.9843 | 1.0014 / 1.0538 | +| N02300 / A02300 | Schedule 1, 7 | 0.9671 / 0.9699 | 0.9839 / 0.9798 | 1.0272 / 1.0546 | +| N02500 / A02500 | 6b | 0.9797 / 0.9831 | 0.9874 / 0.9888 | 1.0641 / 1.1692 | +| A04470 | 12 | 0.9404 | 0.9657 | 1.0997 | + +The last column is what the aged control is over the aged district US row +on the pinned feed: the coverage gap times the TY2022 to TY2023 change of the +column. The district rows age only 2023 to 2024, because of the stamp. +Within the file, the 51 states sum exactly to the US row and each state's +districts sum exactly to the state, for all 11 columns. The register +records each column's IRS name (`source_column`), so the guide check is a +test (`test_the_register_names_each_columns_reviewed_irs_id`). +`~/PolicyEngine/_reviews/cd-soi-national-controls-20261012/scripts/cd_identities.py` +reproduces the table from the sha256-matched `22incd.csv`, `22in14ar.xls` +and `22in55cmcsv.csv`. + +### How the rows combine + +`_rebase_stale_soi_district_distributions` runs after the HT2 pass. It uses +the same register, the same control chooser and the same refusal +(`AmbiguousSoiNationalControlError`). Its rule is the one #1229 (taxable +interest) and #1040 (capital gains) use for other district families: + +- **Shares of the file's own US row.** Each district-file state and district + row becomes `value x control / district US`. The US row is the same + record set's national row of the same measure, filing status, universe and + period. One factor per measure scales the states and the districts alike, + so the district-to-state reconciliation that runs next keeps factor 1. + Every share names its denominator in `soi_share_total_source_record_id` + and lands at the control's period, as the HT2 shares do. +- **One national level.** The district US row retires. +- **No eligible control.** With no control, one after the build period, or + one older than the row's period, the rows keep their levels and the US row + stays. +- **No denominator.** With an eligible control but no nonzero district US + row, a row cannot be a share, and as a level it would be the stale second + value this pass removes, so it is dropped. That is the district rule of + #1229 and #1040. The HT2 pass keeps HT2 rows as levels in the same case. + Neither case occurs on the pinned feed. +- **One denominator.** Two different district US values for one key refuse + the compile; equal twins name the smaller record id, so feed order never + picks the denominator. +- **Reviewed columns only.** A district fact of a registered measure whose + control is eligible must read the register's `source_column`. Otherwise + the compile refuses (`UnreviewedSoiNationalControlColumnError`), as it + does for a missing column. The check runs before the crosswalk, which + merges source districts into one fact with a single contributor's layout. + The district package's SALT and PTC measures read the wrong columns + (microcosm#1038), so a re-pin could do the same here. +- **The stamp cancels.** Row and US row carry the same stamp, so stamping the + file truthfully at 2022 gives the same values + (`test_the_share_ignores_the_package_stamp`). Under #1030's data-year + correction each row is a share of a restamped total of its own package, + which #1030 admits. + +### Effect on the pinned feed + +Compiled as the release does; before is #1247's head (`490b67181`). + +| Quantity | District file, 51 states, before | After | Move | +|---|---:|---:|---:| +| `taxable_social_security` sum | $490.09B | $573.04B | +16.92% | +| `taxable_social_security` count | 24,167,050 | 25,716,763 | +6.41% | +| `itemized_taxable_income_deductions` sum | $682.97B | $751.10B | +9.97% | +| `adjusted_gross_income` sum | $15,682.85B | $16,619.16B | +5.97% | +| `unemployment_compensation` sum | $31.89B | $33.64B | +5.46% | +| `taxable_pension_income` sum | $961.67B | $1,013.42B | +5.38% | +| `ira_distributions` count | 16,227,430 | 16,694,154 | +2.88% | +| `count` | 156,342,760 | 160,602,107 | +2.72% | +| `unemployment_compensation` count | 4,573,040 | 4,697,502 | +2.72% | +| `ira_distributions` sum | $464.87B | $476.36B | +2.47% | +| `taxable_pension_income` count | 29,501,160 | 29,541,284 | +0.14% | + +- **Retired: 11 rows**, the district US rows. **Added: none.** Compiled + targets go from 33,235 to 33,224; the full registry from `0bbbad9b22d1` to + `ce77f86efb24`. `national_state` holds no district-file row and is + unchanged (6,087, `0e899cd7f893`). +- **5,357 values move** (11 x (51 + 436)), each family by its one factor + (the last column above). No other spec changes, in value or metadata. +- After aging, the file's 51 states, and its 436 districts, each sum to the + control (relative error at most 1e-12), and every district reconciles to + its state with factor 1. +- On `full`, every quantity a control of this register measures now has one + national value. Quantities with more than one national value go from 38 to + 27, the 22 and 5 under "What this does not change". (Two more EITC groups + look like duplicates only to `_soi_national_quantity`: their Table 2.5 rows + take the child count from the record set, which the key does not read.) +- **At-large districts are bound twice**, as before: the district file's + row, and HT2's state row standing in for the district (AK, DE, DC, ND, SD, + VT, WY and Montana's two districts; 99 rows). Both are now shares of the + same control, so the two differ only by the two files' shares of that + state: HT2 / district file is 0.966 to 1.072 (median 0.988), against + 0.986 to 1.17 (median 1.037) before, except Wyoming's and South Dakota's + itemized deductions (1.64 and 1.35), where the two files disagree about the + state. Counting both, the 445 district rows of a measure sum to 1.015 to + 1.022 times the control; each family alone sums to at most the control. +- **ACS-local `state_cd`.** All 26,425 values are unchanged (relative + difference at most 4.2e-15), as are its counts, drops and band verdicts. + It takes district rows only as within-state shares of HT2, which a + national factor does not move. Its level bridges read a rebased sibling + at the file's own level ([us-acs-local-soi-target-surface.md](us-acs-local-soi-target-surface.md)). + Only the receipt's per-measure rebase factors for these columns move, to + 0.975 to 1.063 (medians 0.995 to 1.009): what is left is where the two + files disagree about a state's share. + +`scripts/diff_cd.py`, `scripts/at_large_pairs.py` and +`scripts/compare_statecd.py` in the evidence folder reproduce these figures. + ## What this does not change - **Share assumption.** TY2022 HT2 state shares stand in for TY2023 shares. @@ -195,9 +332,12 @@ evidence folder. ids, keeps them instead, and this pass rebases them onto the TY2023 controls by the same-year difference alone (`test_a_control_at_the_hts_own_period_still_rebases`). -- **Congressional-district rows** of these columns stay TY2022 levels on the - `full` surface. They are not on `national_state`; #1040, #1229 and #1231 - reconcile other district families. +- **Congressional-district rows of other columns.** The district file's US + row is still a second national value on `full` for 22 quantities whose + other national row is the TY2022 HT2 US row (dividends, Schedule C, + partnership, rental and tax-exempt income, EITC, ACTC, income tax, medical + deductions), and for capital gains, SALT and PTC returns, which #1040 and + #1224 handle. - **Columns without an all-returns control** keep their HT2 levels: N04470 (itemizing returns) and N04800 (returns with taxable income) appear nationally only in Table 4.3, which excludes dependent filers. diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_cd_properties.py b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_cd_properties.py index c4e03307b9..4a8a7eee7a 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_cd_properties.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_cd_properties.py @@ -970,7 +970,69 @@ def verdicts(receipt): ) assert verdicts(after.receipt) == verdicts(before.receipt) + # Every verdict's relative gap is unchanged. Where the share ages like the + # file (as on the pinned feed), a sibling's recorded level ratio is too: + # it compares the two files' own levels. A block's rebase factor is the + # parent over the compiled district rows, so it moves with the share + # factor while its verdict does not. + recorded = { + (row["state_fips"], row["measure"], row["verdict"]): row + for row in before.receipt["factor_band"]["out_of_band"] + } + for row in after.receipt["factor_band"]["out_of_band"]: + old_row = recorded[(row["state_fips"], row["measure"], row["verdict"])] + keys = ["relative"] + if row["verdict"] == "sibling_vintage_ratio" and own_aging == control_aging: + keys += ["ratio", "median_ratio"] + for key in keys: + assert row[key] == pytest.approx(old_row[key], rel=1e-12, nan_ok=True) assert ( after.receipt["level_bridge_factor_by_measure"] == before.receipt["level_bridge_factor_by_measure"] ) + + +def test_a_national_share_sibling_keeps_its_bridges_and_recorded_ratios() -> None: + """State 14's two return-count levels disagree 2x, so its bridge is out of + band. Binding the district file's return counts as national shares (x1.5) + leaves every bound value, the out-of-band rows and their recorded ratios + where they were: the sibling is still read at the file's own level.""" + + specs, crosswalk = _bridge_world( + {"10": 1.0, "11": 1.0, "12": 1.0, "13": 1.0, "14": 2.0}, {} + ) + base = [ + _aged(spec, 1.0) + if spec.metadata["ledger_layout_record_set_spec_id"] == _CD_FILE + else spec + for spec in specs + ] + shared = [ + _as_national_share(spec, 1.5, 1.0) + if spec.metadata["ledger_layout_record_set_spec_id"] == _CD_FILE + and spec.metadata["source_measure_id"] == "return_count" + else spec + for spec in base + ] + before, after = ( + _CD.state_cd_soi_surface( + rows, + state_surface_predicate=_TOOL.soi_surface_predicate("state"), + crosswalk=crosswalk, + level_bridges={"charitable_amount": "return_count"}, + ) + for rows in (base, shared) + ) + assert {spec.name: spec.value for spec in after.specs} == pytest.approx( + {spec.name: spec.value for spec in before.specs}, rel=1e-12 + ) + rows = { + (row["state_fips"], row["measure"], row["verdict"]): row + for row in after.receipt["factor_band"]["out_of_band"] + } + vintage = rows[("14", "charitable_amount", "sibling_vintage_ratio")] + assert (vintage["ratio"], vintage["median_ratio"]) == pytest.approx((2.0, 1.0)) + assert set(rows) == { + (row["state_fips"], row["measure"], row["verdict"]) + for row in before.receipt["factor_band"]["out_of_band"] + } From bc08b5df30da6d2678539ff15f3a8fe227727912 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 20:11:57 -0400 Subject: [PATCH 05/12] State the at-large exceptions' before and after values Co-Authored-By: Claude Opus 5.5 --- docs/us-soi-national-controls.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/us-soi-national-controls.md b/docs/us-soi-national-controls.md index e473a2dc48..91cbf80d28 100644 --- a/docs/us-soi-national-controls.md +++ b/docs/us-soi-national-controls.md @@ -307,9 +307,9 @@ Compiled as the release does; before is #1247's head (`490b67181`). VT, WY and Montana's two districts; 99 rows). Both are now shares of the same control, so the two differ only by the two files' shares of that state: HT2 / district file is 0.966 to 1.072 (median 0.988), against - 0.986 to 1.17 (median 1.037) before, except Wyoming's and South Dakota's - itemized deductions (1.64 and 1.35), where the two files disagree about the - state. Counting both, the 445 district rows of a measure sum to 1.015 to + 0.986 to 1.17 (median 1.037) before. The exceptions are Wyoming's and South + Dakota's itemized deductions (1.64 and 1.35, from 1.81 and 1.49), where the + two files disagree about the state. Counting both, the 445 district rows of a measure sum to 1.015 to 1.022 times the control; each family alone sums to at most the control. - **ACS-local `state_cd`.** All 26,425 values are unchanged (relative difference at most 4.2e-15), as are its counts, drops and band verdicts. From dddcfddd4f0c9a681c2428c47635fa79c6c97253 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 20:56:45 -0400 Subject: [PATCH 06/12] Keep state_cd receipts on the file's level; rebase district AGI slices (#1250 review) From the in-session Opus review (review-opus.md): - F3: state_cd district blocks read their rows through cd_file_level, so state_cd_cd_file_value, state_cd_cd_file_state_sum and the rebase factor record the district file's own values again and the receipt's rebase factors match #1247's base. Bound values are unchanged. - F2: re-aging a share sibling like the bridged row refuses when the two age on different series, and the sibling's band median is re-aged with it, so its recorded ratio matches the unrebased case. - F4: AGI slices of the district file take the measure's one factor, as #1229 and #1040 rebase district slices, and the column guard checks them; tests pin the guard's eligibility rule and the slice rebase. Co-Authored-By: Claude Opus 5.5 --- .../build/us_runtime/fiscal_targets.py | 20 ++-- .../us/test_us_acs_local_cd_properties.py | 91 ++++++++++++++++--- .../test_us_soi_district_national_controls.py | 75 +++++++++++++++ tools/us_acs_local_cd_surface.py | 61 ++++++++++--- 4 files changed, 213 insertions(+), 34 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py b/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py index 06fe36f6e9..02cae2c4e9 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py @@ -2285,8 +2285,9 @@ def _rebase_stale_soi_district_distributions( 1.x control: on the pinned feed the control was 1.001x (taxable pension returns) to 1.169x (taxable Social Security) the district US row. - Each state and district row of such a measure now ships as its share of - the file's own US row at its own period, times the control of + Each state and district row of such a measure (and any AGI slice of the + file) now ships as its share of the file's own all-AGI US row at its own + period, times the control of ``_soi_ht2_national_controls`` at or after that period, as microcosm#1229 does for taxable interest and #1040 for capital gains. Every share row names its denominator in ``soi_share_total_source_record_id``. One factor @@ -2353,7 +2354,12 @@ def _rebase_stale_soi_district_distributions( def _soi_district_rebase_measure(spec: TargetSpec) -> str | None: - """The registered measure of an all-AGI congressional-district spec.""" + """The registered measure of a congressional-district spec, else ``None``. + + An AGI slice of the file is a share of the same US row, so it takes the + same factor (as #1229 and #1040 rebase district slices); the current + package publishes none. + """ if spec.family != "irs_soi": return None measure_id = spec.metadata.get("source_measure_id", "") @@ -2362,13 +2368,11 @@ def _soi_district_rebase_measure(spec: TargetSpec) -> str | None: record_set_id = spec.metadata.get("ledger_layout_record_set_id", "") if _soi_record_set_family(record_set_id) != "congressional_district": return None - if _bounds_from_metadata(spec) != (-float("inf"), float("inf")): - return None return measure_id def _soi_district_fact_key(fact: object) -> tuple[str, str, str] | None: - """(measure, filing status, universe) of an all-AGI district fact.""" + """(measure, filing status, universe) of a district fact, any AGI range.""" if _source_name(fact) != "irs_soi": return None measure_id = _measure_id(fact) @@ -2377,8 +2381,6 @@ def _soi_district_fact_key(fact: object) -> tuple[str, str, str] | None: record_set_id = _str_at(fact, "layout", "record_set_id") if _soi_record_set_family(record_set_id) != "congressional_district": return None - if not _is_all_agi_range_fact(fact): - return None status = _filing_status_label(_dimensions(fact).get("filing_status")) if status is None: return None @@ -2401,6 +2403,8 @@ def _soi_district_us_totals( key = _soi_district_fact_key(fact) if key is None or _geography_level(fact) != "country": continue + if not _is_all_agi_range_fact(fact): + continue source_record_id = _source_record_id(fact) if not source_record_id: continue diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_cd_properties.py b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_cd_properties.py index 4a8a7eee7a..dc9dee1ff5 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_cd_properties.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_cd_properties.py @@ -911,7 +911,9 @@ def test_national_share_district_rows_leave_the_state_cd_surface_unchanged( a national control: value x control / the file's US row, aged from the control's period. ``state_cd`` uses the file for within-state shares, and for level-bridge siblings through ``cd_file_level``, so the rebase moves - no ``state_cd`` value, drop or band verdict, with or without the band. + no ``state_cd`` value, drop, band verdict, recorded ratio or factor, with + or without the band, whatever the share's aging (the bridged row and its + sibling share one aging series, as every registered bridge does). Only concepts Historic Table 2 carries in every state are rebased here, as the compiler's registered concepts are on the feed: a rebased concept @@ -970,11 +972,14 @@ def verdicts(receipt): ) assert verdicts(after.receipt) == verdicts(before.receipt) - # Every verdict's relative gap is unchanged. Where the share ages like the - # file (as on the pinned feed), a sibling's recorded level ratio is too: - # it compares the two files' own levels. A block's rebase factor is the - # parent over the compiled district rows, so it moves with the share - # factor while its verdict does not. + # The receipt reads the district file's own levels. A bridge re-ages the + # sibling like the bridged row, so its factor and the sibling's recorded + # ratio and median are unchanged whatever the aging. A block reads its + # shares at the file level aged as the shares are, from the control's + # period: where that matches the file's own aging (as on the pinned feed), + # its rebase factor and every district row's record of the file value, + # state sum and factor are unchanged too. + same_aging = own_aging == control_aging recorded = { (row["state_fips"], row["measure"], row["verdict"]): row for row in before.receipt["factor_band"]["out_of_band"] @@ -982,14 +987,30 @@ def verdicts(receipt): for row in after.receipt["factor_band"]["out_of_band"]: old_row = recorded[(row["state_fips"], row["measure"], row["verdict"])] keys = ["relative"] - if row["verdict"] == "sibling_vintage_ratio" and own_aging == control_aging: + if same_aging or row["verdict"] == "sibling_vintage_ratio": keys += ["ratio", "median_ratio"] for key in keys: assert row[key] == pytest.approx(old_row[key], rel=1e-12, nan_ok=True) - assert ( - after.receipt["level_bridge_factor_by_measure"] - == before.receipt["level_bridge_factor_by_measure"] - ) + keys = ["level_bridge_factor_by_measure"] + if same_aging: + keys.append("rebase_factor_by_measure") + for key in keys: + assert after.receipt[key].keys() == before.receipt[key].keys() + for measure, summary in after.receipt[key].items(): + assert summary == pytest.approx(before.receipt[key][measure], rel=1e-12) + metadata_keys = ["state_cd_level_bridge_factor"] + if same_aging: + metadata_keys += [ + "state_cd_cd_file_value", + "state_cd_cd_file_state_sum", + "state_cd_rebase_factor", + ] + for old, new in zip(before.specs, after.specs, strict=True): + for key in metadata_keys: + if key in old.metadata: + assert float(new.metadata[key]) == pytest.approx( + float(old.metadata[key]), rel=1e-12 + ), (new.name, key) def test_a_national_share_sibling_keeps_its_bridges_and_recorded_ratios() -> None: @@ -1036,3 +1057,51 @@ def test_a_national_share_sibling_keeps_its_bridges_and_recorded_ratios() -> Non (row["state_fips"], row["measure"], row["verdict"]) for row in before.receipt["factor_band"]["out_of_band"] } + + +def test_a_bridge_refuses_a_share_sibling_on_another_aging_series() -> None: + """Re-aging a share's file level like the bridged row stands in for the + sibling's own aging only when both age on one series; a pair on different + series refuses rather than giving the bridged row its sibling's growth.""" + + specs, crosswalk = _bridge_world({"10": 1.0, "11": 1.0, "12": 1.0}, {}) + + def series(spec, source: str): + return dataclasses.replace( + spec, + metadata={ + **spec.metadata, + "aging_factor_source": source, + "aging_factor": "1.5", + }, + ) + + rows = [] + for spec in specs: + if spec.metadata["ledger_layout_record_set_spec_id"] != _CD_FILE: + rows.append(spec) + elif spec.metadata["source_measure_id"] == "return_count": + rows.append(_as_national_share(series(spec, "cbo.series_a"), 1.25, 1.5)) + else: + rows.append(series(spec, "chained:irs_soi.x+cbo.series_b")) + with pytest.raises(ValueError, match="ages on 'cbo.series_a'"): + _CD.state_cd_soi_surface( + rows, + state_surface_predicate=_TOOL.soi_surface_predicate("state"), + crosswalk=crosswalk, + level_bridges={"charitable_amount": "return_count"}, + ) + # The same chained series, as #1030 writes it, re-ages. + same = [ + series(spec, "chained:irs_soi.x+cbo.series_a") + if spec.metadata["ledger_layout_record_set_spec_id"] == _CD_FILE + and spec.metadata["source_measure_id"] == "charitable_amount" + else spec + for spec in rows + ] + _CD.state_cd_soi_surface( + same, + state_surface_predicate=_TOOL.soi_surface_predicate("state"), + crosswalk=crosswalk, + level_bridges={"charitable_amount": "return_count"}, + ) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_soi_district_national_controls.py b/packages/microcosm-build/tests/engine_free/us/test_us_soi_district_national_controls.py index a4d9bb56e3..eba3596983 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_soi_district_national_controls.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_soi_district_national_controls.py @@ -417,6 +417,23 @@ def test_a_district_fact_under_an_unreviewed_column_refuses(column) -> None: assert rows["hi_total"].value == 500.0 +@pytest.mark.parametrize("control_period", [2022, 2025], ids=["older", "future"]) +def test_the_column_guard_ignores_rows_without_an_eligible_control( + control_period: int, +) -> None: + """The guard checks exactly the rows the pass would rebase: beside an + older or a future control the rows keep their levels, so an unreviewed + column is not refused.""" + measure_id = "taxable_social_security_returns" + facts = _cd_facts(measure_id, column="N02510") + rows = _cd_rows( + _compile([*facts, _control_fact(measure_id, period=control_period)]), + measure_id, + ) + assert rows["us"].value == 1_000.0 + assert rows["hi_total"].value == 500.0 + + def test_the_column_guard_sees_each_source_district_before_the_crosswalk() -> None: """The crosswalk merges source districts into one fact that keeps the first contributor's layout, so a mislabeled second contributor would hide @@ -445,6 +462,64 @@ def test_the_column_guard_sees_each_source_district_before_the_crosswalk() -> No _compile([*facts, _control_fact(measure_id)], crosswalk=crosswalk) +def _cd_agi_slice_fact( + measure_id: str, value: float, *, stamp: int, column: object = _REVIEWED +) -> dict[str, object]: + """Hawaii's $50,000-$75,000 AGI slice of the district file (agi_stub 5).""" + row = "hi_total_50k_to_75k" + fact = _dynamic_ledger_fact( + source_record_id=f"{_record_set(stamp)}.{row}.{measure_id}", + source_name="irs_soi", + measure_id=measure_id, + value=value, + period_value=stamp, + geography_level="state", + geography_id="0400000US15", + dimensions={"income_range": "50k_to_75k", "filing_status": "all"}, + universe_constraints=[ + {"variable": "adjusted_gross_income", "operator": ">=", "value": 50_000}, + {"variable": "adjusted_gross_income", "operator": "<", "value": 75_000}, + ], + layout_record_set_id=_record_set(stamp), + groupby_dimension="irs_soi.congressional_district", + groupby_value_id=row, + ) + fact["layout"]["source_column_id"] = ( + REGISTER[measure_id].source_column if column is _REVIEWED else column + ) + return fact + + +def test_an_agi_slice_of_the_file_takes_the_same_factor() -> None: + """A district-file AGI slice is a share of the same all-AGI US row, so it + takes the measure's one factor (as #1229 and #1040 rebase district + slices) and the column guard checks it. The pinned package has no + slices, and a slice compiles only at its own period (here 2024).""" + measure_id = "adjusted_gross_income" + + def facts(column: object = _REVIEWED) -> list[dict[str, object]]: + return [ + *_cd_facts(measure_id, stamp=2024), + _cd_agi_slice_fact(measure_id, 120.0, stamp=2024, column=column), + _control_fact(measure_id, period=2024), + ] + + (slice_spec,) = [ + spec + for spec in _compile(facts()).specs + if spec.metadata["ledger_source_record_id"].endswith( + f"hi_total_50k_to_75k.{measure_id}" + ) + ] + assert slice_spec.metadata["agi_lower_bound"] == "50000.0" + assert slice_spec.value == pytest.approx(132.0, rel=1e-12) + assert slice_spec.metadata["soi_share_total_source_record_id"] == _us_id( + measure_id, stamp=2024 + ) + with pytest.raises(fiscal_targets.UnreviewedSoiNationalControlColumnError): + _compile(facts(column="A00101")) + + # -------------------------------------------------------------------------- # The denominator diff --git a/tools/us_acs_local_cd_surface.py b/tools/us_acs_local_cd_surface.py index 11f6377cec..525b8259ef 100644 --- a/tools/us_acs_local_cd_surface.py +++ b/tools/us_acs_local_cd_surface.py @@ -208,11 +208,21 @@ def cd_file_level(spec, *, aged_like=None) -> float: (``soi_share_total_source_record_id``) and its ``uprating_factor`` is control / the file's US row (microcosm#1229, #1040, and the ``_SOI_HT2_NATIONAL_CONTROLS`` columns). Dividing the factor out gives the - file's own level, which is what a level bridge compares with Historic - Table 2. A share ages from its control's period; with ``aged_like``, the - level is re-aged to that spec's aging factor, so it sits on the same - basis as an unrebased district-file row (the two differ once #1030 ages - the file from its data year). Other specs return their value. + file's own level: what a district block's shares, its receipt and a level + bridge read. Other specs return their value. + + A share ages from its control's period, and so does its file level. That + is the file's own aging while the file is stamped at the control's period + (the pinned feed); under #1030 the file ages from its data year, one link + earlier, so a block's recorded file values sit that link apart (its + shares, and so its bound values, do not move). With ``aged_like`` (an unrebased + district-file row), the level is re-aged to that row's aging factor, so + the two sit on one basis; they differ once #1030 ages the file from its + data year. That stands in for the share's own unrebased aging only when + both age on the same series, so a pair on different series refuses. + + Raises: + ValueError: ``aged_like`` ages on a different series from ``spec``. """ metadata = _metadata(spec) @@ -221,12 +231,26 @@ def cd_file_level(spec, *, aged_like=None) -> float: return value value /= float(metadata["uprating_factor"]) if aged_like is not None: - value *= float(_metadata(aged_like).get("aging_factor", "1")) / float( + other = _metadata(aged_like) + if _aging_series(other) != _aging_series(metadata): + raise ValueError( + f"{spec.name} ages on {_aging_series(metadata)!r} and " + f"{aged_like.name} on {_aging_series(other)!r}; a share's file " + "level can be re-aged only like a row on its own series." + ) + value *= float(other.get("aging_factor", "1")) / float( metadata.get("aging_factor", "1") ) return value +def _aging_series(metadata: Mapping[str, str]) -> str: + """The growth series a spec's aging ends on (its last link, if chained).""" + + source = str(metadata.get("aging_factor_source", "not_dollar_amount")) + return source.removeprefix("chained:").rsplit("+", 1)[-1] + + def soi_materializer_semantics(spec) -> tuple: """Everything the SOI slice reads from a spec, except its geography. @@ -477,11 +501,12 @@ def within_band( kept_cd_by_key[(state, soi_concept_identity(spec))].append(spec) # Each Historic-Table-2-parented block's rebase factor (parent over its - # district rows' sum), judged against its concept's median across states. + # district rows' sum at the file's level), judged against its concept's + # median across states. block_factor: dict[tuple, float] = {} for key, block in kept_cd_by_key.items(): parent = ht2_by_key.get(key) - child_sum = math.fsum(float(spec.value) for spec in block) + child_sum = math.fsum(cd_file_level(spec) for spec in block) if parent is not None and child_sum != 0.0: block_factor[key] = float(parent.value) / child_sum factors_by_identity: dict[tuple, list[float]] = defaultdict(list) @@ -533,11 +558,13 @@ def within_band( ) # The sibling is trusted for bridging only if both of the band's # verdicts on it pass: its two state levels, and (where it has district - # rows) the rebase of its own district block. + # rows) the rebase of its own district block. Re-aging scales every + # state's ratio alike, so the median is re-aged by the same constant. + reaging = factor / vintage_ratio[sibling_key] trusted = within_band( sibling_key, - vintage_ratio[sibling_key], - median_ratio.get(sibling_key[1], math.nan), + factor, + median_ratio.get(sibling_key[1], math.nan) * reaging, f"level_bridge:{measure}", measure, verdict="sibling_vintage_ratio", @@ -619,7 +646,11 @@ def within_band( f"State parent {parent.name} materializes differently from " f"its district rows ({identity}, state {state})." ) - child_sum = math.fsum(float(spec.value) for spec in children) + # Shares and the receipt read the district file's own levels, so a + # district row the compiler bound as a national share records the + # file's value, as every other district row does. + levels = {spec.name: cd_file_level(spec) for spec in children} + child_sum = math.fsum(levels.values()) parent_value = float(parent.value) if child_sum == 0.0: if parent_value != 0.0: @@ -639,7 +670,7 @@ def within_band( # The district file must agree with itself before its shares are used. cd_state = cd_file_state_by_key.get(key) if cd_state is not None: - cd_state_value = float(cd_state.value) + cd_state_value = cd_file_level(cd_state) gap = abs(cd_state_value - child_sum) internal_gaps.append(gap / max(abs(cd_state_value), 1.0)) if gap > STATE_CD_INTERNAL_RTOL * max(abs(cd_state_value), 1.0): @@ -675,12 +706,12 @@ def within_band( "state_cd_parent_target_name": parent.name, "state_cd_parent_basis": basis, "state_cd_parent_value": repr(parent_value), - "state_cd_cd_file_value": repr(float(child.value)), + "state_cd_cd_file_value": repr(levels[child.name]), "state_cd_cd_file_state_sum": repr(child_sum), "state_cd_rebase_factor": repr(factor), } ) - value = float(child.value) * factor + value = levels[child.name] * factor rebased[child.name] = replace( child, value=value, From 0072c21590c0eeebb4e3ead4f4686fe6d9f3b6d9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 21:13:38 -0400 Subject: [PATCH 07/12] Document the review fixes; pin national AGI slices as shares The docs and changelog now say the state_cd receipt and per-row records are unchanged, explain the bridge's aging-series assumption, and note that AGI slices take the measure's factor. A national AGI slice of the district file is now tested: it is a share, never a second denominator. Co-Authored-By: Claude Opus 5.5 --- .../1250-cd-national-controls.fixed.md | 2 +- docs/us-acs-local-soi-target-surface.md | 31 ++++++++------ docs/us-soi-national-controls.md | 21 +++++----- .../test_us_soi_district_national_controls.py | 40 +++++++++++-------- 4 files changed, 56 insertions(+), 38 deletions(-) diff --git a/changelog.d/1250-cd-national-controls.fixed.md b/changelog.d/1250-cd-national-controls.fixed.md index 210dcbbc95..8370c4f7ad 100644 --- a/changelog.d/1250-cd-national-controls.fixed.md +++ b/changelog.d/1250-cd-national-controls.fixed.md @@ -1 +1 @@ -Rebase the congressional-district file's state and district rows of the 11 Table 1.x-controlled SOI columns (return count, AGI, taxable IRA distributions, taxable pensions, unemployment compensation and taxable Social Security, returns and amounts; itemized deductions) as shares of the file's own US row times their #1247 control, and retire the district US row. On the pinned feed's `full` surface no controlled quantity binds two national values any more (control / district US row was 1.001 to 1.169); the district file's 51-state sums move from +0.14% (taxable pension returns) to +16.9% (taxable Social Security) and its states and districts each sum to the control. A district fact that would be rebased under an unreviewed IRS column refuses the compile. `national_state` and every ACS-local `state_cd` value are unchanged; the `state_cd` level bridges read their rebased siblings at the district file's own level (microcosm#1250). +Rebase the congressional-district file's state and district rows of the 11 Table 1.x-controlled SOI columns (return count, AGI, taxable IRA distributions, taxable pensions, unemployment compensation and taxable Social Security, returns and amounts; itemized deductions) as shares of the file's own US row times their #1247 control, and retire the district US row. On the pinned feed's `full` surface no controlled quantity binds two national values any more (control / district US row was 1.001 to 1.169); the district file's 51-state sums move from +0.14% (taxable pension returns) to +16.9% (taxable Social Security) and its states and districts each sum to the control. A district fact that would be rebased under an unreviewed IRS column refuses the compile. `national_state` and the ACS-local `state_cd` surface (values, receipt and per-row records) are unchanged, because `state_cd` reads district rows bound as national shares at the district file's own level (microcosm#1250). diff --git a/docs/us-acs-local-soi-target-surface.md b/docs/us-acs-local-soi-target-surface.md index 7626e5efd2..dfd1b92019 100644 --- a/docs/us-acs-local-soi-target-surface.md +++ b/docs/us-acs-local-soi-target-surface.md @@ -203,13 +203,16 @@ district rows, are mostly 1.00 to 1.10. 2.70, the Table 4.3 rebase) and capital-gains amounts (0.77 to 1.00). - **The 11 Table 1.x-controlled columns** (return count, AGI, taxable IRA distributions, pensions, unemployment compensation and Social Security, - itemized deductions) are compiled on both sides as shares of the same - national control: Historic Table 2's since #1247, the district file's since - #1250 ([us-soi-national-controls.md](us-soi-national-controls.md)). Their - factors therefore carry neither the coverage nor the aging gap, only where - the two files disagree about a state's share: 0.975 to 1.063 (medians - 0.995 to 1.009). The two bullets above describe the files before those - rebases. + itemized deductions) have Historic Table 2 parents that are shares of a + Table 1.x control since #1247, so their factors also carry the Table 1.x / + HT2 gap: the return-count median is 1.023 and AGI's 1.054 (the bullets + above were measured before #1247), taxable Social Security amounts run + 1.16 to 1.18. The district file's own rows of these columns are bound as + shares of the same controls since #1250 + ([us-soi-national-controls.md](us-soi-national-controls.md)). `state_cd` + reads such a row at the file's own level (`cd_file_level`: its + `uprating_factor` divided out), so its shares, factors, records and bridges + are what they would be without that rebase. - **The largest single-state factors** were on qualified and ordinary dividends (Hawaii, 2.68 and 2.17), tax-exempt interest (Utah, 1.43) and rental income (New York, 0.72), where the two tables disagree about one @@ -298,11 +301,15 @@ All three siblings are columns the compiler binds as shares of a national control (#1250), so the district file's sibling row no longer sits at the file's level. The bridge reads it there anyway (`cd_file_level`: the share's `uprating_factor` divided out, re-aged as the bridged row is), and the -sibling's band verdict compares the two files' levels. Reading the share as -compiled would lift nothing: on the pinned feed the bridged charitable and -interest-paid amounts would fall 9.07%, QBI amounts 5.63% and the bridged -counts 2.65%. A property test checks that national-share district rows move -no `state_cd` value, drop or verdict. +sibling's band verdict compares the two files' levels, its median re-aged by +the same constant. Re-aging like the bridged row stands in for the sibling's +own aging only when both age on one series, as every registered bridge does +(counts unaged, amounts on the CBO AGI series); a pair on different series +refuses. Reading the share as compiled would lift nothing: on the pinned +feed the bridged charitable and interest-paid amounts would fall 9.07%, QBI +amounts 5.63% and the bridged counts 2.65%. A property test checks that +national-share district rows move no `state_cd` value, drop, verdict or +recorded factor. Every district row records its parent (`state_cd_parent_target_name`), the parent's basis, the district file's own value and the rebase factor; a diff --git a/docs/us-soi-national-controls.md b/docs/us-soi-national-controls.md index 91cbf80d28..f3e2a76f6f 100644 --- a/docs/us-soi-national-controls.md +++ b/docs/us-soi-national-controls.md @@ -225,8 +225,9 @@ on the pinned feed: the coverage gap times the TY2022 to TY2023 change of the column. The district rows age only 2023 to 2024, because of the stamp. Within the file, the 51 states sum exactly to the US row and each state's districts sum exactly to the state, for all 11 columns. The register -records each column's IRS name (`source_column`), so the guide check is a -test (`test_the_register_names_each_columns_reviewed_irs_id`). +records each column's IRS name (`source_column`), and +`test_the_register_names_each_columns_reviewed_irs_id` pins it to the names +in the table above, read from the guide. `~/PolicyEngine/_reviews/cd-soi-national-controls-20261012/scripts/cd_identities.py` reproduces the table from the sha256-matched `22incd.csv`, `22in14ar.xls` and `22in55cmcsv.csv`. @@ -244,7 +245,9 @@ interest) and #1040 (capital gains) use for other district families: period. One factor per measure scales the states and the districts alike, so the district-to-state reconciliation that runs next keeps factor 1. Every share names its denominator in `soi_share_total_source_record_id` - and lands at the control's period, as the HT2 shares do. + and lands at the control's period, as the HT2 shares do. An AGI slice of + the file takes the same factor, as #1229 and #1040 rebase district slices + (the current package has none). - **One national level.** The district US row retires. - **No eligible control.** With no control, one after the build period, or one older than the row's period, the rows keep their levels and the US row @@ -312,13 +315,13 @@ Compiled as the release does; before is #1247's head (`490b67181`). two files disagree about the state. Counting both, the 445 district rows of a measure sum to 1.015 to 1.022 times the control; each family alone sums to at most the control. - **ACS-local `state_cd`.** All 26,425 values are unchanged (relative - difference at most 4.2e-15), as are its counts, drops and band verdicts. + difference at most 4.2e-15), as are its counts, drops, band verdicts and + receipt (rebase and bridge factors and recorded ratios, within 5e-15). It takes district rows only as within-state shares of HT2, which a - national factor does not move. Its level bridges read a rebased sibling - at the file's own level ([us-acs-local-soi-target-surface.md](us-acs-local-soi-target-surface.md)). - Only the receipt's per-measure rebase factors for these columns move, to - 0.975 to 1.063 (medians 0.995 to 1.009): what is left is where the two - files disagree about a state's share. + national factor does not move, and it reads a district row bound as a + national share at the file's own level (`cd_file_level`), so its blocks, + bridges and per-row records see what they saw before + ([us-acs-local-soi-target-surface.md](us-acs-local-soi-target-surface.md)). `scripts/diff_cd.py`, `scripts/at_large_pairs.py` and `scripts/compare_statecd.py` in the evidence folder reproduce these figures. diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_soi_district_national_controls.py b/packages/microcosm-build/tests/engine_free/us/test_us_soi_district_national_controls.py index eba3596983..a80f740700 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_soi_district_national_controls.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_soi_district_national_controls.py @@ -463,18 +463,24 @@ def test_the_column_guard_sees_each_source_district_before_the_crosswalk() -> No def _cd_agi_slice_fact( - measure_id: str, value: float, *, stamp: int, column: object = _REVIEWED + measure_id: str, + value: float, + *, + stamp: int, + column: object = _REVIEWED, + national: bool = False, ) -> dict[str, object]: - """Hawaii's $50,000-$75,000 AGI slice of the district file (agi_stub 5).""" - row = "hi_total_50k_to_75k" + """The district file's $50,000-$75,000 AGI slice (agi_stub 5), for Hawaii + or for the US row.""" + row = "us_50k_to_75k" if national else "hi_total_50k_to_75k" fact = _dynamic_ledger_fact( source_record_id=f"{_record_set(stamp)}.{row}.{measure_id}", source_name="irs_soi", measure_id=measure_id, value=value, period_value=stamp, - geography_level="state", - geography_id="0400000US15", + geography_level="country" if national else "state", + geography_id="0100000US" if national else "0400000US15", dimensions={"income_range": "50k_to_75k", "filing_status": "all"}, universe_constraints=[ {"variable": "adjusted_gross_income", "operator": ">=", "value": 50_000}, @@ -501,21 +507,23 @@ def facts(column: object = _REVIEWED) -> list[dict[str, object]]: return [ *_cd_facts(measure_id, stamp=2024), _cd_agi_slice_fact(measure_id, 120.0, stamp=2024, column=column), + _cd_agi_slice_fact(measure_id, 240.0, stamp=2024, national=True), _control_fact(measure_id, period=2024), ] - (slice_spec,) = [ - spec - for spec in _compile(facts()).specs - if spec.metadata["ledger_source_record_id"].endswith( - f"hi_total_50k_to_75k.{measure_id}" - ) - ] - assert slice_spec.metadata["agi_lower_bound"] == "50000.0" - assert slice_spec.value == pytest.approx(132.0, rel=1e-12) - assert slice_spec.metadata["soi_share_total_source_record_id"] == _us_id( - measure_id, stamp=2024 + specs = _by_record(_compile(facts())) + state_slice = specs[f"{_record_set(2024)}.hi_total_50k_to_75k.{measure_id}"] + us_slice = specs[f"{_record_set(2024)}.us_50k_to_75k.{measure_id}"] + assert state_slice.metadata["agi_lower_bound"] == "50000.0" + # The US slice is a share too, never a second denominator. + assert (state_slice.value, us_slice.value) == pytest.approx( + (132.0, 264.0), rel=1e-12 ) + for spec in (state_slice, us_slice): + assert spec.metadata["soi_share_total_source_record_id"] == _us_id( + measure_id, stamp=2024 + ) + assert _us_id(measure_id, stamp=2024) not in specs with pytest.raises(fiscal_targets.UnreviewedSoiNationalControlColumnError): _compile(facts(column="A00101")) From 01aec387a1b2e659f1ab2930858a874258264790 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 22:10:47 -0400 Subject: [PATCH 08/12] Key district denominators by package; guard only rebased rows; read aging series from the rule (#1250 Sol review) From the GPT-6.1 Sol review (review-sol.md, REQUEST CHANGES, three P2): - A row could divide by another district file's US row at the same stamp. District US totals are now keyed by record set, so each package uses its own; a package without one drops its rows beside a control. - The column guard refused facts that never bind (a stale AGI slice, a superseded vintage). It now runs inside the pass on the rows it rebases: each row's source facts (traced through the crosswalk's contributions and the at-large state-total proxies) and its denominator, read by column as the feed carries them, before the crosswalk merges layouts. - state_cd refused a valid bridge when a share landed at the build year (receipt source_equals_build). The aging series now comes from target aging's own rule (_not_ageable_reason, _cbo_income_source_for_spec). The pinned compile and every state_cd value and receipt are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../build/us_runtime/fiscal_targets.py | 142 ++++++---- .../us/test_us_acs_local_cd_properties.py | 259 +++++++++++++++--- .../test_us_soi_district_national_controls.py | 43 +++ tools/us_acs_local_cd_surface.py | 28 +- 4 files changed, 376 insertions(+), 96 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py b/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py index 02cae2c4e9..781a5c0245 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py @@ -1036,9 +1036,9 @@ def compile_us_fiscal_target_registry( ``period_contract_waiver`` metadata in diagnostics. """ materialized_facts = tuple(facts) - _require_reviewed_soi_district_columns( - materialized_facts, target_period=target_period - ) + # Read before the crosswalk merges source districts into one fact that + # keeps a single contributor's layout. + district_source_columns = _soi_district_source_columns(materialized_facts) if congressional_district_vintage_crosswalk is not None: materialized_facts = translate_congressional_district_facts_to_current_vintage( materialized_facts, @@ -1081,6 +1081,7 @@ def compile_us_fiscal_target_registry( registry, materialized_facts, target_period=target_period, + source_columns=district_source_columns, ) registry = _rebase_soi_state_agi_bands(registry, materialized_facts) registry = apply_ledger_target_profile( @@ -2274,6 +2275,7 @@ def _rebase_stale_soi_district_distributions( facts: tuple[object, ...], *, target_period: int | str, + source_columns: Mapping[str, str] | None = None, ) -> TargetRegistry: """Use congressional-district rows of the registered columns as shares. @@ -2304,10 +2306,29 @@ def _rebase_stale_soi_district_distributions( US row stays. With an eligible control but no nonzero US row of the file's own, a row cannot be a share and is dropped; a stale level beside the control is the defect this pass removes (#1229 and #1040 do the same). + + A share assumes its measure id reads the register's ``source_column``. A + package that maps the measure id to another column (the district SALT + and PTC measures read the wrong columns, microcosm#1038), or names none, + would be rebased as the wrong concept, so every source fact of a row + rebased here, and its denominator, must read that column. + ``source_columns`` maps each district fact's record id to its column as + the feed carries it, before the crosswalk merges source districts into + one fact with a single contributor's layout; without it the columns are + read from ``facts``. Rows that keep their levels, and facts that compile + to no target, are not checked. + + Raises: + UnreviewedSoiNationalControlColumnError: A rebased row or its + denominator reads another column, or names none. """ controls = _soi_ht2_national_controls(facts, target_period=target_period) us_totals = _soi_district_us_totals(facts) + if source_columns is None: + source_columns = _soi_district_source_columns(facts) + contributors = _soi_district_contributors(facts) + unreviewed: dict[str, str] = {} specs: list[TargetSpec] = [] for spec in registry.specs: measure_id = _soi_district_rebase_measure(spec) @@ -2324,9 +2345,19 @@ def _rebase_stale_soi_district_distributions( continue if _is_national_all_agi_spec(spec): continue - us_total = us_totals.get((*key, source_period)) + record_set_id = spec.metadata.get("ledger_layout_record_set_id", "") + us_total = us_totals.get((record_set_id, *key, source_period)) if us_total is None or us_total.value == 0: continue + record_id = spec.metadata.get("ledger_source_record_id", "") + expected = _SOI_HT2_NATIONAL_CONTROLS[measure_id].source_column + for source_id in ( + *contributors.get(record_id, (record_id,)), + us_total.source_record_id, + ): + column = source_columns.get(source_id, "") + if column != expected: + unreviewed[source_id] = column or "" concept = _SOI_HT2_NATIONAL_CONTROLS[measure_id].concept factor = control.value / us_total.value specs.append( @@ -2350,6 +2381,14 @@ def _rebase_stale_soi_district_distributions( }, ) ) + if unreviewed: + sample = dict(sorted(unreviewed.items())[:5]) + raise UnreviewedSoiNationalControlColumnError( + f"{len(unreviewed)} congressional-district fact(s) behind rebased " + "rows read an IRS column other than the reviewed one, e.g. " + f"{sample}. Review the column against the district documentation " + "guide and _SOI_HT2_NATIONAL_CONTROLS before rebasing it." + ) return TargetRegistry(specs, country=registry.country) @@ -2389,16 +2428,17 @@ def _soi_district_fact_key(fact: object) -> tuple[str, str, str] | None: def _soi_district_us_totals( facts: tuple[object, ...], -) -> dict[tuple[str, str, str, str], _SoiDistrictShareTotal]: - """The district file's US totals per (measure, filing status, universe, period). - - A total is a share denominator, so feed order must not pick it: two - different values for one key raise, and equal values under two record - ids name the smaller id (as the #1229 and #1040 denominators do). Two - district files compiled at one period, of different boundary vintages, - share the key, so their US rows must agree. +) -> dict[tuple[str, str, str, str, str], _SoiDistrictShareTotal]: + """Each district package's all-AGI US totals. + + Keyed by (record set, measure, filing status, universe, period), so a + package's rows divide by its own US row and never by another district + file's at the same stamp (another data year or boundary vintage). A total + is a share denominator, so feed order must not pick it: two different + values for one key raise, and equal values under two record ids name the + smaller id (as the #1229 and #1040 denominators do). """ - totals: dict[tuple[str, str, str, str], _SoiDistrictShareTotal] = {} + totals: dict[tuple[str, str, str, str, str], _SoiDistrictShareTotal] = {} for fact in facts: key = _soi_district_fact_key(fact) if key is None or _geography_level(fact) != "country": @@ -2408,7 +2448,11 @@ def _soi_district_us_totals( source_record_id = _source_record_id(fact) if not source_record_id: continue - total_key = (*key, str(_period_value(fact))) + total_key = ( + _str_at(fact, "layout", "record_set_id"), + *key, + str(_period_value(fact)), + ) candidate = _SoiDistrictShareTotal(_numeric_value(fact), source_record_id) current = totals.get(total_key) if current is not None and current.value != candidate.value: @@ -2424,50 +2468,46 @@ def _soi_district_us_totals( return totals -def _require_reviewed_soi_district_columns( +def _soi_district_source_columns(facts: tuple[object, ...]) -> dict[str, str]: + """Each registered district fact's IRS column, by source record id.""" + return { + _source_record_id(fact): _str_at(fact, "layout", "source_column_id") + for fact in facts + if _soi_district_fact_key(fact) is not None + } + + +def _soi_district_contributors( facts: tuple[object, ...], - *, - target_period: int | str, -) -> None: - """Refuse a district fact the rebase would bind under an unreviewed column. - - ``_rebase_stale_soi_district_distributions`` relies on each registered - measure id reading ``source_column``. A package that maps the measure id - to another column (the CD SALT and PTC measures read the wrong columns, - microcosm#1038), or names none, would be rebased as the wrong concept, so - it refuses the compile. Only a fact with an eligible control is checked; - the others keep their levels as before. This runs on the facts as the - feed carries them, before the crosswalk merges source districts into one - fact with a single contributor's layout. +) -> dict[str, tuple[str, ...]]: + """The source facts behind each crosswalk-merged district fact. - Raises: - UnreviewedSoiNationalControlColumnError: A checked fact reads a column - other than its measure's ``source_column``, or names none. + The crosswalk lists a translated fact's contributions under + ``lineage.congressional_district_vintage_contributions``. A contributor + that proxies a state total for an at-large district names its origin + before ``.state_total_proxy_cd.``. Untranslated facts are their own + source and are not listed. """ - controls: dict[tuple[str, str, str], _SoiTotalControl] | None = None - unreviewed: dict[str, str] = {} + contributors: dict[str, tuple[str, ...]] = {} for fact in facts: - key = _soi_district_fact_key(fact) - if key is None: + if _soi_district_fact_key(fact) is None: continue - if controls is None: - controls = _soi_ht2_national_controls(facts, target_period=target_period) - control = controls.get(key) - if control is None or not _period_not_before( - control.period_key, _period_key(fact) - ): + contributions = _at( + fact, "lineage", "congressional_district_vintage_contributions" + ) + if not contributions: continue - column = _str_at(fact, "layout", "source_column_id") - if column != _SOI_HT2_NATIONAL_CONTROLS[key[0]].source_column: - unreviewed[_source_record_id(fact)] = column or "" - if unreviewed: - sample = dict(sorted(unreviewed.items())[:5]) - raise UnreviewedSoiNationalControlColumnError( - f"{len(unreviewed)} congressional-district fact(s) of registered " - f"measures read an IRS column other than the reviewed one, e.g. " - f"{sample}. Review the column against the district documentation " - "guide and _SOI_HT2_NATIONAL_CONTROLS before rebasing it." + contributors[_source_record_id(fact)] = tuple( + sorted( + { + str(_at(item, "source_record_id")).split(".state_total_proxy_cd.")[ + 0 + ] + for item in contributions + } + ) ) + return contributors #: Control universes admitted for the stale-interest rebase. A stale spec's diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_cd_properties.py b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_cd_properties.py index dc9dee1ff5..c3ac595b85 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_cd_properties.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_cd_properties.py @@ -958,7 +958,27 @@ def surface(rows): factor_band=band, ) - before, after = surface(base), surface(shared) + # A bridged row re-ages its share sibling only when both age on one + # series; this world bridges wages through the return count (AGI series), + # so rebasing the return count there refuses. + cd_state = { + (spec.metadata["state_fips"], spec.metadata["source_measure_id"]): spec + for spec in base + if is_cd_file(spec) and spec.metadata["ledger_geography_level"] == "state" + } + mismatched = any( + _BRIDGES[measure] in factors + and _CD._aging_series(spec) + != _CD._aging_series(cd_state[(state, _BRIDGES[measure])]) + for (state, measure), spec in cd_state.items() + if (state, measure) not in ht2 + ) + before = surface(base) + if mismatched: + with pytest.raises(ValueError, match="re-aged only like a row on its own"): + surface(shared) + return + after = surface(shared) assert [spec.name for spec in after.specs] == [spec.name for spec in before.specs] for old, new in zip(before.specs, after.specs, strict=True): assert new.value == pytest.approx(old.value, rel=1e-12), new.name @@ -1059,49 +1079,218 @@ def test_a_national_share_sibling_keeps_its_bridges_and_recorded_ratios() -> Non } -def test_a_bridge_refuses_a_share_sibling_on_another_aging_series() -> None: - """Re-aging a share's file level like the bridged row stands in for the - sibling's own aging only when both age on one series; a pair on different - series refuses rather than giving the bridged row its sibling's growth.""" +def test_a_share_sibling_re_ages_by_series_not_by_receipt() -> None: + """The series comes from the compiler's aging rule, not the receipt + (#1250 review). A share landed at the build year carries the no-op + ``source_equals_build`` receipt and still re-ages like an unrebased row + on its series; a pair on different series refuses.""" - specs, crosswalk = _bridge_world({"10": 1.0, "11": 1.0, "12": 1.0}, {}) - - def series(spec, source: str): + def amount(name, measure, value, aging, source): + spec = _soi_spec( + name, measure, value, level="state", record_set=_CD_FILE, state="10" + ) return dataclasses.replace( spec, metadata={ **spec.metadata, + "aging_factor": repr(aging), "aging_factor_source": source, - "aging_factor": "1.5", }, ) - rows = [] - for spec in specs: - if spec.metadata["ledger_layout_record_set_spec_id"] != _CD_FILE: - rows.append(spec) - elif spec.metadata["source_measure_id"] == "return_count": - rows.append(_as_national_share(series(spec, "cbo.series_a"), 1.25, 1.5)) - else: - rows.append(series(spec, "chained:irs_soi.x+cbo.series_b")) - with pytest.raises(ValueError, match="ages on 'cbo.series_a'"): - _CD.state_cd_soi_surface( - rows, - state_surface_predicate=_TOOL.soi_surface_predicate("state"), - crosswalk=crosswalk, - level_bridges={"charitable_amount": "return_count"}, + agi_series = "cbo.revenue_projection.ty2024.income_by_source.agi.projected_amount" + charitable = amount( + "cdfile.10_total.charitable_amount", "charitable_amount", 11.0, 1.1, agi_series + ) + sibling = _as_national_share( + amount( + "cdfile.10_total.adjusted_gross_income", + "adjusted_gross_income", + 100.0, + 1.0, + "x", + ), + 1.25, + 1.0, + ) + sibling = dataclasses.replace( + sibling, + metadata={**sibling.metadata, "aging_factor_source": "source_equals_build"}, + ) + assert _CD.cd_file_level(sibling, aged_like=charitable) == pytest.approx(110.0) + wages = amount( + "cdfile.10_total.wages_salaries_amount", + "wages_salaries_amount", + 50.0, + 1.1, + agi_series, + ) + with pytest.raises(ValueError, match="'wages_and_salaries'"): + _CD.cd_file_level(sibling, aged_like=wages) + + +def test_a_build_year_control_keeps_state_cd_bridges() -> None: + """End to end (#1250 review): district itemized and charitable rows stamped + 2023, an itemized control at the 2024 build year, and CBO AGI projections. + The itemized shares land at 2024 unaged while charitable ages 2023 to 2024, + and ``state_cd`` binds the same values as with the district pass off.""" + import microcosm.build.us_runtime.fiscal_targets as fiscal_targets + from microcosm.build.us_runtime import compile_us_fiscal_target_registry + from test_support.microcosm_build.us_fiscal_targets import ( + _cbo_income_source_projection_fact, + _dynamic_ledger_fact, + packaged_reference_facts, + ) + + cd_set = "irs_soi.ty2023.congressional_district_2022.all_returns" + + def fact(record_set, row, measure, value, period, level, geography, spec_id): + result = _dynamic_ledger_fact( + source_record_id=f"{record_set}.{row}.{measure}", + source_name="irs_soi", + measure_id=measure, + value=value, + period_value=period, + geography_level=level, + geography_id=geography, + dimensions={"income_range": "all", "filing_status": "all"}, + layout_record_set_id=record_set, + groupby_dimension="irs_soi.congressional_district", + groupby_value_id=row, ) - # The same chained series, as #1030 writes it, re-ages. - same = [ - series(spec, "chained:irs_soi.x+cbo.series_a") - if spec.metadata["ledger_layout_record_set_spec_id"] == _CD_FILE - and spec.metadata["source_measure_id"] == "charitable_amount" - else spec - for spec in rows + result["layout"]["record_set_spec_id"] = spec_id + if measure == "itemized_deductions_amount": + result["layout"]["source_column_id"] = "A04470" + return result + + cd_spec = "irs_soi.congressional_district_2022.all_returns.v1" + facts = [*packaged_reference_facts()] + for measure, us, states in ( + ( + "itemized_deductions_amount", + 1_000.0, + {"15": (300.0, 200.0), "33": (150.0, 100.0)}, + ), + ("charitable_amount", 100.0, {"15": (30.0, 20.0), "33": (15.0, 10.0)}), + ): + facts.append( + fact(cd_set, "us", measure, us, 2023, "country", "0100000US", cd_spec) + ) + for fips, (first, second) in states.items(): + facts.append( + fact( + cd_set, + f"s{fips}_total", + measure, + first + second, + 2023, + "state", + f"0400000US{fips}", + cd_spec, + ) + ) + for index, value in ((1, first), (2, second)): + facts.append( + fact( + cd_set, + f"s{fips}_0{index}", + measure, + value, + 2023, + "congressional_district", + f"5001700US{fips}0{index}", + cd_spec, + ) + ) + ht2_spec = "irs_soi.historic_table_2.state_broad_totals.v1" + facts.append( + fact( + "irs_soi.ty2022.historic_table_2.us", + "all", + "itemized_deductions_amount", + 1_050.0, + 2022, + "country", + "0100000US", + ht2_spec, + ) + ) + for fips, postal, value in (("15", "hi", 520.0), ("33", "nh", 260.0)): + facts.append( + fact( + f"irs_soi.ty2022.historic_table_2.state_broad.{postal}", + "all", + "itemized_deductions_amount", + value, + 2022, + "state", + f"0400000US{fips}", + ht2_spec, + ) + ) + control = _dynamic_ledger_fact( + source_record_id="irs_soi.ty2024.table_1_2.all_returns.all.total_itemized_deductions_amount", + source_name="irs_soi", + measure_id="total_itemized_deductions_amount", + value=1_200.0, + period_value=2024, + dimensions={"income_range": "all", "filing_status": "all"}, + layout_record_set_id="irs_soi.ty2024.table_1_2.all_returns", + groupby_dimension="us:statutes/26/62#adjusted_gross_income", + groupby_value_id="all", + ) + facts.append(control) + for year, value in ((2022, 100.0), (2023, 110.0), (2024, 121.0)): + facts.append( + _cbo_income_source_projection_fact( + year, "adjusted_gross_income", value=value + ) + ) + crosswalk = [ + { + "source_geography_id": f"5001700US{fips}0{index}", + "target_geography_id": f"5001900US{fips}0{index}", + "weight": 1.0, + } + for fips in ("15", "33") + for index in (1, 2) ] - _CD.state_cd_soi_surface( - same, - state_surface_predicate=_TOOL.soi_surface_predicate("state"), - crosswalk=crosswalk, - level_bridges={"charitable_amount": "return_count"}, + + def surface(): + registry = compile_us_fiscal_target_registry( + facts, + target_period=2024, + congressional_district_vintage_crosswalk=crosswalk, + age_targets=True, + ) + soi = [spec for spec in registry.specs if spec.family == "irs_soi"] + return soi, _CD.state_cd_soi_surface( + soi, + state_surface_predicate=_TOOL.soi_surface_predicate("state"), + crosswalk=pd.DataFrame(crosswalk), + level_bridges={"charitable_amount": "itemized_deductions_amount"}, + factor_band=math.inf, + ) + + soi, rebased = surface() + by_name = {spec.name: spec for spec in soi} + share = by_name[f"{cd_set}.s15_total.itemized_deductions_amount"] + assert share.metadata["aging_factor_source"] == "source_equals_build" + assert share.value == pytest.approx(600.0) + charitable = by_name[f"{cd_set}.s15_total.charitable_amount"] + assert charitable.value == pytest.approx(55.0) + + original = fiscal_targets._rebase_stale_soi_district_distributions + try: + fiscal_targets._rebase_stale_soi_district_distributions = ( + lambda registry, facts, **_: registry + ) + _, levels = surface() + finally: + fiscal_targets._rebase_stale_soi_district_distributions = original + assert {spec.name: spec.value for spec in rebased.specs} == pytest.approx( + {spec.name: spec.value for spec in levels.specs}, rel=1e-12 + ) + assert ( + rebased.receipt["level_bridge_factor_by_measure"]["charitable_amount"]["n"] == 2 ) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_soi_district_national_controls.py b/packages/microcosm-build/tests/engine_free/us/test_us_soi_district_national_controls.py index a80f740700..5fcdb001c2 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_soi_district_national_controls.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_soi_district_national_controls.py @@ -528,6 +528,32 @@ def facts(column: object = _REVIEWED) -> list[dict[str, object]]: _compile(facts(column="A00101")) +def test_the_column_guard_skips_facts_that_compile_to_no_target() -> None: + """The guard checks the rows the pass rebases (#1250 review). A stale AGI + slice compiles to no target at another period, and a superseded vintage + loses latest-vintage selection, so neither is checked whatever its + column.""" + measure_id = "adjusted_gross_income" + stale_slice = _cd_agi_slice_fact(measure_id, 120.0, stamp=2023, column="A00101") + specs = _by_record( + _compile([*_cd_facts(measure_id), stale_slice, _control_fact(measure_id)]) + ) + assert stale_slice["lineage"]["source_record_id"] not in specs + assert specs[f"{_record_set(2023)}.hi_total.{measure_id}"].value == 550.0 + + measure_id = "return_count" + superseded = _cd_facts(measure_id, stamp=2022, column="N2") + current = _cd_facts(measure_id, stamp=2024) + rows = _cd_rows( + _compile([*superseded, *current, _control_fact(measure_id, period=2024)]), + measure_id, + ) + assert rows["hi_total"].metadata["ledger_source_record_id"] == ( + f"{_record_set(2024)}.hi_total.{measure_id}" + ) + assert rows["hi_total"].value == 550.0 + + # -------------------------------------------------------------------------- # The denominator @@ -550,6 +576,23 @@ def test_conflicting_district_us_totals_refuse_the_compile() -> None: _compile([rival, us, *rest, _control_fact(measure_id)]) +def test_rows_never_borrow_another_district_files_us_row() -> None: + """Each package divides by its own US row (#1250 review). A 22incd + package without one keeps no rows beside the control, even when another + district file at the same stamp carries a US row of the measure.""" + measure_id = "return_count" + other = _cd_fact( + measure_id, 2_000.0, row="us", level="country", geography_id="0100000US" + ) + other_set = "irs_soi.ty2023.congressional_district_2023.all_returns" + other["layout"]["record_set_id"] = other_set + other["lineage"]["source_record_id"] = f"{other_set}.us.{measure_id}" + registry = _compile( + [*_cd_facts(measure_id, us=None), other, _control_fact(measure_id)] + ) + assert _cd_rows(registry, measure_id) == {} + + def test_equal_district_us_totals_name_the_smaller_record_id() -> None: """Equal twins are one total; the named denominator is the smaller record id in either feed order, and both twins retire.""" diff --git a/tools/us_acs_local_cd_surface.py b/tools/us_acs_local_cd_surface.py index 525b8259ef..d2dff4709c 100644 --- a/tools/us_acs_local_cd_surface.py +++ b/tools/us_acs_local_cd_surface.py @@ -231,24 +231,32 @@ def cd_file_level(spec, *, aged_like=None) -> float: return value value /= float(metadata["uprating_factor"]) if aged_like is not None: - other = _metadata(aged_like) - if _aging_series(other) != _aging_series(metadata): + if _aging_series(aged_like) != _aging_series(spec): raise ValueError( - f"{spec.name} ages on {_aging_series(metadata)!r} and " - f"{aged_like.name} on {_aging_series(other)!r}; a share's file " - "level can be re-aged only like a row on its own series." + f"{spec.name} ages on {_aging_series(spec)!r} and " + f"{aged_like.name} on {_aging_series(aged_like)!r}; a share's " + "file level can be re-aged only like a row on its own series." ) - value *= float(other.get("aging_factor", "1")) / float( + value *= float(_metadata(aged_like).get("aging_factor", "1")) / float( metadata.get("aging_factor", "1") ) return value -def _aging_series(metadata: Mapping[str, str]) -> str: - """The growth series a spec's aging ends on (its last link, if chained).""" +def _aging_series(spec) -> str: + """The growth series target aging ages ``spec`` on, from the spec alone. - source = str(metadata.get("aging_factor_source", "not_dollar_amount")) - return source.removeprefix("chained:").rsplit("+", 1)[-1] + Read from the compiler's rule, not the receipt: a receipt can be a no-op + (``source_equals_build`` for a share landed at the build period) that + names no series. Rows that never age return why. + """ + + from microcosm.build.us_runtime import target_aging + + reason = target_aging._not_ageable_reason(spec) + if reason is not None: + return reason + return target_aging._cbo_income_source_for_spec(spec) def soi_materializer_semantics(spec) -> tuple: From abe8df5daf064593968b7dc17e37c6597c2b6808 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 22:13:13 -0400 Subject: [PATCH 09/12] Require district source columns; test the guard on merged and US rows source_columns is now a required argument of the district pass: translated facts alone cannot name their sources' columns. New tests: a reviewed crosswalk merge compiles and rebases, and a US row read from another column refuses although every state and district row is reviewed. Co-Authored-By: Claude Opus 5.5 --- .../build/us_runtime/fiscal_targets.py | 12 +++---- .../test_us_soi_district_national_controls.py | 32 ++++++++++++++++++- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py b/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py index 781a5c0245..4b3d64599f 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py @@ -2275,7 +2275,7 @@ def _rebase_stale_soi_district_distributions( facts: tuple[object, ...], *, target_period: int | str, - source_columns: Mapping[str, str] | None = None, + source_columns: Mapping[str, str], ) -> TargetRegistry: """Use congressional-district rows of the registered columns as shares. @@ -2313,10 +2313,10 @@ def _rebase_stale_soi_district_distributions( would be rebased as the wrong concept, so every source fact of a row rebased here, and its denominator, must read that column. ``source_columns`` maps each district fact's record id to its column as - the feed carries it, before the crosswalk merges source districts into - one fact with a single contributor's layout; without it the columns are - read from ``facts``. Rows that keep their levels, and facts that compile - to no target, are not checked. + the feed carries it, read before the crosswalk merges source districts + into one fact with a single contributor's layout + (``_soi_district_source_columns``). Rows that keep their levels, and + facts that compile to no target, are not checked. Raises: UnreviewedSoiNationalControlColumnError: A rebased row or its @@ -2325,8 +2325,6 @@ def _rebase_stale_soi_district_distributions( controls = _soi_ht2_national_controls(facts, target_period=target_period) us_totals = _soi_district_us_totals(facts) - if source_columns is None: - source_columns = _soi_district_source_columns(facts) contributors = _soi_district_contributors(facts) unreviewed: dict[str, str] = {} specs: list[TargetSpec] = [] diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_soi_district_national_controls.py b/packages/microcosm-build/tests/engine_free/us/test_us_soi_district_national_controls.py index 5fcdb001c2..fd2b5015bc 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_soi_district_national_controls.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_soi_district_national_controls.py @@ -457,9 +457,39 @@ def test_the_column_guard_sees_each_source_district_before_the_crosswalk() -> No "target_geography_id": "5001900US1501", "weight": 1.0, }, + { + "source_geography_id": "5001700US1502", + "target_geography_id": "5001900US1502", + "weight": 1.0, + }, ] - with pytest.raises(fiscal_targets.UnreviewedSoiNationalControlColumnError): + with pytest.raises( + fiscal_targets.UnreviewedSoiNationalControlColumnError, match="hi_02" + ): _compile([*facts, _control_fact(measure_id)], crosswalk=crosswalk) + # With both contributors reviewed, the merged district is rebased. + hi_02["layout"]["source_column_id"] = "A01700" + specs = _compile([*facts, _control_fact(measure_id)], crosswalk=crosswalk).specs + (merged,) = [ + spec + for spec in specs + if spec.metadata.get("congressional_district_geoid") == "1501" + and spec.metadata.get("source_measure_id") == measure_id + ] + # 300 + 200 / 2 of the file's districts, times 1,100 / 1,000. + assert merged.value == pytest.approx(440.0, rel=1e-12) + + +def test_the_column_guard_checks_the_denominator() -> None: + """A share divides by the US row, so a US row read from another column + refuses even when every state and district row is reviewed.""" + measure_id = "unemployment_compensation_amount" + facts = _cd_facts(measure_id) + facts[0]["layout"]["source_column_id"] = "A02310" + with pytest.raises( + fiscal_targets.UnreviewedSoiNationalControlColumnError, match=r"\.us\." + ): + _compile([*facts, _control_fact(measure_id)]) def _cd_agi_slice_fact( From 4b75b2e5a57abd9212203cc656932b24cfefb919 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 22:15:13 -0400 Subject: [PATCH 10/12] Document the per-package denominators and the in-pass column guard Co-Authored-By: Claude Opus 5.5 --- docs/us-acs-local-soi-target-surface.md | 4 +++- docs/us-soi-national-controls.md | 23 ++++++++++++++--------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/docs/us-acs-local-soi-target-surface.md b/docs/us-acs-local-soi-target-surface.md index dfd1b92019..e4310e99cb 100644 --- a/docs/us-acs-local-soi-target-surface.md +++ b/docs/us-acs-local-soi-target-surface.md @@ -305,7 +305,9 @@ sibling's band verdict compares the two files' levels, its median re-aged by the same constant. Re-aging like the bridged row stands in for the sibling's own aging only when both age on one series, as every registered bridge does (counts unaged, amounts on the CBO AGI series); a pair on different series -refuses. Reading the share as compiled would lift nothing: on the pinned +refuses. The series comes from target aging's own rule, not the aging +receipt, which is a no-op (`source_equals_build`) for a share landed at the +build year. Reading the share as compiled would lift nothing: on the pinned feed the bridged charitable and interest-paid amounts would fall 9.07%, QBI amounts 5.63% and the bridged counts 2.65%. A property test checks that national-share district rows move no `state_cd` value, drop, verdict or diff --git a/docs/us-soi-national-controls.md b/docs/us-soi-national-controls.md index f3e2a76f6f..421ad0e971 100644 --- a/docs/us-soi-national-controls.md +++ b/docs/us-soi-national-controls.md @@ -257,15 +257,20 @@ interest) and #1040 (capital gains) use for other district families: value this pass removes, so it is dropped. That is the district rule of #1229 and #1040. The HT2 pass keeps HT2 rows as levels in the same case. Neither case occurs on the pinned feed. -- **One denominator.** Two different district US values for one key refuse - the compile; equal twins name the smaller record id, so feed order never - picks the denominator. -- **Reviewed columns only.** A district fact of a registered measure whose - control is eligible must read the register's `source_column`. Otherwise - the compile refuses (`UnreviewedSoiNationalControlColumnError`), as it - does for a missing column. The check runs before the crosswalk, which - merges source districts into one fact with a single contributor's layout. - The district package's SALT and PTC measures read the wrong columns +- **One denominator.** Each package divides by its own US row: the key + carries the record set, so another district file at the same stamp (a + different data year or boundary vintage) never supplies it. Two different + US values for one key refuse the compile; equal twins name the smaller + record id, so feed order never picks the denominator. +- **Reviewed columns only.** Every source fact of a rebased row, and its US + row, must read the register's `source_column`. Otherwise the compile + refuses (`UnreviewedSoiNationalControlColumnError`), as it does for a + missing column. The columns are read as the feed carries them, before the + crosswalk merges source districts into one fact with a single + contributor's layout, and each merged district is traced back to its + contributors. Rows that keep their levels, and facts that compile to no + target (a stale AGI slice, a superseded vintage), are not checked. The + district package's SALT and PTC measures read the wrong columns (microcosm#1038), so a re-pin could do the same here. - **The stamp cancels.** Row and US row carry the same stamp, so stamping the file truthfully at 2022 gives the same values From 5768ffd49f5b2c391d948220d05a0f0e4c532940 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 22:36:57 -0400 Subject: [PATCH 11/12] Read each crosswalk contributor's column from the lineage (#1250 Sol delta review) From the GPT-6.1 Sol delta review (review-sol-r2.md: the three earlier P2s resolved, two new P2): - A duplicate record id could hide an unreviewed contributor, because the column inventory kept the last column per id. - Compiling the translator's output refused, because the inventory held only the incoming ids while the guard looked up ancestral ones. The crosswalk now records each contributor's source_column_id in lineage.congressional_district_vintage_contributions, and the guard reads every contributor's column from there (any other fact is its own source; facts sharing an id all count). The pre-translation inventory and its argument are gone. Tests: a duplicate merged by the crosswalk refuses in either order; translated facts compile to the crosswalk compile's registry; a recorded-ratio example with a share aged unlike the file. The pinned compile, state_cd values and receipt are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../congressional_district_vintage.py | 4 + .../build/us_runtime/fiscal_targets.py | 79 ++++++++----------- .../us/test_us_acs_local_cd_properties.py | 7 +- .../test_us_soi_district_national_controls.py | 55 +++++++++++++ 4 files changed, 97 insertions(+), 48 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/congressional_district_vintage.py b/packages/microcosm-build/src/microcosm/build/us_runtime/congressional_district_vintage.py index 94b1b03435..43fa757d3e 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/congressional_district_vintage.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/congressional_district_vintage.py @@ -707,6 +707,10 @@ def _append_lineage_contribution( contributions.append( { "source_record_id": _source_record_id(source_fact), + # The contributor's own column: the merged fact keeps only the + # first contributor's layout, and the SOI district rebase checks + # every contributor (fiscal_targets, microcosm#1250). + "source_column_id": _str_at(source_fact, "layout", "source_column_id"), "source_geography_id": source_geography_id, "target_geography_id": target_geography_id, "share": share, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py b/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py index 4b3d64599f..2b3b5a403e 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py @@ -1036,9 +1036,6 @@ def compile_us_fiscal_target_registry( ``period_contract_waiver`` metadata in diagnostics. """ materialized_facts = tuple(facts) - # Read before the crosswalk merges source districts into one fact that - # keeps a single contributor's layout. - district_source_columns = _soi_district_source_columns(materialized_facts) if congressional_district_vintage_crosswalk is not None: materialized_facts = translate_congressional_district_facts_to_current_vintage( materialized_facts, @@ -1081,7 +1078,6 @@ def compile_us_fiscal_target_registry( registry, materialized_facts, target_period=target_period, - source_columns=district_source_columns, ) registry = _rebase_soi_state_agi_bands(registry, materialized_facts) registry = apply_ledger_target_profile( @@ -2275,7 +2271,6 @@ def _rebase_stale_soi_district_distributions( facts: tuple[object, ...], *, target_period: int | str, - source_columns: Mapping[str, str], ) -> TargetRegistry: """Use congressional-district rows of the registered columns as shares. @@ -2311,10 +2306,9 @@ def _rebase_stale_soi_district_distributions( package that maps the measure id to another column (the district SALT and PTC measures read the wrong columns, microcosm#1038), or names none, would be rebased as the wrong concept, so every source fact of a row - rebased here, and its denominator, must read that column. - ``source_columns`` maps each district fact's record id to its column as - the feed carries it, read before the crosswalk merges source districts - into one fact with a single contributor's layout + rebased here, and its denominator, must read that column. A district + merged by the crosswalk keeps only its first contributor's layout, so its + contributors are read from the lineage, which records each one's column (``_soi_district_source_columns``). Rows that keep their levels, and facts that compile to no target, are not checked. @@ -2325,7 +2319,7 @@ def _rebase_stale_soi_district_distributions( controls = _soi_ht2_national_controls(facts, target_period=target_period) us_totals = _soi_district_us_totals(facts) - contributors = _soi_district_contributors(facts) + source_columns = _soi_district_source_columns(facts) unreviewed: dict[str, str] = {} specs: list[TargetSpec] = [] for spec in registry.specs: @@ -2347,15 +2341,16 @@ def _rebase_stale_soi_district_distributions( us_total = us_totals.get((record_set_id, *key, source_period)) if us_total is None or us_total.value == 0: continue - record_id = spec.metadata.get("ledger_source_record_id", "") expected = _SOI_HT2_NATIONAL_CONTROLS[measure_id].source_column - for source_id in ( - *contributors.get(record_id, (record_id,)), + for checked_id in ( + spec.metadata.get("ledger_source_record_id", ""), us_total.source_record_id, ): - column = source_columns.get(source_id, "") - if column != expected: - unreviewed[source_id] = column or "" + for source_id, column in source_columns.get( + checked_id, ((checked_id, ""),) + ): + if column != expected: + unreviewed[source_id] = column or "" concept = _SOI_HT2_NATIONAL_CONTROLS[measure_id].concept factor = control.value / us_total.value specs.append( @@ -2466,46 +2461,38 @@ def _soi_district_us_totals( return totals -def _soi_district_source_columns(facts: tuple[object, ...]) -> dict[str, str]: - """Each registered district fact's IRS column, by source record id.""" - return { - _source_record_id(fact): _str_at(fact, "layout", "source_column_id") - for fact in facts - if _soi_district_fact_key(fact) is not None - } - - -def _soi_district_contributors( +def _soi_district_source_columns( facts: tuple[object, ...], -) -> dict[str, tuple[str, ...]]: - """The source facts behind each crosswalk-merged district fact. +) -> dict[str, tuple[tuple[str, str], ...]]: + """The (source record id, IRS column) of each source behind a district fact. - The crosswalk lists a translated fact's contributions under - ``lineage.congressional_district_vintage_contributions``. A contributor - that proxies a state total for an at-large district names its origin - before ``.state_total_proxy_cd.``. Untranslated facts are their own - source and are not listed. + A district the crosswalk merged lists every contributor, with its own + column, under ``lineage.congressional_district_vintage_contributions``; + any other fact is its own source. Facts that share a record id all count, + so a duplicate cannot hide another column. """ - contributors: dict[str, tuple[str, ...]] = {} + sources: dict[str, list[tuple[str, str]]] = {} for fact in facts: if _soi_district_fact_key(fact) is None: continue contributions = _at( fact, "lineage", "congressional_district_vintage_contributions" ) - if not contributions: - continue - contributors[_source_record_id(fact)] = tuple( - sorted( - { - str(_at(item, "source_record_id")).split(".state_total_proxy_cd.")[ - 0 - ] - for item in contributions - } - ) + entries = ( + [ + ( + str(_at(item, "source_record_id") or ""), + str(_at(item, "source_column_id") or ""), + ) + for item in contributions + ] + if contributions + else [ + (_source_record_id(fact), _str_at(fact, "layout", "source_column_id")) + ] ) - return contributors + sources.setdefault(_source_record_id(fact), []).extend(entries) + return {record_id: tuple(entries) for record_id, entries in sources.items()} #: Control universes admitted for the stale-interest rebase. A stale spec's diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_cd_properties.py b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_cd_properties.py index c3ac595b85..31091e39d6 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_cd_properties.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_cd_properties.py @@ -1033,7 +1033,10 @@ def verdicts(receipt): ), (new.name, key) -def test_a_national_share_sibling_keeps_its_bridges_and_recorded_ratios() -> None: +@pytest.mark.parametrize("control_aging", [1.0, 2.0]) +def test_a_national_share_sibling_keeps_its_bridges_and_recorded_ratios( + control_aging: float, +) -> None: """State 14's two return-count levels disagree 2x, so its bridge is out of band. Binding the district file's return counts as national shares (x1.5) leaves every bound value, the out-of-band rows and their recorded ratios @@ -1049,7 +1052,7 @@ def test_a_national_share_sibling_keeps_its_bridges_and_recorded_ratios() -> Non for spec in specs ] shared = [ - _as_national_share(spec, 1.5, 1.0) + _as_national_share(spec, 1.5, control_aging) if spec.metadata["ledger_layout_record_set_spec_id"] == _CD_FILE and spec.metadata["source_measure_id"] == "return_count" else spec diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_soi_district_national_controls.py b/packages/microcosm-build/tests/engine_free/us/test_us_soi_district_national_controls.py index fd2b5015bc..013695aae9 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_soi_district_national_controls.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_soi_district_national_controls.py @@ -480,6 +480,61 @@ def test_the_column_guard_sees_each_source_district_before_the_crosswalk() -> No assert merged.value == pytest.approx(440.0, rel=1e-12) +def _identity_crosswalk() -> list[dict[str, object]]: + return [ + { + "source_geography_id": f"5001700US{geoid}", + "target_geography_id": f"5001900US{geoid}", + "weight": 1.0, + } + for _, _, geoids in STATES[:2] + for geoid in geoids + ] + + +@pytest.mark.parametrize("duplicate_first", [True, False], ids=["first", "last"]) +def test_a_duplicate_record_id_cannot_hide_another_column(duplicate_first) -> None: + """Two facts with one record id both count (#1250 review). Without a + crosswalk the compile already refuses the duplicate id; the crosswalk + merges the copies into one district, and a copy of Hawaii's first district + read from N2 then refuses in either feed order.""" + measure_id = "return_count" + facts = _cd_facts(measure_id) + original = next( + fact + for fact in facts + if fact["lineage"]["source_record_id"].endswith(f"hi_01.{measure_id}") + ) + duplicate = {**original, "layout": {**original["layout"], "source_column_id": "N2"}} + index = facts.index(original) + facts.insert(index if duplicate_first else index + 1, duplicate) + with pytest.raises(ValueError, match="matched multiple Ledger facts"): + _compile([*facts, _control_fact(measure_id)]) + with pytest.raises( + fiscal_targets.UnreviewedSoiNationalControlColumnError, match="'N2'" + ): + _compile([*facts, _control_fact(measure_id)], crosswalk=_identity_crosswalk()) + + +def test_translated_facts_compile_as_the_crosswalk_compile_does() -> None: + """The translator's output is itself a valid feed: each merged district + carries its contributors' columns, so compiling translated facts gives + the same registry as compiling the source facts with the crosswalk.""" + from microcosm.build.us_runtime.congressional_district_vintage import ( + translate_congressional_district_facts_to_current_vintage, + ) + + measure_id = "taxable_social_security_returns" + facts = [*_cd_facts(measure_id), _control_fact(measure_id)] + crosswalk = _identity_crosswalk() + direct = _compile(facts, crosswalk=crosswalk) + translated = _compile( + translate_congressional_district_facts_to_current_vintage(facts, crosswalk) + ) + assert translated.version == direct.version + assert _cd_rows(translated, measure_id)["hi_total"].value == 550.0 + + def test_the_column_guard_checks_the_denominator() -> None: """A share divides by the US row, so a US row read from another column refuses even when every state and district row is reviewed.""" From c13a31b6746c80492d0e167af36b1084606ee122 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 22:37:54 -0400 Subject: [PATCH 12/12] Document the lineage-based column check Co-Authored-By: Claude Opus 5.5 --- docs/us-soi-national-controls.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/us-soi-national-controls.md b/docs/us-soi-national-controls.md index 421ad0e971..ed079a5239 100644 --- a/docs/us-soi-national-controls.md +++ b/docs/us-soi-national-controls.md @@ -265,11 +265,11 @@ interest) and #1040 (capital gains) use for other district families: - **Reviewed columns only.** Every source fact of a rebased row, and its US row, must read the register's `source_column`. Otherwise the compile refuses (`UnreviewedSoiNationalControlColumnError`), as it does for a - missing column. The columns are read as the feed carries them, before the - crosswalk merges source districts into one fact with a single - contributor's layout, and each merged district is traced back to its - contributors. Rows that keep their levels, and facts that compile to no - target (a stale AGI slice, a superseded vintage), are not checked. The + missing column. The crosswalk merges source districts into one fact with a + single contributor's layout, so it records each contributor's column in + the merged fact's lineage, and the check reads every one (facts sharing a + record id all count). Rows that keep their levels, and facts that compile + to no target (a stale AGI slice, a superseded vintage), are not checked. The district package's SALT and PTC measures read the wrong columns (microcosm#1038), so a re-pin could do the same here. - **The stamp cancels.** Row and US row carry the same stamp, so stamping the