From 2b7164e1533a0f7d777643016a3ab402ce7b9cc3 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 10:28:11 -0400 Subject: [PATCH 1/3] Rebase congressional-district taxable interest onto Table 4.3 as shares (#1038) The congressional-district file (22incd.csv, TY2022 data stamped ty2023) was barred from being a taxable-interest control but still shipped its rows as TY2022 levels, and its US row compiled as a second national target. On the pinned 51aa40f feed's full surface its US row, states and districts each summed to $134.6B against Table 4.3's $340.4B (0.395x) and 48.5M against 54.2M returns (0.896x). The interest rebase now treats Historic Table 2 and the district file as share families: each row is its share of its own family's national all-AGI row times the Pub 1304 control (one factor per family, quantity and period), each family's US row retires, and each share row names its denominator in soi_share_total_source_record_id. Conflicting family totals refuse; equal ones name the smaller id. A share row with no family total is dropped beside an eligible control, as capital gains already does. Pinned feed: 974 district-file values move by x2.52943 (amounts) and x1.11577 (returns); the two district-file US rows retire (33,250 -> 33,248); 1,052 HT2 interest rows gain the denominator key, so national_state moves e1f9906bf0e0 -> 1b318eafb174 with no value change. state_cd values, counts and band verdicts are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../1038-cd-taxable-interest-shares.fixed.md | 1 + docs/us-acs-local-soi-target-surface.md | 16 +- .../build/us_runtime/fiscal_targets.py | 216 +++-- .../engine_free/us/test_us_fiscal_targets.py | 784 +++++++++++++++++- tools/us_acs_local_cd_surface.py | 4 +- 5 files changed, 936 insertions(+), 85 deletions(-) create mode 100644 changelog.d/1038-cd-taxable-interest-shares.fixed.md diff --git a/changelog.d/1038-cd-taxable-interest-shares.fixed.md b/changelog.d/1038-cd-taxable-interest-shares.fixed.md new file mode 100644 index 0000000000..76cfbb029e --- /dev/null +++ b/changelog.d/1038-cd-taxable-interest-shares.fixed.md @@ -0,0 +1 @@ +Rebase congressional-district taxable interest onto the Table 4.3 control as shares of the district file's own US row, as Historic Table 2's rows already are, and retire that US row: on the full surface the file's states and districts summed to $134.6B against the $340.4B national target (0.395x) and 48.5M against 54.2M returns. diff --git a/docs/us-acs-local-soi-target-surface.md b/docs/us-acs-local-soi-target-surface.md index e072362569..00a2dbd610 100644 --- a/docs/us-acs-local-soi-target-surface.md +++ b/docs/us-acs-local-soi-target-surface.md @@ -179,9 +179,11 @@ Why Historic Table 2: - **The district file's state levels are off in ways the shares are not.** The feed labels the file "Congressional District Data 2022" but stamps it tax year 2023 (PR #1040, #1030), so it is aged one year less than Historic Table - 2; its taxable-interest rows are not rebased to Table 4.3 (Historic Table - 2's are; the rebase factor here is about 2.47). A within-state share is - unaffected by a level factor common to the state. + 2. Its taxable-interest rows are now shares of its own US row times Table + 4.3, like Historic Table 2's, but that level still differs from Historic + Table 2's state levels by Historic Table 2's other-areas sliver (its states + sum to 98.4% of the control). A within-state share is unaffected by a level + factor common to the state. - **Two vintages of one concept are contradictory constraints**; the solve can only split the difference. @@ -199,8 +201,12 @@ district rows, are mostly 1.00 to 1.10. chained SOI and CBO series (about 1.120). The smallest amount factor, 1.03, is that aging ratio alone. Correcting the district file's stamp (#1030) would remove the aging part only. -- **Two factors reflect known level differences:** taxable interest (2.35 to - 2.70, the Table 4.3 rebase) and capital-gains amounts (0.77 to 1.00). +- **One factor reflects a known level difference:** capital-gains amounts + (0.77 to 1.00). Taxable interest's factors (2.35 to 2.70) carried the Table + 4.3 rebase that Historic Table 2's rows had and the district file's lacked; + since the district file's interest rows take the same control as shares of + its US row, they divide by 2.529 (amounts) and 1.116 (returns), which moves + every state alike, so the factor band's verdicts are unchanged. - **The largest single-state factors** were on qualified and ordinary dividends (Hawaii, 2.68 and 2.17), tax-exempt interest (Utah, 1.43) and rental income (New York, 0.72), where the two tables disagree about one diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py b/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py index 7df8db9b95..28fdb0fa6c 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py @@ -1715,7 +1715,7 @@ def _rebase_stale_soi_taxable_interest_distributions( *, target_period: int | str, ) -> TargetRegistry: - """Use stale HT2 taxable-interest rows as shares, not hard old-year totals. + """Use stale taxable-interest rows as shares, not hard old-year totals. The microcosm#489 adjudication, with every value from the official IRS workbooks: HT2 is a correct photo of its own tax year (TY2022 US A00300 @@ -1732,23 +1732,44 @@ def _rebase_stale_soi_taxable_interest_distributions( all-row) while sitting exactly on the Table 4.3 truth. Remedy (the net-capital-gains doctrine, x0.7719 precedent): every stale - HT2 interest row rebases by ONE national realized factor onto the - latest Pub 1304-class national actual, preserving TY2022 cross-sectional - shares; the stale national all-AGI rows retire because the active - control's own compiled row owns the national concept. Congressional- + interest row of a share family (``_soi_taxable_interest_share_family``: + Historic Table 2 and the congressional-district file) ships as its share + of its OWN family's national all-AGI row at its own period, scaled to the + latest Pub 1304-class national actual — one national realized factor per + family, quantity and period, preserving TY2022 cross-sectional shares. + Each family's national all-AGI row retires because the active control's + own compiled row owns the national concept, and each share row names its + denominator in ``soi_share_total_source_record_id``. Congressional- district aggregates are never controls: the CD table is a processing- window subset of the filing universe (US A00300 $123.791B = 92.7% of the same-year Table 1.4 actual) and the feed's ty2023 stamp on 22incd.csv is a vintage error — IRS's latest congressional-district publication is tax year 2022. The rebase lands values at the CONTROL's period; target aging owns the remaining links (microcosm#488). + + The congressional-district file joined the share families after + microcosm#1038's sweep. Before, it was barred from being a control but + still shipped its TY2022 rows as levels, its US row a second national + target beside Table 4.3: on the full surface its US row, its states and + its districts each summed to $134.6B against $340.4B (0.395x) and to + 48.5M returns against 54.2M (0.896x). As shares of the file's own US row, + its states and its districts each sum to the control; the stamp cancels + in the share; and because one factor scales a family's state and + district rows alike, the district-to-state reconciliation that runs next + keeps factor 1. + + A share row whose own family carries no nonzero national total cannot + be a share. It is dropped if it is an AGI slice (unverifiable) or if a + control is eligible (a stale level beside a newer control is the defect + this pass removes); only a full-range row with no control anywhere stays + a level of its own, aged generically. """ controls = _soi_taxable_interest_active_totals( facts, target_period=target_period, ) - stale_national_totals = _soi_taxable_interest_stale_national_totals(facts) + share_totals = _soi_taxable_interest_share_national_totals(facts) # The all-returns concept has exactly one live national owner: when an # exact all-returns control exists for a (measure, status), any compiled # excluding-dependents national surrogate of the same pair retires — the @@ -1761,11 +1782,11 @@ def _rebase_stale_soi_taxable_interest_distributions( } specs: list[TargetSpec] = [] for spec in registry.specs: - kind = _soi_taxable_interest_kind(spec) - if kind is None: + if _soi_taxable_interest_kind(spec) is None: specs.append(spec) continue - if not _is_stale_soi_historic_taxable_interest_spec(spec): + family = _soi_taxable_interest_share_family_from_spec(spec) + if family is None: if _is_retired_soi_interest_bridge_surrogate_spec( spec, retired_bridge_owners ): @@ -1778,36 +1799,40 @@ def _rebase_stale_soi_taxable_interest_distributions( spec_universe = spec.metadata.get("soi_return_universe", "all_returns") source_period = spec.metadata.get("source_period", "") source_total = ( - stale_national_totals.get( - (measure_id, status, spec_universe, source_period) - ) + share_totals.get((family, measure_id, status, spec_universe, source_period)) if source_period else None ) - if source_total in (None, 0): - # A stale AGI slice whose own family carries no national total - # is unverifiable — dropped, never shipped unanchored. The - # full-range rows ARE levels of their own and stay. - if _bounds_from_metadata(spec) != (-float("inf"), float("inf")): - continue - specs.append(spec) - continue # Eligibility is part of SELECTION: an unusably old control in a # preferred universe must not shadow an eligible one behind it. - source_period_key = _period_key_from_value(source_period) control: _SoiTotalControl | None = None control_universe = "" - for universe in _soi_taxable_interest_control_universe_preference( - spec_universe - ): - candidate = controls.get((measure_id, status, universe)) - if candidate is None: - continue - if not _period_not_before(candidate.period_key, source_period_key): + if source_period: + source_period_key = _period_key_from_value(source_period) + for universe in _soi_taxable_interest_control_universe_preference( + spec_universe + ): + candidate = controls.get((measure_id, status, universe)) + if candidate is None: + continue + if not _period_not_before(candidate.period_key, source_period_key): + continue + control = candidate + control_universe = universe + break + if source_total is None or source_total.value == 0: + # No denominator of its own family: never divided by another + # family's total. An AGI slice is unverifiable, and beside an + # eligible control a stale level is the defect itself, so both + # are dropped; a full-range row with no control anywhere is a + # level of its own and stays. + if control is not None or _bounds_from_metadata(spec) != ( + -float("inf"), + float("inf"), + ): continue - control = candidate - control_universe = universe - break + specs.append(spec) + continue if control is None: specs.append(spec) continue @@ -1815,7 +1840,7 @@ def _rebase_stale_soi_taxable_interest_distributions( if _is_national_all_agi_spec(spec): continue - factor = control.value / source_total + factor = control.value / source_total.value metadata = { **dict(spec.metadata), "uprating_index": _soi_total_uprating_index(measure_id), @@ -1828,6 +1853,9 @@ def _rebase_stale_soi_taxable_interest_distributions( "uprating_index_source_record_id": control.source_record_id, "uprating_factor": _format_float(factor), "stale_distribution_rebased_to_active_total": "true", + # The share's denominator: this family's own national row, from + # the same release as the row itself. + "soi_share_total_source_record_id": source_total.source_record_id, } if control_universe != spec_universe: # A bridged control is declared, never silent: the manifest must @@ -1843,10 +1871,10 @@ def _is_retired_soi_interest_bridge_surrogate_spec( ) -> bool: """Whether this national excluding-dependents row lost concept ownership. - True only for a non-HT2, national, full-AGI-range interest spec in the - bridge universe whose (measure, status) has an exact all-returns control - compiled — the all-returns row owns the concept and the surrogate must - not also bind. + True only for a national, full-AGI-range interest spec of no share family + in the bridge universe whose (measure, status) has an exact all-returns + control compiled — the all-returns row owns the concept and the surrogate + must not also bind. """ if spec.family != "irs_soi": return False @@ -1875,8 +1903,6 @@ def _soi_taxable_interest_active_totals( key = _soi_taxable_interest_control_key_from_fact(fact) if key is None: continue - if _is_stale_soi_historic_taxable_interest_fact(fact): - continue period_key = _period_key(fact) if not _not_after_target_period(period_key, target_period_key): continue @@ -1899,14 +1925,42 @@ def _soi_taxable_interest_active_totals( return controls -def _soi_taxable_interest_stale_national_totals( +class AmbiguousSoiTaxableInterestShareTotalError(ValueError): + """One taxable-interest share family carries two national totals for one key.""" + + +@dataclass(frozen=True) +class _SoiTaxableInterestShareTotal: + value: float + source_record_id: str + + +def _soi_taxable_interest_share_national_totals( facts: tuple[object, ...], -) -> dict[tuple[str, str, str, str], float]: - totals: dict[tuple[str, str, str, str], float] = {} +) -> dict[tuple[str, str, str, str, str], _SoiTaxableInterestShareTotal]: + """Each share family's national all-AGI total, the denominator of its shares. + + Keyed by (family, measure, filing status, universe, period), so a Historic + Table 2 row divides by the Historic Table 2 US row and a + congressional-district row by that file's US row, never by another + family's. Two different values under one key refuse the compile instead of + letting feed order pick the denominator; equal values under two record ids + keep the smaller id, so the named denominator does not depend on feed order + either. Congressional-district packages of different boundary vintages for + one tax year share the ``congressional_district`` family, so their US rows + must agree or the compile refuses. + """ + totals: dict[tuple[str, str, str, str, str], _SoiTaxableInterestShareTotal] = {} for fact in facts: - if not _is_stale_soi_historic_taxable_interest_fact(fact): + if _source_name(fact) != "irs_soi": continue - if _geography_level(fact) != "country": + measure_id = _measure_id(fact) + if _soi_taxable_interest_kind_from_measure(measure_id) is None: + continue + family = _soi_taxable_interest_share_family( + _str_at(fact, "layout", "record_set_id") + ) + if family is None or _geography_level(fact) != "country": continue # Full AGI range only — filing-status slices keep their own totals # (keyed by status below), matching the capital-gains template. @@ -1915,14 +1969,29 @@ def _soi_taxable_interest_stale_national_totals( status = _filing_status_label(_dimensions(fact).get("filing_status")) if status is None: continue - totals[ - ( - _measure_id(fact), - status, - _soi_return_universe_from_fact(fact), - str(_period_value(fact)), + source_record_id = _source_record_id(fact) + if not source_record_id: + continue + key = ( + family, + measure_id, + status, + _soi_return_universe_from_fact(fact), + str(_period_value(fact)), + ) + candidate = _SoiTaxableInterestShareTotal( + _numeric_value(fact), source_record_id + ) + current = totals.get(key) + if current is not None and current.value != candidate.value: + raise AmbiguousSoiTaxableInterestShareTotalError( + "SOI taxable-interest share family has two national totals for " + f"{key}: {current.value!r} ({current.source_record_id}) and " + f"{candidate.value!r} ({candidate.source_record_id}). " + "Deduplicate them upstream." ) - ] = _numeric_value(fact) + if current is None or candidate.source_record_id < current.source_record_id: + totals[key] = candidate return totals @@ -1940,20 +2009,34 @@ def _soi_taxable_interest_kind_from_measure(measure_id: str) -> str | None: return None -def _is_stale_soi_historic_taxable_interest_spec(spec: TargetSpec) -> bool: - if spec.family != "irs_soi": - return False - if _soi_taxable_interest_kind(spec) is None: - return False - return ".historic_table_2." in spec.metadata.get("ledger_layout_record_set_id", "") +#: Record-set families whose taxable-interest rows are stale photos of an +#: older tax year and therefore ship only as shares of their own national +#: all-AGI row, scaled to a Pub 1304-class control (microcosm#489, #1038): +#: Historic Table 2 (TY2022) and the congressional-district file (22incd.csv, +#: TY2022 data stamped ty2023, of any boundary vintage). Neither ever anchors +#: the family as a control. +_SOI_TAXABLE_INTEREST_SHARE_FAMILIES: dict[str, str] = { + ".historic_table_2.": "historic_table_2", + ".congressional_district_": "congressional_district", +} -def _is_stale_soi_historic_taxable_interest_fact(fact: object) -> bool: - if _source_name(fact) != "irs_soi": - return False - if _soi_taxable_interest_kind_from_measure(_measure_id(fact)) is None: - return False - return ".historic_table_2." in _str_at(fact, "layout", "record_set_id") +def _soi_taxable_interest_share_family(record_set_id: str) -> str | None: + """The share family of a taxable-interest record set, if it is one.""" + for marker, family in _SOI_TAXABLE_INTEREST_SHARE_FAMILIES.items(): + if marker in record_set_id: + return family + return None + + +def _soi_taxable_interest_share_family_from_spec(spec: TargetSpec) -> str | None: + if spec.family != "irs_soi": + return None + if _soi_taxable_interest_kind(spec) is None: + return None + return _soi_taxable_interest_share_family( + spec.metadata.get("ledger_layout_record_set_id", "") + ) def _soi_taxable_interest_control_key_from_fact( @@ -1966,10 +2049,11 @@ def _soi_taxable_interest_control_key_from_fact( return None if _geography_level(fact) != "country": return None - # Congressional-district aggregates are a processing-window subset of - # the filing universe (and the feed's CD vintage stamp is in error — - # microcosm#489); they never anchor the interest family. - if _is_soi_congressional_district_record_set(fact): + # A share family never anchors the interest family: Historic Table 2 is a + # stale photo, and congressional-district aggregates are a + # processing-window subset of the filing universe whose feed vintage + # stamp is in error (microcosm#489). + if _soi_taxable_interest_share_family(_str_at(fact, "layout", "record_set_id")): return None # Full AGI range only — a filing-status total is a valid control for its # own status family (the key carries the status), matching the diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_targets.py b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_targets.py index 3cd976bac3..23e85e38c7 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_targets.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_targets.py @@ -893,10 +893,14 @@ def test_pinned_feed_exclusion_receipt(pinned_feed_national_state_surface) -> No def test_pinned_feed_national_state_surface_restores_the_fences( pinned_feed_national_state_surface, ) -> None: - """The release surface on the pinned feed: 33,250 compiled targets after + """The release surface on the pinned feed: 33,248 compiled targets after Medicaid substitution, 6,102 national_state targets at registry - e1f9906bf0e0, 32 CHIP rows none of them for an M-CHIP state, no - other-income row and no tips return count. microcosm#1206 moved the + 1b318eafb174, 32 CHIP rows none of them for an M-CHIP state, no + other-income row and no tips return count. The congressional-district + taxable-interest share rebase retired that file's two US interest rows + (33,250 before) and named the share denominator on the 1,034 Historic + Table 2 interest rows of national_state, a metadata-only change from + e1f9906bf0e0 (no national_state value moved). microcosm#1206 moved the version from f96be9f41781 by binding the Part B premium target to gross_medicare_part_b_premium_if_enrolled; the counts are unchanged. The 408 state AGI-band rows of @@ -906,9 +910,9 @@ def test_pinned_feed_national_state_surface_restores_the_fences( ty2020 tips return count it was 32,843 / 5,695 at 386fac439e77 (docs/us-chronicle-feed-repin.md).""" registry, surface, _ = pinned_feed_national_state_surface - assert len(registry.specs) == 33_250 + assert len(registry.specs) == 33_248 assert len(surface.specs) == 6_102 - assert surface.version == "e1f9906bf0e0" + assert surface.version == "1b318eafb174" chip = [ spec for spec in surface.specs @@ -935,6 +939,135 @@ def test_pinned_feed_national_state_surface_restores_the_fences( ) +def _soi_bridged_quantity(spec, is_noop_filter) -> tuple[object, ...]: + """The model quantity and filters an SOI target constrains, read the way + the materializer reads them: a bridged return universe is the universe of + the control that set the row's level (``soi_return_universe_bridge``), and + a ledger filter the materializer treats as a no-op (``all``) constrains + nothing. So a rebased all-returns row compares with the excluding- + dependents national row it was scaled to, which names a different no-op + filter (income percentile range ``all``).""" + metadata = spec.metadata + return ( + metadata.get("variable"), + metadata.get("measure_mode"), + metadata.get("agi_lower_bound"), + metadata.get("agi_upper_bound"), + metadata.get("filing_status"), + metadata.get( + "soi_return_universe_bridge", + metadata.get("soi_return_universe", "all_returns"), + ), + metadata.get("itemized_only"), + tuple( + sorted( + (key, value) + for key, value in metadata.items() + if key.startswith("ledger_filter_") and not is_noop_filter(str(value)) + ) + ), + ) + + +def test_pinned_feed_congressional_district_taxable_interest_takes_the_table_4_3_level( + pinned_feed_national_state_surface, +) -> None: + """On the full surface the congressional-district file's 51 state and 436 + district taxable-interest rows per measure are shares of that file's US + row times the Table 4.3 ty2023 level, and the US row retires. The states, + and the districts, each sum to the national target: $340.43B (aged) and + 54,167,410 returns, where the TY2022 levels summed to $134.59B (0.395x) and + 48,547,160 (0.896x), with the US row a second national target at $134.59B. + + Invariant, two-sided: every stale taxable-interest family on the surface + (the district file's states and districts, Historic Table 2's states) + sums to within [0.85, 1] of the national target it was scaled to, compared + under the bridged universe key, and none is missing. Historic Table 2's + states sit at 0.984 (its US row includes Puerto Rico and other areas).""" + registry, _, _ = pinned_feed_national_state_surface + builder = _load_repo_tool("build_us_fiscal_refresh_release") + full, _ = builder._select_target_surface(registry.specs, "full") + + def quantity(spec) -> tuple[object, ...]: + return _soi_bridged_quantity(spec, builder._is_noop_ledger_filter_value) + + national: dict[tuple[object, ...], list[float]] = {} + families: dict[tuple[object, ...], dict[str, list]] = {} + for spec in full: + if spec.family != "irs_soi": + continue + if spec.metadata.get("source_measure_id") not in _INTEREST_MEASURES: + continue + level = spec.metadata["ledger_geography_level"] + family = fiscal_targets._soi_taxable_interest_share_family( + spec.metadata["ledger_layout_record_set_id"] + ) + if family is None: + if level == "country": + national.setdefault(quantity(spec), []).append(spec.value) + continue + assert level != "country" or family != "congressional_district", spec.name + families.setdefault( + (family, spec.metadata["source_measure_id"], quantity(spec)), + {}, + ).setdefault(level, []).append(spec) + + for measure in _INTEREST_MEASURES: + control_id = f"{_INTEREST_T43_RECORD_SET}.all.{measure}" + control = next(spec for spec in full if spec.name == control_id) + (cd,) = [ + levels + for (family, family_measure, _), levels in families.items() + if family == "congressional_district" and family_measure == measure + ] + assert {level: len(rows) for level, rows in cd.items()} == { + "state": 51, + "congressional_district": 436, + } + rows = [*cd["state"], *cd["congressional_district"]] + assert len({spec.metadata["uprating_factor"] for spec in rows}) == 1 + for spec in rows: + assert spec.metadata["uprating_index_source_record_id"] == control_id + assert spec.metadata["soi_share_total_source_record_id"] == ( + f"{_INTEREST_CD_RECORD_SET}.us.{measure}" + ) + for level in ("state", "congressional_district"): + total = math.fsum(spec.value for spec in cd[level]) + assert math.isclose(total, control.value, rel_tol=1e-12) + assert math.isclose( + next( + spec.value + for spec in full + if spec.name == f"{_INTEREST_T43_RECORD_SET}.all.taxable_interest_returns" + ), + 54_167_410, + rel_tol=1e-12, + ) + + ratios = { + (family, measure, level): math.fsum(spec.value for spec in rows) + / max(national[quantity]) + for (family, measure, quantity), levels in families.items() + if quantity[2:4] == ("-inf", "inf") and quantity in national + for level, rows in levels.items() + if (family, level) != ("historic_table_2", "congressional_district") + } + assert set(ratios) == { + (family, measure, level) + for measure in _INTEREST_MEASURES + for family, level in ( + ("congressional_district", "state"), + ("congressional_district", "congressional_district"), + ("historic_table_2", "state"), + ) + } + assert { + key: round(ratio, 4) + for key, ratio in ratios.items() + if not 0.85 <= ratio <= 1 + 1e-9 + } == {} + + def test_reviewed_zero_support_facts_are_not_active_targets() -> None: excluded_source_record_id = ( "hhs_acf_tanf.fy2024.cash_assistance.ar." @@ -2884,6 +3017,9 @@ def test_stale_soi_taxable_interest_never_uses_congressional_district_controls() "irs_soi.ty2023.table_4_3.all_returns_excluding_dependents.all." "taxable_interest_amount" ) + # Beside a control the CD US row is its family's share denominator, not a + # second national target. + assert cd_fact["lineage"]["source_record_id"] not in specs def test_stale_soi_taxable_interest_prefers_exact_universe_control_over_bridge() -> ( @@ -3579,8 +3715,12 @@ def test_stale_soi_taxable_interest_period_selection_laws() -> None: assert bin_spec.metadata["uprating_to_period"] == "2022" assert "irs_soi.ty2022.historic_table_2.us.all.taxable_interest_amount" not in specs - # An unanchored full-range state row survives unchanged when its family - # has no national total at all (no ht2 us.all fact in the feed). + # An unanchored full-range state row (no ht2 us.all fact in the feed) has + # no share to take. Beside an eligible control it is dropped, as the + # capital-gains rows are: kept, it would ship its TY2022 level against the + # TY2023 control, the contradiction this pass removes. With no control + # anywhere it survives unchanged + # (test_historic_table_2_taxable_interest_state_row_without_its_us_row). registry = compile_us_fiscal_target_registry( [ *packaged_reference_facts(), @@ -3607,12 +3747,10 @@ def test_stale_soi_taxable_interest_period_selection_laws() -> None: target_period=2024, allow_unaged_dollar_targets=True, ) - specs = {spec.name: spec for spec in registry.specs} - ca_broad = specs[ + assert ( "irs_soi.ty2022.historic_table_2.state_broad.ca.all.taxable_interest_amount" - ] - assert ca_broad.value == 25_000_000_000 - assert "uprating_factor" not in ca_broad.metadata + not in {spec.name for spec in registry.specs} + ) def test_compiled_interest_family_ages_coherently_end_to_end() -> None: @@ -3684,6 +3822,628 @@ def test_compiled_interest_family_ages_coherently_end_to_end() -> None: assert abs(bin_spec.value / national.value - 0.2) < 1e-12 +_INTEREST_MEASURES = ("taxable_interest_returns", "taxable_interest_amount") +_INTEREST_T43_RECORD_SET = "irs_soi.ty2023.table_4_3.all_returns_excluding_dependents" +_INTEREST_CD_RECORD_SET = "irs_soi.ty2023.congressional_district_2022.all_returns" +_INTEREST_HT2_US = "irs_soi.ty2022.historic_table_2.us.all.{measure}" +_INTEREST_HT2_CA = "irs_soi.ty2022.historic_table_2.state_broad.ca.all.{measure}" + + +def _taxable_interest_t43_facts( + *, returns: float = 11_000_000.0, amount: float = 235_000_000_000.0 +) -> list[dict[str, object]]: + """Table 4.3 ty2023, all returns excluding dependents: the Pub 1304 + control the pinned feed carries (the all-returns concept bridges to it).""" + return [ + _soi_taxable_interest_fact( + 2023, + source_record_id=f"{_INTEREST_T43_RECORD_SET}.all.{measure}", + measure_id=measure, + layout_record_set_id=_INTEREST_T43_RECORD_SET, + value=value, + ) + for measure, value in zip(_INTEREST_MEASURES, (returns, amount), strict=True) + ] + + +def _taxable_interest_ht2_facts() -> list[dict[str, object]]: + """TY2022 Historic Table 2 rows: US 10M / $100B and CA 2.5M / $25B.""" + facts = [] + for measure, us, ca in zip( + _INTEREST_MEASURES, + (10_000_000.0, 100_000_000_000.0), + (2_500_000.0, 25_000_000_000.0), + strict=True, + ): + facts.append( + _soi_taxable_interest_fact( + 2022, + source_record_id=_INTEREST_HT2_US.format(measure=measure), + measure_id=measure, + value=us, + ) + ) + facts.append( + _soi_taxable_interest_fact( + 2022, + source_record_id=_INTEREST_HT2_CA.format(measure=measure), + measure_id=measure, + geography_level="state", + geography_id="0400000US06", + layout_record_set_id="irs_soi.ty2022.historic_table_2.state_broad.ca", + value=ca, + ) + ) + return facts + + +def _taxable_interest_cd_package_facts( + *, + us: tuple[float, float] = (9_400_000.0, 94_000_000_000.0), + hawaii: tuple[float, float] = (400_000.0, 4_000_000_000.0), + districts: tuple[tuple[float, float], ...] = ( + (150_000.0, 1_000_000_000.0), + (250_000.0, 3_000_000_000.0), + ), +) -> list[dict[str, object]]: + """A congressional-district package's taxable-interest rows, stamped + ty2023 like the pinned feed: the US row, Hawaii, and Hawaii's two + districts, each as (returns, amount).""" + facts = [] + for index, measure in enumerate(_INTEREST_MEASURES): + facts.append( + _soi_congressional_district_fact( + measure, + us[index], + groupby_value_id="us", + geography_level="country", + geography_id="0100000US", + ) + ) + facts.append( + _soi_congressional_district_fact( + measure, + hawaii[index], + groupby_value_id="hi_total", + geography_level="state", + geography_id="0400000US15", + ) + ) + for number, values in enumerate(districts, start=1): + facts.append( + _soi_congressional_district_fact( + measure, + values[index], + groupby_value_id=f"hi_0{number}", + geography_id=f"5001700US150{number}", + ) + ) + return facts + + +def _congressional_district_taxable_interest_specs(registry) -> dict[str, object]: + return { + spec.name: spec + for spec in registry.specs + if spec.metadata.get("source_measure_id") in _INTEREST_MEASURES + and ".congressional_district_" in spec.metadata["ledger_layout_record_set_id"] + } + + +def test_congressional_district_taxable_interest_rows_are_shares_of_the_control() -> ( + None +): + """The congressional-district file is TY2022 data (stamped ty2023), the + same stale photo as Historic Table 2, so its state and district taxable- + interest rows ship as shares of that file's own US row times the Pub 1304 + control, and the US row retires. On the pinned feed they shipped as TY2022 + levels, and the US row compiled as a second national target: $134.6B + against Table 4.3's $340.4B (microcosm#1038 follow-up, #489 doctrine).""" + registry = compile_us_fiscal_target_registry( + [ + *packaged_reference_facts(), + *_taxable_interest_ht2_facts(), + *_taxable_interest_cd_package_facts(), + *_taxable_interest_t43_facts(), + ], + allow_unaged_dollar_targets=True, + ) + + cd = _congressional_district_taxable_interest_specs(registry) + assert sorted(spec.metadata["ledger_geography_level"] for spec in cd.values()) == ( + ["congressional_district"] * 4 + ["state"] * 2 + ) + for measure, control, cd_us, hawaii, districts in ( + ( + "taxable_interest_returns", + 11_000_000.0, + 9_400_000.0, + 400_000.0, + (150_000.0, 250_000.0), + ), + ("taxable_interest_amount", 235e9, 94e9, 4e9, (1e9, 3e9)), + ): + factor = control / cd_us + state = cd[f"{_INTEREST_CD_RECORD_SET}.hi_total.{measure}"] + children = [cd[f"{_INTEREST_CD_RECORD_SET}.hi_0{n}.{measure}"] for n in (1, 2)] + assert math.isclose(state.value, hawaii * factor, rel_tol=1e-12) + for child, raw in zip(children, districts, strict=True): + assert math.isclose(child.value, raw * factor, rel_tol=1e-12) + # One factor scales the state and its districts, so the hierarchy + # reconciliation that follows leaves the proportions alone. + assert child.metadata["hierarchy_reconciliation_factor"] == "1" + for spec in (state, *children): + assert spec.metadata["uprating_factor"] == fiscal_targets._format_float( + factor + ) + assert spec.metadata["uprating_index_source_record_id"] == ( + f"{_INTEREST_T43_RECORD_SET}.all.{measure}" + ) + assert spec.metadata["soi_share_total_source_record_id"] == ( + f"{_INTEREST_CD_RECORD_SET}.us.{measure}" + ) + # The all-returns rows take the excluding-dependents control, and + # say so. + assert spec.metadata["soi_return_universe_bridge"] == ( + "returns_excluding_dependents" + ) + assert spec.metadata["uprating_to_period"] == "2023" + assert spec.metadata["stale_distribution_rebased_to_active_total"] == "true" + + # Historic Table 2 rows keep dividing by the HT2 US row (x2.35 and x1.1), + # not by the congressional-district US row, and name it. + specs = {spec.metadata["ledger_source_record_id"]: spec for spec in registry.specs} + for measure, value in zip(_INTEREST_MEASURES, (2_750_000.0, 58.75e9), strict=True): + ca = specs[_INTEREST_HT2_CA.format(measure=measure)] + assert math.isclose(ca.value, value, rel_tol=1e-12) + assert ca.metadata["soi_share_total_source_record_id"] == ( + _INTEREST_HT2_US.format(measure=measure) + ) + + +def _taxable_interest_cd_package_at(stamp: int) -> list[dict[str, object]]: + """The rows of ``_taxable_interest_cd_package_facts`` stamped ``stamp``, + carrying the pinned 22incd.csv's digest and raw key.""" + digest = "137522878af78d624cfddc0e17cd36ae76a21b7bed166c7bb96ad3243a18a668" + record_set_id = f"irs_soi.ty{stamp}.congressional_district_2022.all_returns" + facts = [] + for fact in _taxable_interest_cd_package_facts(): + row = fact["layout"]["groupby_value_id"] + measure = fact["layout"]["measure_id"] + restamped = _dynamic_ledger_fact( + source_record_id=f"{record_set_id}.{row}.{measure}", + source_name="irs_soi", + measure_id=measure, + value=fact["value"], + period_value=stamp, + geography_level=fact["geography"]["level"], + geography_id=fact["geography"]["id"], + dimensions={"income_range": "all", "filing_status": "all"}, + layout_record_set_id=record_set_id, + groupby_dimension="irs_soi.congressional_district", + groupby_value_id=row, + ) + restamped["source"].update( + { + "source_file": "22incd.csv", + "source_sha256": digest, + "raw_r2_key": ( + f"raw/irs_soi/soi-congressional-district-2022/2022/{digest}/" + "22incd.csv" + ), + "vintage": f"tax_year_{stamp}", + } + ) + facts.append(restamped) + return facts + + +def test_congressional_district_taxable_interest_ignores_the_package_stamp() -> None: + """Differential: a congressional-district taxable-interest row is its share + of the same package's US row times the Table 4.3 level, so the package's + period stamp cancels. Compiled and aged to 2024, the package stamped + ty2023, as on the pinned feed, gives exactly the values of the same package + stamped truthfully at 2022, and both age from the control's 2023 period on + the CBO AGI link, as the Table 4.3 row does. Before, the ty2023 stamp aged + the levels one link (x1.087) where the truthful stamp would age two, and + neither reached the control (microcosm#1030, #1038).""" + cbo = [ + _cbo_income_source_projection_fact( + 2023, "adjusted_gross_income", value=15_000_000_000_000 + ), + _cbo_income_source_projection_fact( + 2024, "adjusted_gross_income", value=16_500_000_000_000 + ), + ] + compiled = {} + for stamp in (2023, 2022): + registry = compile_us_fiscal_target_registry( + [ + *packaged_reference_facts(), + *_taxable_interest_cd_package_at(stamp), + *_taxable_interest_t43_facts(), + *cbo, + ], + target_period=2024, + age_targets=True, + allow_unaged_dollar_targets=True, + ) + specs = {spec.name: spec for spec in registry.specs} + compiled[stamp] = ( + { + ( + spec.metadata["source_measure_id"], + spec.metadata["ledger_geography_id"], + ): spec + for spec in _congressional_district_taxable_interest_specs( + registry + ).values() + }, + specs[f"{_INTEREST_T43_RECORD_SET}.all.taxable_interest_amount"], + ) + + (stamped, national), (truthful, _) = compiled[2023], compiled[2022] + assert set(stamped) == set(truthful) + assert len(stamped) == 6 + agi_link = 16_500_000_000_000 / 15_000_000_000_000 + assert math.isclose(national.value, 235e9 * agi_link, rel_tol=1e-12) + for key, spec in stamped.items(): + assert math.isclose(spec.value, truthful[key].value, rel_tol=1e-12), key + assert spec.metadata["uprating_to_period"] == "2023" + if key[0] == "taxable_interest_amount": + for aged in (spec, truthful[key]): + assert math.isclose( + float(aged.metadata["aging_factor"]), agi_link, rel_tol=1e-12 + ) + raw = {"0400000US15": 4e9, "5001700US1501": 1e9}.get(key[1], 3e9) + # The share of the aged national row, exactly. + assert math.isclose(spec.value, raw / 94e9 * national.value, rel_tol=1e-12) + + +def test_congressional_district_taxable_interest_without_a_control_keeps_its_levels() -> ( + None +): + """With no Pub 1304-class control in the feed nothing can be a share, so the + congressional-district rows keep their source values, as Historic Table 2's + do, and generic aging owns the projection (the pre-identification + behaviour, test_stale_soi_taxable_interest_without_control_keeps_source_values).""" + package = _taxable_interest_cd_package_facts() + registry = compile_us_fiscal_target_registry( + [*packaged_reference_facts(), *package], + allow_unaged_dollar_targets=True, + ) + + cd = _congressional_district_taxable_interest_specs(registry) + assert {name: spec.value for name, spec in cd.items()} == { + fiscal_targets._source_record_id(fact): fact["value"] for fact in package + } + assert not [spec.name for spec in cd.values() if "uprating_factor" in spec.metadata] + + +def test_taxable_interest_share_row_without_its_family_total_is_dropped() -> None: + """A congressional-district row with an eligible control but no US row of + its own family has no defined share: it is dropped, never divided by the + Historic Table 2 US row and never shipped as a TY2022 level beside the + control. The HT2 rows beside it keep their own denominator.""" + registry = compile_us_fiscal_target_registry( + [ + *packaged_reference_facts(), + *( + fact + for fact in _taxable_interest_cd_package_facts() + if fact["layout"]["groupby_value_id"] != "us" + ), + *_taxable_interest_ht2_facts(), + *_taxable_interest_t43_facts(), + ], + allow_unaged_dollar_targets=True, + ) + + assert _congressional_district_taxable_interest_specs(registry) == {} + specs = {spec.metadata["ledger_source_record_id"]: spec for spec in registry.specs} + ca = specs[_INTEREST_HT2_CA.format(measure="taxable_interest_amount")] + assert math.isclose(ca.value, 58.75e9, rel_tol=1e-12) + assert ca.metadata["soi_share_total_source_record_id"] == ( + _INTEREST_HT2_US.format(measure="taxable_interest_amount") + ) + + +def test_historic_table_2_taxable_interest_state_row_without_its_us_row() -> None: + """The same rule for Historic Table 2: a full-range state row whose US row + is missing is dropped beside an eligible control (it would ship a TY2022 + level against a TY2023 national), and kept as a level only when no control + exists anywhere.""" + state_only = [ + fact + for fact in _taxable_interest_ht2_facts() + if ".state_broad." in fiscal_targets._source_record_id(fact) + ] + ca_amount = _INTEREST_HT2_CA.format(measure="taxable_interest_amount") + + with_control = compile_us_fiscal_target_registry( + [*packaged_reference_facts(), *state_only, *_taxable_interest_t43_facts()], + allow_unaged_dollar_targets=True, + ) + without_control = compile_us_fiscal_target_registry( + [*packaged_reference_facts(), *state_only], + allow_unaged_dollar_targets=True, + ) + + assert ca_amount not in {spec.name for spec in with_control.specs} + kept = {spec.name: spec for spec in without_control.specs}[ca_amount] + assert kept.value == 25e9 + assert "uprating_factor" not in kept.metadata + + +def test_taxable_interest_share_family_refuses_two_national_totals() -> None: + """Two different national totals for one share family, quantity and period + refuse the compile: feed order must not pick a share denominator.""" + rival = _soi_congressional_district_fact( + "taxable_interest_amount", + 95e9, + groupby_value_id="us_rival", + geography_level="country", + geography_id="0100000US", + ) + facts = [ + *packaged_reference_facts(), + *_taxable_interest_cd_package_facts(), + *_taxable_interest_t43_facts(), + rival, + ] + + with pytest.raises(fiscal_targets.AmbiguousSoiTaxableInterestShareTotalError): + fiscal_targets._soi_taxable_interest_share_national_totals(tuple(facts)) + with pytest.raises(fiscal_targets.AmbiguousSoiTaxableInterestShareTotalError): + compile_us_fiscal_target_registry(facts, allow_unaged_dollar_targets=True) + + +@pytest.mark.parametrize("twin_first", [True, False], ids=["twin-first", "us-first"]) +def test_taxable_interest_share_denominator_is_order_free(twin_first: bool) -> None: + """Two records carrying the same national total for one share family are + not ambiguous, and the denominator a share row names is the smaller record + id whichever comes first in the feed.""" + twin = _soi_congressional_district_fact( + "taxable_interest_amount", + 94e9, + groupby_value_id="us_twin", + geography_level="country", + geography_id="0100000US", + ) + package = _taxable_interest_cd_package_facts() + registry = compile_us_fiscal_target_registry( + [ + *packaged_reference_facts(), + *((twin, *package) if twin_first else (*package, twin)), + *_taxable_interest_t43_facts(), + ], + allow_unaged_dollar_targets=True, + ) + + hawaii = _congressional_district_taxable_interest_specs(registry)[ + f"{_INTEREST_CD_RECORD_SET}.hi_total.taxable_interest_amount" + ] + assert hawaii.metadata["soi_share_total_source_record_id"] == min( + f"{_INTEREST_CD_RECORD_SET}.us.taxable_interest_amount", + f"{_INTEREST_CD_RECORD_SET}.us_twin.taxable_interest_amount", + ) + assert math.isclose(hawaii.value, 4e9 * 235e9 / 94e9, rel_tol=1e-12) + + +@pytest.mark.parametrize( + ("record_set_id", "family"), + [ + ("irs_soi.ty2022.historic_table_2.us", "historic_table_2"), + ("irs_soi.ty2022.historic_table_2.state_agi.ca", "historic_table_2"), + ( + "irs_soi.ty2023.congressional_district_2022.all_returns", + "congressional_district", + ), + ( + "irs_soi.ty2024.congressional_district_2024.all_returns", + "congressional_district", + ), + ("irs_soi.ty2023.table_4_3.all_returns_excluding_dependents", None), + ("irs_soi.ty2023.table_2_1.itemized_all_returns", None), + ("irs_soi.ty2023.table_1_4", None), + ], +) +def test_taxable_interest_share_families(record_set_id: str, family) -> None: + """Historic Table 2 and the congressional-district file, of any boundary + vintage, are stale photos that supply shares; the Pub 1304 tables are + controls.""" + assert fiscal_targets._soi_taxable_interest_share_family(record_set_id) == family + + +def test_taxable_interest_shares_conserve_the_control_in_any_feed_order() -> None: + """Property: whatever the congressional-district package, the Historic + Table 2 rows beside it, the Table 4.3 level and the feed order, + - no congressional-district row ships at the country level, and each + family's rows divide by, and name, their own family's US row; + - each district-file state row is its share of the file's US row times + the control, and each state's districts sum to that state row; + - the district-file states, and its districts, sum to the control times + the states' share of the US row, so never past the control (exactly the + control when the states partition the US row, as in 22incd.csv); + - the compiled registry is identical in every feed order.""" + pytest.importorskip("hypothesis") + from hypothesis import given, settings + from hypothesis import strategies as st + + # Two-district states and one single-district state, whose source row is + # the state total only (22incd.csv carries no district row for it). + two_district_states = (("hi", "15"), ("id", "16"), ("me", "23")) + at_large = ("vt", "50") + values = st.integers(min_value=1, max_value=10_000_000) + + @settings(max_examples=25, deadline=None) + @given( + measure=st.sampled_from(_INTEREST_MEASURES), + districts=st.lists( + st.tuples(values, values), + min_size=len(two_district_states), + max_size=len(two_district_states), + ), + state_excess=st.lists( + st.integers(min_value=0, max_value=1_000_000), + min_size=len(two_district_states), + max_size=len(two_district_states), + ), + at_large_value=values, + other_areas=st.integers(min_value=0, max_value=50_000_000), + control=values, + ht2=st.tuples(values, values), + order=st.randoms(use_true_random=False), + ) + def check( + measure, + districts, + state_excess, + at_large_value, + other_areas, + control, + ht2, + order, + ): + ht2_state, ht2_rest = ht2 + ht2_us_id = _INTEREST_HT2_US.format(measure=measure) + ht2_state_id = f"irs_soi.ty2022.historic_table_2.state_broad.hi.all.{measure}" + facts = [ + _soi_taxable_interest_fact( + 2022, + source_record_id=ht2_us_id, + measure_id=measure, + value=float(ht2_state + ht2_rest), + ), + _soi_taxable_interest_fact( + 2022, + source_record_id=ht2_state_id, + measure_id=measure, + geography_level="state", + geography_id="0400000US15", + layout_record_set_id="irs_soi.ty2022.historic_table_2.state_broad.hi", + value=float(ht2_state), + ), + _soi_taxable_interest_fact( + 2023, + source_record_id=f"{_INTEREST_T43_RECORD_SET}.all.{measure}", + measure_id=measure, + layout_record_set_id=_INTEREST_T43_RECORD_SET, + value=float(control), + ), + ] + state_values = {} + for (postal, fips), pair, excess in zip( + two_district_states, districts, state_excess, strict=True + ): + state_values[fips] = float(sum(pair) + excess) + facts.append( + _soi_congressional_district_fact( + measure, + state_values[fips], + groupby_value_id=f"{postal}_total", + geography_level="state", + geography_id=f"0400000US{fips}", + ) + ) + for number, value in enumerate(pair, start=1): + facts.append( + _soi_congressional_district_fact( + measure, + float(value), + groupby_value_id=f"{postal}_0{number}", + geography_id=f"5001700US{fips}0{number}", + ) + ) + state_values[at_large[1]] = float(at_large_value) + facts.append( + _soi_congressional_district_fact( + measure, + float(at_large_value), + groupby_value_id=f"{at_large[0]}_total", + geography_level="state", + geography_id=f"0400000US{at_large[1]}", + ) + ) + cd_us = sum(state_values.values()) + other_areas + facts.append( + _soi_congressional_district_fact( + measure, + cd_us, + groupby_value_id="us", + geography_level="country", + geography_id="0100000US", + ) + ) + + def compiled(feed): + registry = compile_us_fiscal_target_registry( + [*packaged_reference_facts(), *feed], allow_unaged_dollar_targets=True + ) + return registry, { + spec.name: (spec.value, dict(spec.metadata)) for spec in registry.specs + } + + _, canonical = compiled(facts) + shuffled = list(facts) + order.shuffle(shuffled) + registry, reordered = compiled(shuffled) + assert reordered == canonical + + hawaii_ht2 = next( + spec + for spec in registry.specs + if spec.metadata["ledger_source_record_id"] == ht2_state_id + ) + assert math.isclose( + hawaii_ht2.value, + ht2_state * control / (ht2_state + ht2_rest), + rel_tol=1e-12, + ) + assert hawaii_ht2.metadata["soi_share_total_source_record_id"] == ht2_us_id + + by_level: dict[str, list] = {} + for spec in _congressional_district_taxable_interest_specs(registry).values(): + by_level.setdefault(spec.metadata["ledger_geography_level"], []).append( + spec + ) + assert spec.metadata["uprating_index_source_record_id"] == ( + f"{_INTEREST_T43_RECORD_SET}.all.{measure}" + ) + assert spec.metadata["soi_share_total_source_record_id"] == ( + f"{_INTEREST_CD_RECORD_SET}.us.{measure}" + ) + assert spec.metadata["soi_return_universe_bridge"] == ( + "returns_excluding_dependents" + ) + assert set(by_level) == {"state", "congressional_district"} + assert len(by_level["state"]) == len(state_values) + assert len(by_level["congressional_district"]) == 2 * len(two_district_states) + + factor = control / cd_us + state_by_fips = {} + for spec in by_level["state"]: + raw = state_values[spec.metadata["state_fips"]] + assert math.isclose(spec.value, raw * factor, rel_tol=1e-12) + state_by_fips[spec.metadata["state_fips"]] = spec.value + district_totals: dict[str, float] = {} + for spec in by_level["congressional_district"]: + fips = spec.metadata["state_fips"] + district_totals[fips] = district_totals.get(fips, 0.0) + spec.value + for fips, total in district_totals.items(): + assert math.isclose(total, state_by_fips[fips], rel_tol=1e-12) + + states_total = sum(state_by_fips.values()) + expected = control * sum(state_values.values()) / cd_us + assert math.isclose(states_total, expected, rel_tol=1e-12) + assert states_total <= control * (1 + 1e-12) + assert sum(district_totals.values()) <= control * (1 + 1e-12) + if other_areas == 0: + assert math.isclose(states_total, control, rel_tol=1e-12) + + check() + + def test_stale_soi_capital_gains_state_rows_rebase_to_newer_national_total() -> None: facts = [ *packaged_reference_facts(), diff --git a/tools/us_acs_local_cd_surface.py b/tools/us_acs_local_cd_surface.py index edd37055fa..24782a66b2 100644 --- a/tools/us_acs_local_cd_surface.py +++ b/tools/us_acs_local_cd_surface.py @@ -105,8 +105,8 @@ #: How far a state's Historic Table 2 / district-file ratio may sit from its #: measure's median across states before the district file's data for that #: (state, concept) is refused. The ratio mixes a common part (the file's -#: coverage and aging gap, or a measure-wide rebase such as taxable interest's -#: x2.47 onto Table 4.3), which the median absorbs, with a state-specific part. +#: coverage and aging gap, or a measure-wide rebase such as a share family's +#: control factor), which the median absorbs, with a state-specific part. #: A state-specific part beyond 25% means the two publications disagree about #: that state (Hawaii's dividends differ 2.4x from the typical gap on the #: pinned feed), so the district file's shares, or a bridge built on that From 4602d939d4b3a6d253831249535c195f4b1c7a48 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 10:54:44 -0400 Subject: [PATCH 2/3] Pin family-keyed denominators under a corrected stamp (#1038) The stamp differential now compiles Historic Table 2's TY2022 interest rows beside the district package. Stamped truthfully at 2022, the two families' US rows share a period, universe and status, so only the family in the denominator key keeps each on its own US row; a family-agnostic key fails this test (mutation run). Co-Authored-By: Claude Opus 5.5 --- .../tests/engine_free/us/test_us_fiscal_targets.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_targets.py b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_targets.py index 23e85e38c7..ebc8d296b4 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_targets.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_targets.py @@ -4046,7 +4046,12 @@ def test_congressional_district_taxable_interest_ignores_the_package_stamp() -> stamped truthfully at 2022, and both age from the control's 2023 period on the CBO AGI link, as the Table 4.3 row does. Before, the ty2023 stamp aged the levels one link (x1.087) where the truthful stamp would age two, and - neither reached the control (microcosm#1030, #1038).""" + neither reached the control (microcosm#1030, #1038). + + Historic Table 2's TY2022 rows compile alongside. Stamped truthfully, the + district file's US row and Historic Table 2's share a period, universe and + status, so only the family in the denominator key keeps each family on its + own US row (94 vs 100): the case a corrected stamp creates.""" cbo = [ _cbo_income_source_projection_fact( 2023, "adjusted_gross_income", value=15_000_000_000_000 @@ -4061,6 +4066,7 @@ def test_congressional_district_taxable_interest_ignores_the_package_stamp() -> [ *packaged_reference_facts(), *_taxable_interest_cd_package_at(stamp), + *_taxable_interest_ht2_facts(), *_taxable_interest_t43_facts(), *cbo, ], @@ -4069,6 +4075,11 @@ def test_congressional_district_taxable_interest_ignores_the_package_stamp() -> allow_unaged_dollar_targets=True, ) specs = {spec.name: spec for spec in registry.specs} + ht2_ca = specs[_INTEREST_HT2_CA.format(measure="taxable_interest_amount")] + assert math.isclose(ht2_ca.value, 25e9 * 2.35 * 1.1, rel_tol=1e-12) + assert ht2_ca.metadata["soi_share_total_source_record_id"] == ( + _INTEREST_HT2_US.format(measure="taxable_interest_amount") + ) compiled[stamp] = ( { ( From 8bf49c35e0453b0de087afc7479b53454bd0bf8a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 11:43:53 -0400 Subject: [PATCH 3/3] Address the in-session review of #1229: property teeth, data-year wording, doc numbers - The Hypothesis property now draws the district stamp (2022 or 2023) and an equal-valued twin US row on either side of "us", so its own-denominator and order-free clauses can fail (a family-blind key or a feed-order denominator breaks it), and checks that every rebased row's control is a compiled target. - Docstrings: "_2022" in congressional_district_2022 is the data year, not a boundary vintage; state the stamp <= control-period precondition and the pre-existing per-row-control vs per-pair-retirement corner (main has it for Historic Table 2 too; no current feed carries Table 1.4 interest). - Historic Table 2's states sit at 0.984 of the control for amounts and 0.992 for returns; the state_cd doc gives interest's new factor range (0.93-1.07, 0.981-1.001) and why it sits below 1. - _soi_bridged_quantity says it is the interest comparison key, not the materializer's full key. Co-Authored-By: Claude Opus 5.5 --- docs/us-acs-local-soi-target-surface.md | 19 +-- .../build/us_runtime/fiscal_targets.py | 31 +++-- .../engine_free/us/test_us_fiscal_targets.py | 119 +++++++++++------- 3 files changed, 105 insertions(+), 64 deletions(-) diff --git a/docs/us-acs-local-soi-target-surface.md b/docs/us-acs-local-soi-target-surface.md index 00a2dbd610..9b09a1be5b 100644 --- a/docs/us-acs-local-soi-target-surface.md +++ b/docs/us-acs-local-soi-target-surface.md @@ -180,10 +180,11 @@ Why Historic Table 2: feed labels the file "Congressional District Data 2022" but stamps it tax year 2023 (PR #1040, #1030), so it is aged one year less than Historic Table 2. Its taxable-interest rows are now shares of its own US row times Table - 4.3, like Historic Table 2's, but that level still differs from Historic - Table 2's state levels by Historic Table 2's other-areas sliver (its states - sum to 98.4% of the control). A within-state share is unaffected by a level - factor common to the state. + 4.3, like Historic Table 2's, with no aging gap between the two, but the + levels still differ by Historic Table 2's other-areas sliver: its states sum + to 98.4% (amounts) and 99.2% (returns) of the control, the district file's + to all of it. A within-state share is unaffected by a level factor common + to the state. - **Two vintages of one concept are contradictory constraints**; the solve can only split the difference. @@ -203,10 +204,14 @@ district rows, are mostly 1.00 to 1.10. (#1030) would remove the aging part only. - **One factor reflects a known level difference:** capital-gains amounts (0.77 to 1.00). Taxable interest's factors (2.35 to 2.70) carried the Table - 4.3 rebase that Historic Table 2's rows had and the district file's lacked; - since the district file's interest rows take the same control as shares of + 4.3 rebase that Historic Table 2's rows had and the district file's lacked. + Since the district file's interest rows take the same control as shares of its US row, they divide by 2.529 (amounts) and 1.116 (returns), which moves - every state alike, so the factor band's verdicts are unchanged. + every state alike, so the factor band's verdicts are unchanged. They now run + 0.93 to 1.07 for amounts (median 0.977) and 0.981 to 1.001 for returns + (median 0.995): there is no aging gap, and they sit below 1 because the + district file's states take the whole control while Historic Table 2's take + 98.4% and 99.2% of it. - **The largest single-state factors** were on qualified and ordinary dividends (Hawaii, 2.68 and 2.17), tax-exempt interest (Utah, 1.43) and rental income (New York, 0.72), where the two tables disagree about one diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py b/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py index 28fdb0fa6c..b3457fc538 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py @@ -1753,10 +1753,21 @@ def _rebase_stale_soi_taxable_interest_distributions( target beside Table 4.3: on the full surface its US row, its states and its districts each summed to $134.6B against $340.4B (0.395x) and to 48.5M returns against 54.2M (0.896x). As shares of the file's own US row, - its states and its districts each sum to the control; the stamp cancels - in the share; and because one factor scales a family's state and - district rows alike, the district-to-state reconciliation that runs next - keeps factor 1. + its states and its districts each sum to the control, and because one + factor scales a family's state and district rows alike, the + district-to-state reconciliation that runs next keeps proportions (factor + 1 where the file's districts sum to its states, as in 22incd.csv). The + period stamp cancels in the share while it is no later than the control's + period: a family stamped after every control keeps its rows and its US + row as levels, as before (the next district file under the same +1 stamp + error would; correcting the stamp, microcosm#1030, keeps the share). + + Known corner, as on main: control choice is per row (an exact-universe + control older than the row is skipped for an eligible bridge) while + bridge retirement is per (measure, status) (any exact all-returns control + retires the bridged national row). With Table 1.4 interest only at an + older vintage than Table 4.3, a row can rebase onto the retired Table 4.3 + level; no current feed carries Table 1.4 interest. A share row whose own family carries no nonzero national total cannot be a share. It is dropped if it is an AGI slice (unverifiable) or if a @@ -1946,9 +1957,10 @@ def _soi_taxable_interest_share_national_totals( family's. Two different values under one key refuse the compile instead of letting feed order pick the denominator; equal values under two record ids keep the smaller id, so the named denominator does not depend on feed order - either. Congressional-district packages of different boundary vintages for - one tax year share the ``congressional_district`` family, so their US rows - must agree or the compile refuses. + either. Two district files compiled at one period (for example 22incd.csv + stamped ty2023 beside a truthfully stamped 23incd.csv) share the + ``congressional_district`` family: unequal US rows refuse the compile, and + equal ones let both files bind, which upstream must deduplicate. """ totals: dict[tuple[str, str, str, str, str], _SoiTaxableInterestShareTotal] = {} for fact in facts: @@ -2013,8 +2025,9 @@ def _soi_taxable_interest_kind_from_measure(measure_id: str) -> str | None: #: older tax year and therefore ship only as shares of their own national #: all-AGI row, scaled to a Pub 1304-class control (microcosm#489, #1038): #: Historic Table 2 (TY2022) and the congressional-district file (22incd.csv, -#: TY2022 data stamped ty2023, of any boundary vintage). Neither ever anchors -#: the family as a control. +#: TY2022 data stamped ty2023; its record sets read congressional_district_2022, +#: the data year, and later files match too). Neither ever anchors the family +#: as a control. _SOI_TAXABLE_INTEREST_SHARE_FAMILIES: dict[str, str] = { ".historic_table_2.": "historic_table_2", ".congressional_district_": "congressional_district", diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_targets.py b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_targets.py index ebc8d296b4..c0cb1b0f3a 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_targets.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_targets.py @@ -940,13 +940,16 @@ def test_pinned_feed_national_state_surface_restores_the_fences( def _soi_bridged_quantity(spec, is_noop_filter) -> tuple[object, ...]: - """The model quantity and filters an SOI target constrains, read the way - the materializer reads them: a bridged return universe is the universe of - the control that set the row's level (``soi_return_universe_bridge``), and - a ledger filter the materializer treats as a no-op (``all``) constrains - nothing. So a rebased all-returns row compares with the excluding- - dependents national row it was scaled to, which names a different no-op - filter (income percentile range ``all``).""" + """The taxable-interest quantity and filters an SOI target constrains, for + comparing stale interest families with their control: a bridged return + universe reads as the universe of the control that set the row's level + (``soi_return_universe_bridge``), and a ledger filter the materializer + treats as a no-op (``all``) is dropped, so a rebased all-returns row + compares with the excluding-dependents national row it was scaled to, + which names a different no-op filter (income percentile range ``all``). + It is not the materializer's full key: it omits the EITC child-count and + positive-EITC filters, ``source_variable`` and ``taxable_only``, which + other measures need before reusing it.""" metadata = spec.metadata return ( metadata.get("variable"), @@ -983,7 +986,8 @@ def test_pinned_feed_congressional_district_taxable_interest_takes_the_table_4_3 (the district file's states and districts, Historic Table 2's states) sums to within [0.85, 1] of the national target it was scaled to, compared under the bridged universe key, and none is missing. Historic Table 2's - states sit at 0.984 (its US row includes Puerto Rico and other areas).""" + states sit at 0.984 (amounts) and 0.992 (returns): its US row includes + Puerto Rico and other areas.""" registry, _, _ = pinned_feed_national_state_surface builder = _load_repo_tool("build_us_fiscal_refresh_release") full, _ = builder._select_target_surface(registry.specs, "full") @@ -4260,17 +4264,23 @@ def test_taxable_interest_share_denominator_is_order_free(twin_first: bool) -> N ], ) def test_taxable_interest_share_families(record_set_id: str, family) -> None: - """Historic Table 2 and the congressional-district file, of any boundary - vintage, are stale photos that supply shares; the Pub 1304 tables are - controls.""" + """Historic Table 2 and the congressional-district file, of any data year + (22incd.csv's record sets read congressional_district_2022), are stale + photos that supply shares; the Pub 1304 tables are controls.""" assert fiscal_targets._soi_taxable_interest_share_family(record_set_id) == family def test_taxable_interest_shares_conserve_the_control_in_any_feed_order() -> None: - """Property: whatever the congressional-district package, the Historic - Table 2 rows beside it, the Table 4.3 level and the feed order, + """Property: whatever the congressional-district package and its period + stamp, the Historic Table 2 rows beside it, the Table 4.3 level, a twin US + row carrying the same total and the feed order, - no congressional-district row ships at the country level, and each - family's rows divide by, and name, their own family's US row; + family's rows divide by, and name, their own family's US row. Stamped + truthfully at 2022, the district file's US row shares its period, + universe and status with Historic Table 2's, so only the family keeps + the two denominators apart; with a twin, the named denominator is the + smaller record id; + - every rebased row's control is a compiled national target; - each district-file state row is its share of the file's US row times the control, and each state's districts sum to that state row; - the district-file states, and its districts, sum to the control times @@ -4287,9 +4297,12 @@ def test_taxable_interest_shares_conserve_the_control_in_any_feed_order() -> Non at_large = ("vt", "50") values = st.integers(min_value=1, max_value=10_000_000) - @settings(max_examples=25, deadline=None) + @settings(max_examples=40, deadline=None) @given( measure=st.sampled_from(_INTEREST_MEASURES), + stamp=st.sampled_from((2022, 2023)), + # A twin US row sorting before ("aa_us") or after ("us_twin") "us". + twin=st.sampled_from((None, "aa_us", "us_twin")), districts=st.lists( st.tuples(values, values), min_size=len(two_district_states), @@ -4308,6 +4321,8 @@ def test_taxable_interest_shares_conserve_the_control_in_any_feed_order() -> Non ) def check( measure, + stamp, + twin, districts, state_excess, at_large_value, @@ -4316,9 +4331,27 @@ def check( ht2, order, ): + cd_record_set = f"irs_soi.ty{stamp}.congressional_district_2022.all_returns" + + def cd_fact(value, row, level="congressional_district", geography=""): + return _dynamic_ledger_fact( + source_record_id=f"{cd_record_set}.{row}.{measure}", + source_name="irs_soi", + measure_id=measure, + value=float(value), + period_value=stamp, + geography_level=level, + geography_id=geography, + dimensions={"income_range": "all", "filing_status": "all"}, + layout_record_set_id=cd_record_set, + groupby_dimension="irs_soi.congressional_district", + groupby_value_id=row, + ) + ht2_state, ht2_rest = ht2 ht2_us_id = _INTEREST_HT2_US.format(measure=measure) ht2_state_id = f"irs_soi.ty2022.historic_table_2.state_broad.hi.all.{measure}" + control_id = f"{_INTEREST_T43_RECORD_SET}.all.{measure}" facts = [ _soi_taxable_interest_fact( 2022, @@ -4337,7 +4370,7 @@ def check( ), _soi_taxable_interest_fact( 2023, - source_record_id=f"{_INTEREST_T43_RECORD_SET}.all.{measure}", + source_record_id=control_id, measure_id=measure, layout_record_set_id=_INTEREST_T43_RECORD_SET, value=float(control), @@ -4349,43 +4382,32 @@ def check( ): state_values[fips] = float(sum(pair) + excess) facts.append( - _soi_congressional_district_fact( - measure, - state_values[fips], - groupby_value_id=f"{postal}_total", - geography_level="state", - geography_id=f"0400000US{fips}", + cd_fact( + state_values[fips], f"{postal}_total", "state", f"0400000US{fips}" ) ) for number, value in enumerate(pair, start=1): facts.append( - _soi_congressional_district_fact( - measure, - float(value), - groupby_value_id=f"{postal}_0{number}", - geography_id=f"5001700US{fips}0{number}", + cd_fact( + value, + f"{postal}_0{number}", + geography=f"5001700US{fips}0{number}", ) ) state_values[at_large[1]] = float(at_large_value) facts.append( - _soi_congressional_district_fact( - measure, - float(at_large_value), - groupby_value_id=f"{at_large[0]}_total", - geography_level="state", - geography_id=f"0400000US{at_large[1]}", + cd_fact( + at_large_value, + f"{at_large[0]}_total", + "state", + f"0400000US{at_large[1]}", ) ) cd_us = sum(state_values.values()) + other_areas - facts.append( - _soi_congressional_district_fact( - measure, - cd_us, - groupby_value_id="us", - geography_level="country", - geography_id="0100000US", - ) - ) + us_rows = ["us"] if twin is None else ["us", twin] + for row in us_rows: + facts.append(cd_fact(cd_us, row, "country", "0100000US")) + denominator = min(f"{cd_record_set}.{row}.{measure}" for row in us_rows) def compiled(feed): registry = compile_us_fiscal_target_registry( @@ -4401,6 +4423,11 @@ def compiled(feed): registry, reordered = compiled(shuffled) assert reordered == canonical + names = {spec.name for spec in registry.specs} + for spec in registry.specs: + if spec.metadata.get("stale_distribution_rebased_to_active_total"): + assert spec.metadata["uprating_index_source_record_id"] in names + hawaii_ht2 = next( spec for spec in registry.specs @@ -4418,12 +4445,8 @@ def compiled(feed): by_level.setdefault(spec.metadata["ledger_geography_level"], []).append( spec ) - assert spec.metadata["uprating_index_source_record_id"] == ( - f"{_INTEREST_T43_RECORD_SET}.all.{measure}" - ) - assert spec.metadata["soi_share_total_source_record_id"] == ( - f"{_INTEREST_CD_RECORD_SET}.us.{measure}" - ) + assert spec.metadata["uprating_index_source_record_id"] == control_id + assert spec.metadata["soi_share_total_source_record_id"] == denominator assert spec.metadata["soi_return_universe_bridge"] == ( "returns_excluding_dependents" )