diff --git a/changelog.d/1250-cd-national-controls.fixed.md b/changelog.d/1250-cd-national-controls.fixed.md new file mode 100644 index 0000000000..8370c4f7ad --- /dev/null +++ b/changelog.d/1250-cd-national-controls.fixed.md @@ -0,0 +1 @@ +Rebase the congressional-district file's state and district rows of the 11 Table 1.x-controlled SOI columns (return count, AGI, taxable IRA distributions, taxable pensions, unemployment compensation and taxable Social Security, returns and amounts; itemized deductions) as shares of the file's own US row times their #1247 control, and retire the district US row. On the pinned feed's `full` surface no controlled quantity binds two national values any more (control / district US row was 1.001 to 1.169); the district file's 51-state sums move from +0.14% (taxable pension returns) to +16.9% (taxable Social Security) and its states and districts each sum to the control. A district fact that would be rebased under an unreviewed IRS column refuses the compile. `national_state` and the ACS-local `state_cd` surface (values, receipt and per-row records) are unchanged, because `state_cd` reads district rows bound as national shares at the district file's own level (microcosm#1250). diff --git a/docs/us-acs-local-soi-target-surface.md b/docs/us-acs-local-soi-target-surface.md index e072362569..e4310e99cb 100644 --- a/docs/us-acs-local-soi-target-surface.md +++ b/docs/us-acs-local-soi-target-surface.md @@ -201,6 +201,18 @@ district rows, are mostly 1.00 to 1.10. (#1030) would remove the aging part only. - **Two factors reflect known level differences:** taxable interest (2.35 to 2.70, the Table 4.3 rebase) and capital-gains amounts (0.77 to 1.00). +- **The 11 Table 1.x-controlled columns** (return count, AGI, taxable IRA + distributions, pensions, unemployment compensation and Social Security, + itemized deductions) have Historic Table 2 parents that are shares of a + Table 1.x control since #1247, so their factors also carry the Table 1.x / + HT2 gap: the return-count median is 1.023 and AGI's 1.054 (the bullets + above were measured before #1247), taxable Social Security amounts run + 1.16 to 1.18. The district file's own rows of these columns are bound as + shares of the same controls since #1250 + ([us-soi-national-controls.md](us-soi-national-controls.md)). `state_cd` + reads such a row at the file's own level (`cd_file_level`: its + `uprating_factor` divided out), so its shares, factors, records and bridges + are what they would be without that rebase. - **The largest single-state factors** were on qualified and ordinary dividends (Hawaii, 2.68 and 2.17), tax-exempt interest (Utah, 1.43) and rental income (New York, 0.72), where the two tables disagree about one @@ -249,8 +261,8 @@ On the pinned feed the band drops eight: | Hawaii | `ordinary_dividends_amount` | rebase | 2.17 | 1.07 | 2.03 | | New York | `rental_royalty_income_amount` | rebase | 0.72 | 1.05 | 0.68 | | Utah | `tax_exempt_interest_amount` | rebase | 1.43 | 1.09 | 1.32 | -| Wyoming | `charitable_amount`, `interest_paid_deduction_amount` | bridge (itemized) | 1.75 | 1.06 | 1.65 | -| South Dakota | `charitable_amount`, `interest_paid_deduction_amount` | bridge (itemized) | 1.44 | 1.06 | 1.36 | +| Wyoming | `charitable_amount`, `interest_paid_deduction_amount` | bridge (itemized) | 1.81 | 1.09 | 1.65 | +| South Dakota | `charitable_amount`, `interest_paid_deduction_amount` | bridge (itemized) | 1.49 | 1.09 | 1.36 | That removes 34 district rows (Hawaii 2 x 2, New York 26, Utah 4) and four bridged state rows (Wyoming and South Dakota are at-large, so they had no @@ -276,14 +288,31 @@ The bridge assumes the measure shares its sibling's Historic Table 2 / district-file ratio in that state: the coverage and aging gap above, plus whatever the two publications disagree about for the sibling. It is an estimate, not an identity, which is why the factor band applies to it too. -Without it these state rows and their 2,562 district rows would sit 2–6% +Without it these state rows and their 2,562 district rows would sit 2–10% below every related concept. On the pinned feed the bridge factors are -1.00–1.03 for counts (median 1.016), 1.03–1.75 for the itemized-deduction -amounts (median 1.062; the two above 1.25x the median are dropped) and -1.03–1.08 for QBI amounts (median 1.049). The bridge is stamp-invariant: if #1030 corrects the +1.006–1.032 for counts (median 1.022), 1.06–1.81 for the itemized-deduction +amounts (median 1.095; the two above 1.25x the median are dropped) and +1.03–1.09 for QBI amounts (median 1.053), as measured with #1247 and #1250 +applied. The bridge is stamp-invariant: if #1030 corrects the stamp, the sibling ratio shrinks with it. Its district rows keep their within-state shares, as every other district row does. +All three siblings are columns the compiler binds as shares of a national +control (#1250), so the district file's sibling row no longer sits at the +file's level. The bridge reads it there anyway (`cd_file_level`: the share's +`uprating_factor` divided out, re-aged as the bridged row is), and the +sibling's band verdict compares the two files' levels, its median re-aged by +the same constant. Re-aging like the bridged row stands in for the sibling's +own aging only when both age on one series, as every registered bridge does +(counts unaged, amounts on the CBO AGI series); a pair on different series +refuses. The series comes from target aging's own rule, not the aging +receipt, which is a no-op (`source_equals_build`) for a share landed at the +build year. Reading the share as compiled would lift nothing: on the pinned +feed the bridged charitable and interest-paid amounts would fall 9.07%, QBI +amounts 5.63% and the bridged counts 2.65%. A property test checks that +national-share district rows move no `state_cd` value, drop, verdict or +recorded factor. + Every district row records its parent (`state_cd_parent_target_name`), the parent's basis, the district file's own value and the rebase factor; a bridged state row records its sibling and bridge factor. diff --git a/docs/us-chronicle-feed-repin.md b/docs/us-chronicle-feed-repin.md index d0b444fb8d..e574b9a247 100644 --- a/docs/us-chronicle-feed-repin.md +++ b/docs/us-chronicle-feed-repin.md @@ -216,7 +216,10 @@ same 60 return counts move and `national_state` goes from `e1f9906bf0e0` to `882f4764a248` at 6,102 targets. microcosm#1243 then rebases 14 more Historic Table 2 state families onto their Table 1.x controls and retires 15 stale national rows: 33,235 compiled and 6,087 on `national_state` (`0e899cd7f893`) -([us-soi-national-controls.md](us-soi-national-controls.md)). +([us-soi-national-controls.md](us-soi-national-controls.md)). microcosm#1250 +takes the congressional-district rows of 11 of those columns as shares of the +same controls and retires the district file's 11 US rows: 33,224 compiled, +`national_state` unchanged. The labelled feed exposed one latent defect in microcosm, fixed in this change: `apply_us_medicaid_enrollment_substitutions` built Rhode Island's diff --git a/docs/us-fact-to-target.md b/docs/us-fact-to-target.md index 1f9e11b8a1..241a95d0aa 100644 --- a/docs/us-fact-to-target.md +++ b/docs/us-fact-to-target.md @@ -132,6 +132,15 @@ the Table 1.x column of the same Form 1040 line. - `test_pinned_feed_no_soi_state_family_sums_past_its_national_target` fails a re-pin that adds a second national value for a compared quantity, or a state family past its national target. +- **The congressional-district file** carries 11 of the columns. + `_rebase_stale_soi_district_distributions` takes its state and district + rows as shares of its own US row times the same control and retires that + US row (#1250). Register a column only once the district documentation + guide defines it at the same line under the register's `source_column`; + a district fact that reads another column refuses the compile. On `full`, + `test_pinned_feed_full_surface_has_one_national_value_per_controlled_quantity` + fails a re-pin that leaves a second national value for a controlled + quantity, or a state or district family past it. ### State AGI bands bind as shares of the state total (microcosm#940) diff --git a/docs/us-soi-national-controls.md b/docs/us-soi-national-controls.md index ef1841ce23..ed079a5239 100644 --- a/docs/us-soi-national-controls.md +++ b/docs/us-soi-national-controls.md @@ -4,6 +4,8 @@ Fourteen IRS SOI quantities reach the US target surface from two sources: the Historic Table 2 (HT2) state file and a Pub 1304 Table 1.x national table. This page records why the two count the same thing, which one supplies the national level, and how `fiscal_targets.py` combines them (microcosm#1243). +The congressional-district file carries 11 of the same columns; [its +rows](#congressional-district-rows-1250) take the same controls (#1250). The taxable-interest (#489, #518) and capital-gains (#1035, #1036) families follow the same doctrine through their own passes. @@ -186,6 +188,149 @@ crosswalk, then narrowed to `national_state`. Before is `main` plus #1036. The diff and checks are reproduced by `scripts/diff_compiles.py` in the evidence folder. +## Congressional-district rows (#1250) + +The congressional-district file (`22incd.csv`, packaged as +`irs_soi.ty2023.congressional_district_2022.all_returns`: TY2022 data +stamped ty2023) carries 11 of the 14 columns on the `full` surface. It has +no wage or taxable-income rows. Each column has a US row, 51 state rows +(`_total`), and 428 district rows that the packaged crosswalk maps onto +the 436 districts of the 119th plan; an at-large state enters as its state +total. Before #1250 these rows were TY2022 levels, and the US row was a +second national target beside each control. + +### Same columns, a narrower count + +`22incddocguide.docx` defines each column at the same line as the HT2 +guides, under the same IRS name. Its US row counts fewer returns than Table +1.4 in the same tax year, because the file covers only returns processed in +calendar 2023, drops returns whose ZIP code is missing or does not match the +state, and drops APO/FPO, foreign and territory addresses (guide sections C +and D). It also suppresses items with fewer than 20 returns in a district +and items dominated by one return (section E). It is a narrower count of the +same concept, like HT2 is of Table 1.4, but about 2% narrower again: + +| Column | Line | District US / Table 1.4, TY2022 | District US / HT2 US, TY2022 | Control / district US, pinned feed | +|---|---|---:|---:|---:| +| N1 | returns | 0.9690 | 0.9793 | 1.0272 | +| A00100 | 11 | 0.9724 | 0.9758 | 1.0597 | +| N01400 / A01400 | 4b | 0.9966 / 0.9767 | 0.9877 / 0.9837 | 1.0288 / 1.0247 | +| N01700 / A01700 | 5b | 0.9827 / 0.9702 | 0.9850 / 0.9843 | 1.0014 / 1.0538 | +| N02300 / A02300 | Schedule 1, 7 | 0.9671 / 0.9699 | 0.9839 / 0.9798 | 1.0272 / 1.0546 | +| N02500 / A02500 | 6b | 0.9797 / 0.9831 | 0.9874 / 0.9888 | 1.0641 / 1.1692 | +| A04470 | 12 | 0.9404 | 0.9657 | 1.0997 | + +The last column is what the aged control is over the aged district US row +on the pinned feed: the coverage gap times the TY2022 to TY2023 change of the +column. The district rows age only 2023 to 2024, because of the stamp. +Within the file, the 51 states sum exactly to the US row and each state's +districts sum exactly to the state, for all 11 columns. The register +records each column's IRS name (`source_column`), and +`test_the_register_names_each_columns_reviewed_irs_id` pins it to the names +in the table above, read from the guide. +`~/PolicyEngine/_reviews/cd-soi-national-controls-20261012/scripts/cd_identities.py` +reproduces the table from the sha256-matched `22incd.csv`, `22in14ar.xls` +and `22in55cmcsv.csv`. + +### How the rows combine + +`_rebase_stale_soi_district_distributions` runs after the HT2 pass. It uses +the same register, the same control chooser and the same refusal +(`AmbiguousSoiNationalControlError`). Its rule is the one #1229 (taxable +interest) and #1040 (capital gains) use for other district families: + +- **Shares of the file's own US row.** Each district-file state and district + row becomes `value x control / district US`. The US row is the same + record set's national row of the same measure, filing status, universe and + period. One factor per measure scales the states and the districts alike, + so the district-to-state reconciliation that runs next keeps factor 1. + Every share names its denominator in `soi_share_total_source_record_id` + and lands at the control's period, as the HT2 shares do. An AGI slice of + the file takes the same factor, as #1229 and #1040 rebase district slices + (the current package has none). +- **One national level.** The district US row retires. +- **No eligible control.** With no control, one after the build period, or + one older than the row's period, the rows keep their levels and the US row + stays. +- **No denominator.** With an eligible control but no nonzero district US + row, a row cannot be a share, and as a level it would be the stale second + value this pass removes, so it is dropped. That is the district rule of + #1229 and #1040. The HT2 pass keeps HT2 rows as levels in the same case. + Neither case occurs on the pinned feed. +- **One denominator.** Each package divides by its own US row: the key + carries the record set, so another district file at the same stamp (a + different data year or boundary vintage) never supplies it. Two different + US values for one key refuse the compile; equal twins name the smaller + record id, so feed order never picks the denominator. +- **Reviewed columns only.** Every source fact of a rebased row, and its US + row, must read the register's `source_column`. Otherwise the compile + refuses (`UnreviewedSoiNationalControlColumnError`), as it does for a + missing column. The crosswalk merges source districts into one fact with a + single contributor's layout, so it records each contributor's column in + the merged fact's lineage, and the check reads every one (facts sharing a + record id all count). Rows that keep their levels, and facts that compile + to no target (a stale AGI slice, a superseded vintage), are not checked. The + district package's SALT and PTC measures read the wrong columns + (microcosm#1038), so a re-pin could do the same here. +- **The stamp cancels.** Row and US row carry the same stamp, so stamping the + file truthfully at 2022 gives the same values + (`test_the_share_ignores_the_package_stamp`). Under #1030's data-year + correction each row is a share of a restamped total of its own package, + which #1030 admits. + +### Effect on the pinned feed + +Compiled as the release does; before is #1247's head (`490b67181`). + +| Quantity | District file, 51 states, before | After | Move | +|---|---:|---:|---:| +| `taxable_social_security` sum | $490.09B | $573.04B | +16.92% | +| `taxable_social_security` count | 24,167,050 | 25,716,763 | +6.41% | +| `itemized_taxable_income_deductions` sum | $682.97B | $751.10B | +9.97% | +| `adjusted_gross_income` sum | $15,682.85B | $16,619.16B | +5.97% | +| `unemployment_compensation` sum | $31.89B | $33.64B | +5.46% | +| `taxable_pension_income` sum | $961.67B | $1,013.42B | +5.38% | +| `ira_distributions` count | 16,227,430 | 16,694,154 | +2.88% | +| `count` | 156,342,760 | 160,602,107 | +2.72% | +| `unemployment_compensation` count | 4,573,040 | 4,697,502 | +2.72% | +| `ira_distributions` sum | $464.87B | $476.36B | +2.47% | +| `taxable_pension_income` count | 29,501,160 | 29,541,284 | +0.14% | + +- **Retired: 11 rows**, the district US rows. **Added: none.** Compiled + targets go from 33,235 to 33,224; the full registry from `0bbbad9b22d1` to + `ce77f86efb24`. `national_state` holds no district-file row and is + unchanged (6,087, `0e899cd7f893`). +- **5,357 values move** (11 x (51 + 436)), each family by its one factor + (the last column above). No other spec changes, in value or metadata. +- After aging, the file's 51 states, and its 436 districts, each sum to the + control (relative error at most 1e-12), and every district reconciles to + its state with factor 1. +- On `full`, every quantity a control of this register measures now has one + national value. Quantities with more than one national value go from 38 to + 27, the 22 and 5 under "What this does not change". (Two more EITC groups + look like duplicates only to `_soi_national_quantity`: their Table 2.5 rows + take the child count from the record set, which the key does not read.) +- **At-large districts are bound twice**, as before: the district file's + row, and HT2's state row standing in for the district (AK, DE, DC, ND, SD, + VT, WY and Montana's two districts; 99 rows). Both are now shares of the + same control, so the two differ only by the two files' shares of that + state: HT2 / district file is 0.966 to 1.072 (median 0.988), against + 0.986 to 1.17 (median 1.037) before. The exceptions are Wyoming's and South + Dakota's itemized deductions (1.64 and 1.35, from 1.81 and 1.49), where the + two files disagree about the state. Counting both, the 445 district rows of a measure sum to 1.015 to + 1.022 times the control; each family alone sums to at most the control. +- **ACS-local `state_cd`.** All 26,425 values are unchanged (relative + difference at most 4.2e-15), as are its counts, drops, band verdicts and + receipt (rebase and bridge factors and recorded ratios, within 5e-15). + It takes district rows only as within-state shares of HT2, which a + national factor does not move, and it reads a district row bound as a + national share at the file's own level (`cd_file_level`), so its blocks, + bridges and per-row records see what they saw before + ([us-acs-local-soi-target-surface.md](us-acs-local-soi-target-surface.md)). + +`scripts/diff_cd.py`, `scripts/at_large_pairs.py` and +`scripts/compare_statecd.py` in the evidence folder reproduce these figures. + ## What this does not change - **Share assumption.** TY2022 HT2 state shares stand in for TY2023 shares. @@ -195,9 +340,12 @@ evidence folder. ids, keeps them instead, and this pass rebases them onto the TY2023 controls by the same-year difference alone (`test_a_control_at_the_hts_own_period_still_rebases`). -- **Congressional-district rows** of these columns stay TY2022 levels on the - `full` surface. They are not on `national_state`; #1040, #1229 and #1231 - reconcile other district families. +- **Congressional-district rows of other columns.** The district file's US + row is still a second national value on `full` for 22 quantities whose + other national row is the TY2022 HT2 US row (dividends, Schedule C, + partnership, rental and tax-exempt income, EITC, ACTC, income tax, medical + deductions), and for capital gains, SALT and PTC returns, which #1040 and + #1224 handle. - **Columns without an all-returns control** keep their HT2 levels: N04470 (itemizing returns) and N04800 (returns with taxable income) appear nationally only in Table 4.3, which excludes dependent filers. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/congressional_district_vintage.py b/packages/microcosm-build/src/microcosm/build/us_runtime/congressional_district_vintage.py index 94b1b03435..43fa757d3e 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/congressional_district_vintage.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/congressional_district_vintage.py @@ -707,6 +707,10 @@ def _append_lineage_contribution( contributions.append( { "source_record_id": _source_record_id(source_fact), + # The contributor's own column: the merged fact keeps only the + # first contributor's layout, and the SOI district rebase checks + # every contributor (fiscal_targets, microcosm#1250). + "source_column_id": _str_at(source_fact, "layout", "source_column_id"), "source_geography_id": source_geography_id, "target_geography_id": target_geography_id, "share": share, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py b/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py index ffe8714930..2b3b5a403e 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py @@ -47,6 +47,7 @@ __all__ = [ "AmbiguousSoiCapitalGainsControlError", + "UnreviewedSoiNationalControlColumnError", "US_FISCAL_MACRO_REALISM_BANDS", "US_FISCAL_TARGET_REGISTRY", "US_FISCAL_TARGET_SPECS", @@ -1073,6 +1074,11 @@ def compile_us_fiscal_target_registry( materialized_facts, target_period=target_period, ) + registry = _rebase_stale_soi_district_distributions( + registry, + materialized_facts, + target_period=target_period, + ) registry = _rebase_soi_state_agi_bands(registry, materialized_facts) registry = apply_ledger_target_profile( registry, @@ -1815,11 +1821,15 @@ class _SoiHt2NationalControlRule: ``concept`` names the Form 1040 line both columns report; the control is the latest ``control_measure_id`` fact of the ``control_family`` table. + ``source_column`` is the IRS column that HT2 and the congressional-district + file both publish under the measure id (``22incddocguide.docx`` defines it + at the same line). """ control_family: str control_measure_id: str concept: str + source_column: str #: Historic Table 2 columns whose national level comes from a Pub 1304 table @@ -1850,66 +1860,99 @@ class _SoiHt2NationalControlRule: #: $11,944.447B). Table 1.1 supplies the return count and AGI; Table 1.2 #: repeats them and stays a national target of equal value. A column absent #: from this register keeps its HT2 level and its HT2 national row. +#: +#: The congressional-district file (``22incd.csv``) publishes the same +#: columns, and ``22incddocguide.docx`` defines each at the same line, so +#: ``_rebase_stale_soi_district_distributions`` takes its rows of these +#: measures as shares of the same controls. Its US row is a narrower count of +#: the same thing: TY2022 returns processed in calendar 2023 and matched to a +#: district by ZIP code, 0.940 (itemized deductions) to 0.997 (IRA +#: distribution returns) of Table 1.4's TY2022 cell. +#: The district package carries no wages or taxable-income rows. _SOI_HT2_NATIONAL_CONTROLS: dict[str, _SoiHt2NationalControlRule] = { "return_count": _SoiHt2NationalControlRule( - "table_1_1", "return_count", "form_1040_returns" + "table_1_1", + "return_count", + "form_1040_returns", + "N1", ), "adjusted_gross_income": _SoiHt2NationalControlRule( - "table_1_1", "adjusted_gross_income", "form_1040_line_11_agi" + "table_1_1", + "adjusted_gross_income", + "form_1040_line_11_agi", + "A00100", ), "wages_salaries_amount": _SoiHt2NationalControlRule( - "table_1_4", "wages_salaries_amount", "form_1040_line_1z_wages" + "table_1_4", + "wages_salaries_amount", + "form_1040_line_1z_wages", + "A00200", ), "wages_salaries_returns": _SoiHt2NationalControlRule( - "table_1_4", "wages_salaries_returns", "form_1040_line_1z_wages" + "table_1_4", + "wages_salaries_returns", + "form_1040_line_1z_wages", + "N00200", ), "taxable_ira_distributions_amount": _SoiHt2NationalControlRule( "table_1_4", "taxable_ira_distributions_amount", "form_1040_line_4b_taxable_ira_distributions", + "A01400", ), "taxable_ira_distributions_returns": _SoiHt2NationalControlRule( "table_1_4", "taxable_ira_distributions_returns", "form_1040_line_4b_taxable_ira_distributions", + "N01400", ), "taxable_pension_income_amount": _SoiHt2NationalControlRule( "table_1_4", "taxable_pension_income_amount", "form_1040_line_5b_taxable_pensions_and_annuities", + "A01700", ), "taxable_pension_income_returns": _SoiHt2NationalControlRule( "table_1_4", "taxable_pension_income_returns", "form_1040_line_5b_taxable_pensions_and_annuities", + "N01700", ), "unemployment_compensation_amount": _SoiHt2NationalControlRule( "table_1_4", "unemployment_compensation_amount", "schedule_1_line_7_unemployment_compensation", + "A02300", ), "unemployment_compensation_returns": _SoiHt2NationalControlRule( "table_1_4", "unemployment_compensation_returns", "schedule_1_line_7_unemployment_compensation", + "N02300", ), "taxable_social_security_amount": _SoiHt2NationalControlRule( "table_1_4", "taxable_social_security_amount", "form_1040_line_6b_taxable_social_security", + "A02500", ), "taxable_social_security_returns": _SoiHt2NationalControlRule( "table_1_4", "taxable_social_security_returns", "form_1040_line_6b_taxable_social_security", + "N02500", ), "itemized_deductions_amount": _SoiHt2NationalControlRule( "table_1_2", "total_itemized_deductions_amount", "form_1040_line_12_itemized_deductions", + "A04470", ), "taxable_income_amount": _SoiHt2NationalControlRule( - "table_1_2", "taxable_income_amount", "form_1040_line_15_taxable_income" + "table_1_2", + "taxable_income_amount", + "form_1040_line_15_taxable_income", + "A04800", ), } _SOI_HT2_MEASURE_BY_CONTROL: dict[tuple[str, str], str] = { @@ -2213,6 +2256,245 @@ def _soi_ht2_lineage_data_year(record_set_id: str) -> str | None: return None +class UnreviewedSoiNationalControlColumnError(ValueError): + """A district fact of a registered measure reads another IRS column.""" + + +@dataclass(frozen=True) +class _SoiDistrictShareTotal: + value: float + source_record_id: str + + +def _rebase_stale_soi_district_distributions( + registry: TargetRegistry, + facts: tuple[object, ...], + *, + target_period: int | str, +) -> TargetRegistry: + """Use congressional-district rows of the registered columns as shares. + + The congressional-district file publishes the columns of + ``_SOI_HT2_NATIONAL_CONTROLS`` at the same Form 1040 lines, for TY2022 + returns processed in calendar 2023 and matched to a district by ZIP code + (the feed stamps it ty2023). Its rows shipped as levels on the ``full`` + surface, and its US row was a second national target beside each Table + 1.x control: on the pinned feed the control was 1.001x (taxable pension + returns) to 1.169x (taxable Social Security) the district US row. + + Each state and district row of such a measure (and any AGI slice of the + file) now ships as its share of the file's own all-AGI US row at its own + period, times the control of + ``_soi_ht2_national_controls`` at or after that period, as microcosm#1229 + does for taxable interest and #1040 for capital gains. Every share row + names its denominator in ``soi_share_total_source_record_id``. One factor + per measure scales the file's states and districts alike, so the + district-to-state reconciliation that runs next keeps factor 1, and + because the file's states and districts each sum exactly to its US row, + each sums to the control. The period stamp cancels in the share: the + value lands at the control's period, and target aging adds the remaining + years on the control's own link. + + The US row retires once its rows are shares, because the control's row + owns the national level. Without an eligible control (none, after the + build period, or older than the row) the rows keep their levels and the + US row stays. With an eligible control but no nonzero US row of the + file's own, a row cannot be a share and is dropped; a stale level beside + the control is the defect this pass removes (#1229 and #1040 do the same). + + A share assumes its measure id reads the register's ``source_column``. A + package that maps the measure id to another column (the district SALT + and PTC measures read the wrong columns, microcosm#1038), or names none, + would be rebased as the wrong concept, so every source fact of a row + rebased here, and its denominator, must read that column. A district + merged by the crosswalk keeps only its first contributor's layout, so its + contributors are read from the lineage, which records each one's column + (``_soi_district_source_columns``). Rows that keep their levels, and + facts that compile to no target, are not checked. + + Raises: + UnreviewedSoiNationalControlColumnError: A rebased row or its + denominator reads another column, or names none. + """ + + controls = _soi_ht2_national_controls(facts, target_period=target_period) + us_totals = _soi_district_us_totals(facts) + source_columns = _soi_district_source_columns(facts) + unreviewed: dict[str, str] = {} + specs: list[TargetSpec] = [] + for spec in registry.specs: + measure_id = _soi_district_rebase_measure(spec) + if measure_id is None: + specs.append(spec) + continue + key = _soi_ht2_control_key_from_spec(spec, measure_id) + source_period = spec.metadata.get("source_period", "") + control = controls.get(key) + if control is None or not _period_not_before( + control.period_key, _period_key_from_value(source_period) + ): + specs.append(spec) + continue + if _is_national_all_agi_spec(spec): + continue + record_set_id = spec.metadata.get("ledger_layout_record_set_id", "") + us_total = us_totals.get((record_set_id, *key, source_period)) + if us_total is None or us_total.value == 0: + continue + expected = _SOI_HT2_NATIONAL_CONTROLS[measure_id].source_column + for checked_id in ( + spec.metadata.get("ledger_source_record_id", ""), + us_total.source_record_id, + ): + for source_id, column in source_columns.get( + checked_id, ((checked_id, ""),) + ): + if column != expected: + unreviewed[source_id] = column or "" + concept = _SOI_HT2_NATIONAL_CONTROLS[measure_id].concept + factor = control.value / us_total.value + specs.append( + replace( + spec, + value=spec.value * factor, + metadata={ + **dict(spec.metadata), + "uprating_index": f"total_{measure_id}", + "uprating_from_period": source_period, + # Lands at the CONTROL's period; target aging completes + # the remaining links (#488). + "uprating_to_period": control.source_period, + "uprating_index_source_period": control.source_period, + "uprating_index_source_record_id": control.source_record_id, + "uprating_factor": _format_float(factor), + "stale_distribution_rebased_to_active_total": "true", + "soi_source_concept": concept, + "soi_control_concept": concept, + "soi_share_total_source_record_id": us_total.source_record_id, + }, + ) + ) + if unreviewed: + sample = dict(sorted(unreviewed.items())[:5]) + raise UnreviewedSoiNationalControlColumnError( + f"{len(unreviewed)} congressional-district fact(s) behind rebased " + "rows read an IRS column other than the reviewed one, e.g. " + f"{sample}. Review the column against the district documentation " + "guide and _SOI_HT2_NATIONAL_CONTROLS before rebasing it." + ) + return TargetRegistry(specs, country=registry.country) + + +def _soi_district_rebase_measure(spec: TargetSpec) -> str | None: + """The registered measure of a congressional-district spec, else ``None``. + + An AGI slice of the file is a share of the same US row, so it takes the + same factor (as #1229 and #1040 rebase district slices); the current + package publishes none. + """ + if spec.family != "irs_soi": + return None + measure_id = spec.metadata.get("source_measure_id", "") + if measure_id not in _SOI_HT2_NATIONAL_CONTROLS: + return None + record_set_id = spec.metadata.get("ledger_layout_record_set_id", "") + if _soi_record_set_family(record_set_id) != "congressional_district": + return None + return measure_id + + +def _soi_district_fact_key(fact: object) -> tuple[str, str, str] | None: + """(measure, filing status, universe) of a district fact, any AGI range.""" + if _source_name(fact) != "irs_soi": + return None + measure_id = _measure_id(fact) + if measure_id not in _SOI_HT2_NATIONAL_CONTROLS: + return None + record_set_id = _str_at(fact, "layout", "record_set_id") + if _soi_record_set_family(record_set_id) != "congressional_district": + return None + status = _filing_status_label(_dimensions(fact).get("filing_status")) + if status is None: + return None + return (measure_id, status, _soi_return_universe_from_fact(fact)) + + +def _soi_district_us_totals( + facts: tuple[object, ...], +) -> dict[tuple[str, str, str, str, str], _SoiDistrictShareTotal]: + """Each district package's all-AGI US totals. + + Keyed by (record set, measure, filing status, universe, period), so a + package's rows divide by its own US row and never by another district + file's at the same stamp (another data year or boundary vintage). A total + is a share denominator, so feed order must not pick it: two different + values for one key raise, and equal values under two record ids name the + smaller id (as the #1229 and #1040 denominators do). + """ + totals: dict[tuple[str, str, str, str, str], _SoiDistrictShareTotal] = {} + for fact in facts: + key = _soi_district_fact_key(fact) + if key is None or _geography_level(fact) != "country": + continue + if not _is_all_agi_range_fact(fact): + continue + source_record_id = _source_record_id(fact) + if not source_record_id: + continue + total_key = ( + _str_at(fact, "layout", "record_set_id"), + *key, + str(_period_value(fact)), + ) + candidate = _SoiDistrictShareTotal(_numeric_value(fact), source_record_id) + current = totals.get(total_key) + if current is not None and current.value != candidate.value: + raise AmbiguousSoiNationalControlError( + "SOI congressional-district US total is ambiguous: " + f"{total_key} has {current.value!r} ({current.source_record_id}) " + f"and {candidate.value!r} ({candidate.source_record_id}). " + "Deduplicate them upstream; feed order must not pick a rebase " + "denominator." + ) + if current is None or candidate.source_record_id < current.source_record_id: + totals[total_key] = candidate + return totals + + +def _soi_district_source_columns( + facts: tuple[object, ...], +) -> dict[str, tuple[tuple[str, str], ...]]: + """The (source record id, IRS column) of each source behind a district fact. + + A district the crosswalk merged lists every contributor, with its own + column, under ``lineage.congressional_district_vintage_contributions``; + any other fact is its own source. Facts that share a record id all count, + so a duplicate cannot hide another column. + """ + sources: dict[str, list[tuple[str, str]]] = {} + for fact in facts: + if _soi_district_fact_key(fact) is None: + continue + contributions = _at( + fact, "lineage", "congressional_district_vintage_contributions" + ) + entries = ( + [ + ( + str(_at(item, "source_record_id") or ""), + str(_at(item, "source_column_id") or ""), + ) + for item in contributions + ] + if contributions + else [ + (_source_record_id(fact), _str_at(fact, "layout", "source_column_id")) + ] + ) + sources.setdefault(_source_record_id(fact), []).extend(entries) + return {record_id: tuple(entries) for record_id, entries in sources.items()} + + #: Control universes admitted for the stale-interest rebase. A stale spec's #: own universe always outranks the bridge, and the ONE admitted bridge is #: all_returns -> returns_excluding_dependents: the Pub 1304 excluding- diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_cd_properties.py b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_cd_properties.py index 243568e5e3..31091e39d6 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_cd_properties.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_cd_properties.py @@ -850,3 +850,450 @@ def test_a_sign_conflict_or_a_self_contradicting_district_file_is_refused() -> N crosswalk=crosswalk, level_bridges={"charitable_amount": "return_count"}, ) + + +#: Exact binary scales, so a share rebase and its inverse round-trip without +#: rounding and the band's verdicts cannot flip at an edge. +_EXACT_SCALES = st.sampled_from([0.25, 0.5, 1.0, 2.0, 4.0]) + + +def _aged(spec, aging: float): + return dataclasses.replace( + spec, metadata={**spec.metadata, "aging_factor": repr(aging)} + ) + + +def _as_national_share(spec, factor: float, aging: float): + """``spec`` as the compiler binds it as a share of a national control: + its value times control / the file's US row (``factor``), aged from the + control's period (``aging``) instead of its own (#1250, #1229, #1040).""" + + own = float(spec.metadata["aging_factor"]) + value = spec.value * factor * aging / own + metadata = { + **spec.metadata, + "uprating_factor": repr(factor), + "soi_share_total_source_record_id": ( + f"cdfile.us.{spec.metadata['source_measure_id']}" + ), + "aging_factor": repr(aging), + } + return dataclasses.replace(spec, value=value, signed=value < 0, metadata=metadata) + + +def test_cd_file_level_undoes_a_national_share_and_its_aging() -> None: + row = _aged( + _soi_spec( + "cdfile.10_total.return_count", + "return_count", + 200.0, + level="state", + record_set=_CD_FILE, + state="10", + ), + 1.25, + ) + share = _as_national_share(row, 1.5, 1.0) + assert share.value == 240.0 + # The file's level, aged as the share is (from the control's period) ... + assert _CD.cd_file_level(share) == 160.0 + # ... or as an unrebased district-file row is. + assert _CD.cd_file_level(share, aged_like=row) == 200.0 + assert _CD.cd_file_level(row) == _CD.cd_file_level(row, aged_like=share) == 200.0 + + +@_SETTINGS +@given(_state_cd_inputs(), st.data()) +def test_national_share_district_rows_leave_the_state_cd_surface_unchanged( + inputs, data +) -> None: + """The compiler binds the district-file rows of some concepts as shares of + a national control: value x control / the file's US row, aged from the + control's period. ``state_cd`` uses the file for within-state shares, and + for level-bridge siblings through ``cd_file_level``, so the rebase moves + no ``state_cd`` value, drop, band verdict, recorded ratio or factor, with + or without the band, whatever the share's aging (the bridged row and its + sibling share one aging series, as every registered bridge does). + + Only concepts Historic Table 2 carries in every state are rebased here, + as the compiler's registered concepts are on the feed: a rebased concept + without an HT2 row in a state has no registered bridge, so it refuses.""" + + specs, crosswalk, _layout = inputs + states = {spec.metadata["state_fips"] for spec in specs} + ht2 = { + (spec.metadata["state_fips"], spec.metadata["source_measure_id"]) + for spec in specs + if spec.metadata["ledger_layout_record_set_spec_id"] == _HT2 + } + carried = sorted( + measure + for measure in {spec.metadata["source_measure_id"] for spec in specs} + if all((state, measure) in ht2 for state in states) + ) + rebased = data.draw(st.lists(st.sampled_from(carried), unique=True)) + factors = {measure: data.draw(_EXACT_SCALES) for measure in rebased} + own_aging, control_aging = data.draw(_EXACT_SCALES), data.draw(_EXACT_SCALES) + band = data.draw(st.sampled_from([math.inf, _CD.STATE_CD_FACTOR_BAND])) + + def is_cd_file(spec) -> bool: + return spec.metadata["ledger_layout_record_set_spec_id"] == _CD_FILE + + base = [_aged(spec, own_aging) if is_cd_file(spec) else spec for spec in specs] + shared = [ + _as_national_share( + spec, factors[spec.metadata["source_measure_id"]], control_aging + ) + if is_cd_file(spec) and spec.metadata["source_measure_id"] in factors + else spec + for spec in base + ] + + def surface(rows): + return _CD.state_cd_soi_surface( + rows, + state_surface_predicate=_TOOL.soi_surface_predicate("state"), + crosswalk=crosswalk, + level_bridges=_BRIDGES, + factor_band=band, + ) + + # A bridged row re-ages its share sibling only when both age on one + # series; this world bridges wages through the return count (AGI series), + # so rebasing the return count there refuses. + cd_state = { + (spec.metadata["state_fips"], spec.metadata["source_measure_id"]): spec + for spec in base + if is_cd_file(spec) and spec.metadata["ledger_geography_level"] == "state" + } + mismatched = any( + _BRIDGES[measure] in factors + and _CD._aging_series(spec) + != _CD._aging_series(cd_state[(state, _BRIDGES[measure])]) + for (state, measure), spec in cd_state.items() + if (state, measure) not in ht2 + ) + before = surface(base) + if mismatched: + with pytest.raises(ValueError, match="re-aged only like a row on its own"): + surface(shared) + return + after = surface(shared) + assert [spec.name for spec in after.specs] == [spec.name for spec in before.specs] + for old, new in zip(before.specs, after.specs, strict=True): + assert new.value == pytest.approx(old.value, rel=1e-12), new.name + for key in ("counts", "dropped"): + assert after.receipt[key] == before.receipt[key] + + def verdicts(receipt): + return sorted( + (row["state_fips"], row["measure"], row["basis"], row["verdict"]) + for row in receipt["factor_band"]["out_of_band"] + ) + + assert verdicts(after.receipt) == verdicts(before.receipt) + # The receipt reads the district file's own levels. A bridge re-ages the + # sibling like the bridged row, so its factor and the sibling's recorded + # ratio and median are unchanged whatever the aging. A block reads its + # shares at the file level aged as the shares are, from the control's + # period: where that matches the file's own aging (as on the pinned feed), + # its rebase factor and every district row's record of the file value, + # state sum and factor are unchanged too. + same_aging = own_aging == control_aging + recorded = { + (row["state_fips"], row["measure"], row["verdict"]): row + for row in before.receipt["factor_band"]["out_of_band"] + } + for row in after.receipt["factor_band"]["out_of_band"]: + old_row = recorded[(row["state_fips"], row["measure"], row["verdict"])] + keys = ["relative"] + if same_aging or row["verdict"] == "sibling_vintage_ratio": + keys += ["ratio", "median_ratio"] + for key in keys: + assert row[key] == pytest.approx(old_row[key], rel=1e-12, nan_ok=True) + keys = ["level_bridge_factor_by_measure"] + if same_aging: + keys.append("rebase_factor_by_measure") + for key in keys: + assert after.receipt[key].keys() == before.receipt[key].keys() + for measure, summary in after.receipt[key].items(): + assert summary == pytest.approx(before.receipt[key][measure], rel=1e-12) + metadata_keys = ["state_cd_level_bridge_factor"] + if same_aging: + metadata_keys += [ + "state_cd_cd_file_value", + "state_cd_cd_file_state_sum", + "state_cd_rebase_factor", + ] + for old, new in zip(before.specs, after.specs, strict=True): + for key in metadata_keys: + if key in old.metadata: + assert float(new.metadata[key]) == pytest.approx( + float(old.metadata[key]), rel=1e-12 + ), (new.name, key) + + +@pytest.mark.parametrize("control_aging", [1.0, 2.0]) +def test_a_national_share_sibling_keeps_its_bridges_and_recorded_ratios( + control_aging: float, +) -> None: + """State 14's two return-count levels disagree 2x, so its bridge is out of + band. Binding the district file's return counts as national shares (x1.5) + leaves every bound value, the out-of-band rows and their recorded ratios + where they were: the sibling is still read at the file's own level.""" + + specs, crosswalk = _bridge_world( + {"10": 1.0, "11": 1.0, "12": 1.0, "13": 1.0, "14": 2.0}, {} + ) + base = [ + _aged(spec, 1.0) + if spec.metadata["ledger_layout_record_set_spec_id"] == _CD_FILE + else spec + for spec in specs + ] + shared = [ + _as_national_share(spec, 1.5, control_aging) + if spec.metadata["ledger_layout_record_set_spec_id"] == _CD_FILE + and spec.metadata["source_measure_id"] == "return_count" + else spec + for spec in base + ] + before, after = ( + _CD.state_cd_soi_surface( + rows, + state_surface_predicate=_TOOL.soi_surface_predicate("state"), + crosswalk=crosswalk, + level_bridges={"charitable_amount": "return_count"}, + ) + for rows in (base, shared) + ) + assert {spec.name: spec.value for spec in after.specs} == pytest.approx( + {spec.name: spec.value for spec in before.specs}, rel=1e-12 + ) + rows = { + (row["state_fips"], row["measure"], row["verdict"]): row + for row in after.receipt["factor_band"]["out_of_band"] + } + vintage = rows[("14", "charitable_amount", "sibling_vintage_ratio")] + assert (vintage["ratio"], vintage["median_ratio"]) == pytest.approx((2.0, 1.0)) + assert set(rows) == { + (row["state_fips"], row["measure"], row["verdict"]) + for row in before.receipt["factor_band"]["out_of_band"] + } + + +def test_a_share_sibling_re_ages_by_series_not_by_receipt() -> None: + """The series comes from the compiler's aging rule, not the receipt + (#1250 review). A share landed at the build year carries the no-op + ``source_equals_build`` receipt and still re-ages like an unrebased row + on its series; a pair on different series refuses.""" + + def amount(name, measure, value, aging, source): + spec = _soi_spec( + name, measure, value, level="state", record_set=_CD_FILE, state="10" + ) + return dataclasses.replace( + spec, + metadata={ + **spec.metadata, + "aging_factor": repr(aging), + "aging_factor_source": source, + }, + ) + + agi_series = "cbo.revenue_projection.ty2024.income_by_source.agi.projected_amount" + charitable = amount( + "cdfile.10_total.charitable_amount", "charitable_amount", 11.0, 1.1, agi_series + ) + sibling = _as_national_share( + amount( + "cdfile.10_total.adjusted_gross_income", + "adjusted_gross_income", + 100.0, + 1.0, + "x", + ), + 1.25, + 1.0, + ) + sibling = dataclasses.replace( + sibling, + metadata={**sibling.metadata, "aging_factor_source": "source_equals_build"}, + ) + assert _CD.cd_file_level(sibling, aged_like=charitable) == pytest.approx(110.0) + wages = amount( + "cdfile.10_total.wages_salaries_amount", + "wages_salaries_amount", + 50.0, + 1.1, + agi_series, + ) + with pytest.raises(ValueError, match="'wages_and_salaries'"): + _CD.cd_file_level(sibling, aged_like=wages) + + +def test_a_build_year_control_keeps_state_cd_bridges() -> None: + """End to end (#1250 review): district itemized and charitable rows stamped + 2023, an itemized control at the 2024 build year, and CBO AGI projections. + The itemized shares land at 2024 unaged while charitable ages 2023 to 2024, + and ``state_cd`` binds the same values as with the district pass off.""" + import microcosm.build.us_runtime.fiscal_targets as fiscal_targets + from microcosm.build.us_runtime import compile_us_fiscal_target_registry + from test_support.microcosm_build.us_fiscal_targets import ( + _cbo_income_source_projection_fact, + _dynamic_ledger_fact, + packaged_reference_facts, + ) + + cd_set = "irs_soi.ty2023.congressional_district_2022.all_returns" + + def fact(record_set, row, measure, value, period, level, geography, spec_id): + result = _dynamic_ledger_fact( + source_record_id=f"{record_set}.{row}.{measure}", + source_name="irs_soi", + measure_id=measure, + value=value, + period_value=period, + geography_level=level, + geography_id=geography, + dimensions={"income_range": "all", "filing_status": "all"}, + layout_record_set_id=record_set, + groupby_dimension="irs_soi.congressional_district", + groupby_value_id=row, + ) + result["layout"]["record_set_spec_id"] = spec_id + if measure == "itemized_deductions_amount": + result["layout"]["source_column_id"] = "A04470" + return result + + cd_spec = "irs_soi.congressional_district_2022.all_returns.v1" + facts = [*packaged_reference_facts()] + for measure, us, states in ( + ( + "itemized_deductions_amount", + 1_000.0, + {"15": (300.0, 200.0), "33": (150.0, 100.0)}, + ), + ("charitable_amount", 100.0, {"15": (30.0, 20.0), "33": (15.0, 10.0)}), + ): + facts.append( + fact(cd_set, "us", measure, us, 2023, "country", "0100000US", cd_spec) + ) + for fips, (first, second) in states.items(): + facts.append( + fact( + cd_set, + f"s{fips}_total", + measure, + first + second, + 2023, + "state", + f"0400000US{fips}", + cd_spec, + ) + ) + for index, value in ((1, first), (2, second)): + facts.append( + fact( + cd_set, + f"s{fips}_0{index}", + measure, + value, + 2023, + "congressional_district", + f"5001700US{fips}0{index}", + cd_spec, + ) + ) + ht2_spec = "irs_soi.historic_table_2.state_broad_totals.v1" + facts.append( + fact( + "irs_soi.ty2022.historic_table_2.us", + "all", + "itemized_deductions_amount", + 1_050.0, + 2022, + "country", + "0100000US", + ht2_spec, + ) + ) + for fips, postal, value in (("15", "hi", 520.0), ("33", "nh", 260.0)): + facts.append( + fact( + f"irs_soi.ty2022.historic_table_2.state_broad.{postal}", + "all", + "itemized_deductions_amount", + value, + 2022, + "state", + f"0400000US{fips}", + ht2_spec, + ) + ) + control = _dynamic_ledger_fact( + source_record_id="irs_soi.ty2024.table_1_2.all_returns.all.total_itemized_deductions_amount", + source_name="irs_soi", + measure_id="total_itemized_deductions_amount", + value=1_200.0, + period_value=2024, + dimensions={"income_range": "all", "filing_status": "all"}, + layout_record_set_id="irs_soi.ty2024.table_1_2.all_returns", + groupby_dimension="us:statutes/26/62#adjusted_gross_income", + groupby_value_id="all", + ) + facts.append(control) + for year, value in ((2022, 100.0), (2023, 110.0), (2024, 121.0)): + facts.append( + _cbo_income_source_projection_fact( + year, "adjusted_gross_income", value=value + ) + ) + crosswalk = [ + { + "source_geography_id": f"5001700US{fips}0{index}", + "target_geography_id": f"5001900US{fips}0{index}", + "weight": 1.0, + } + for fips in ("15", "33") + for index in (1, 2) + ] + + def surface(): + registry = compile_us_fiscal_target_registry( + facts, + target_period=2024, + congressional_district_vintage_crosswalk=crosswalk, + age_targets=True, + ) + soi = [spec for spec in registry.specs if spec.family == "irs_soi"] + return soi, _CD.state_cd_soi_surface( + soi, + state_surface_predicate=_TOOL.soi_surface_predicate("state"), + crosswalk=pd.DataFrame(crosswalk), + level_bridges={"charitable_amount": "itemized_deductions_amount"}, + factor_band=math.inf, + ) + + soi, rebased = surface() + by_name = {spec.name: spec for spec in soi} + share = by_name[f"{cd_set}.s15_total.itemized_deductions_amount"] + assert share.metadata["aging_factor_source"] == "source_equals_build" + assert share.value == pytest.approx(600.0) + charitable = by_name[f"{cd_set}.s15_total.charitable_amount"] + assert charitable.value == pytest.approx(55.0) + + original = fiscal_targets._rebase_stale_soi_district_distributions + try: + fiscal_targets._rebase_stale_soi_district_distributions = ( + lambda registry, facts, **_: registry + ) + _, levels = surface() + finally: + fiscal_targets._rebase_stale_soi_district_distributions = original + assert {spec.name: spec.value for spec in rebased.specs} == pytest.approx( + {spec.name: spec.value for spec in levels.specs}, rel=1e-12 + ) + assert ( + rebased.receipt["level_bridge_factor_by_measure"]["charitable_amount"]["n"] == 2 + ) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_targets.py b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_targets.py index 9f9031f852..8d6bf3230c 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_targets.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_targets.py @@ -893,10 +893,12 @@ def test_pinned_feed_exclusion_receipt(pinned_feed_national_state_surface) -> No def test_pinned_feed_national_state_surface_restores_the_fences( pinned_feed_national_state_surface, ) -> None: - """The release surface on the pinned feed: 33,235 compiled targets after + """The release surface on the pinned feed: 33,224 compiled targets after Medicaid substitution, 6,087 national_state targets at registry 0e899cd7f893, 32 CHIP rows none of them for an M-CHIP state, no - other-income row and no tips return count. microcosm#1243 took it from + other-income row and no tips return count. microcosm#1250 retired the + congressional-district file's 11 US rows of the Table 1.x-controlled + columns (from 33,235; national_state unchanged). microcosm#1243 took it from 33,250 / 6,102 at 882f4764a248 by rebasing 14 Historic Table 2 state families onto their Table 1.x controls and retiring the 14 HT2 US rows and HT2's US-workbook return count. microcosm#1035 moved it from @@ -912,7 +914,7 @@ def test_pinned_feed_national_state_surface_restores_the_fences( ty2020 tips return count it was 32,843 / 5,695 at 386fac439e77 (docs/us-chronicle-feed-repin.md).""" registry, surface, _ = pinned_feed_national_state_surface - assert len(registry.specs) == 33_235 + assert len(registry.specs) == 33_224 assert len(surface.specs) == 6_087 assert surface.version == "0e899cd7f893" chip = [ @@ -1147,6 +1149,140 @@ def test_pinned_feed_ht2_state_rows_take_their_national_control_level( ] == [], measure_id +#: Per congressional-district measure on the pinned feed: the factor +#: control / the district file's US row (22incd.csv, TY2022 stamped ty2023). +#: The control is the measure's ``_PINNED_HT2_CONTROLS`` entry. +_PINNED_CD_FACTORS = { + "adjusted_gross_income": "1.05970317276013", + "itemized_deductions_amount": "1.09974729652659", + "return_count": "1.02724364722741", + "taxable_ira_distributions_amount": "1.0247107152496", + "taxable_ira_distributions_returns": "1.02876142432905", + "taxable_pension_income_amount": "1.05382110367184", + "taxable_pension_income_returns": "1.00136008211203", + "taxable_social_security_amount": "1.1692464093015", + "taxable_social_security_returns": "1.06412503801664", + "unemployment_compensation_amount": "1.0546342976899", + "unemployment_compensation_returns": "1.02721646869479", +} + + +def test_pinned_feed_district_rows_take_their_national_control_level( + pinned_feed_national_state_surface, +) -> None: + """On the full surface each of the 11 Table 1.x-controlled columns the + district file carries enters as 51 state and 436 district shares of the + file's US row times the control (microcosm#1250). The file's states and + its districts each sum to the control after aging, every district + reconciles to its state with factor 1, and the US row is gone.""" + registry, _, _ = pinned_feed_national_state_surface + by_name = {spec.name: spec for spec in registry.specs} + for measure_id, factor in _PINNED_CD_FACTORS.items(): + control_id = _PINNED_HT2_CONTROLS[measure_id][0] + control = by_name[control_id] + concept = fiscal_targets._SOI_HT2_NATIONAL_CONTROLS[measure_id].concept + rows = [ + spec + for spec in registry.specs + if spec.metadata.get("source_measure_id") == measure_id + and ".congressional_district_2022." in spec.name + ] + levels = { + level: [ + spec + for spec in rows + if spec.metadata["ledger_geography_level"] == level + ] + for level in ("country", "state", "congressional_district") + } + assert {level: len(found) for level, found in levels.items()} == { + "country": 0, + "state": 51, + "congressional_district": 436, + }, measure_id + for spec in rows: + metadata = spec.metadata + assert metadata["uprating_index_source_record_id"] == control_id + assert metadata["uprating_factor"] == factor + assert metadata["uprating_to_period"] == "2023" + assert metadata["soi_share_total_source_record_id"] == ( + f"irs_soi.ty2023.congressional_district_2022.all_returns.us.{measure_id}" + ) + assert metadata["aging_factor"] == control.metadata["aging_factor"] + assert metadata["soi_source_concept"] == concept + assert metadata["soi_control_concept"] == concept + for level in ("state", "congressional_district"): + total = math.fsum(spec.value for spec in levels[level]) + assert math.isclose(total, control.value, rel_tol=1e-12), ( + measure_id, + level, + ) + assert { + spec.metadata["hierarchy_reconciliation_factor"] + for spec in levels["congressional_district"] + } == {"1"} + + +def test_pinned_feed_full_surface_has_one_national_value_per_controlled_quantity( + pinned_feed_national_state_surface, +) -> None: + """Invariant on the full surface: every model quantity a + ``_SOI_HT2_NATIONAL_CONTROLS`` control measures has one national all-AGI + value, its control's. Before microcosm#1250 the district file's US row + was a second value for 11 of them, 1.001x (taxable pension returns) to + 1.169x (taxable Social Security) below the control. Table 1.2 repeats + Table 1.1's return count and AGI at the same value, and no state or + district family of these quantities sums past it.""" + registry, _, _ = pinned_feed_national_state_surface + controls = { + spec.metadata["source_measure_id"]: spec + for spec in registry.specs + if spec.name in {entry[0] for entry in _PINNED_HT2_CONTROLS.values()} + } + quantities = { + fiscal_targets._soi_national_quantity(spec): spec for spec in controls.values() + } + assert len(quantities) == 14 + national: dict[tuple[object, ...], list[object]] = {} + families: dict[tuple[object, ...], float] = {} + for spec in registry.specs: + if spec.family != "irs_soi": + continue + quantity = fiscal_targets._soi_national_quantity(spec) + if quantity not in quantities: + continue + if not spec.metadata.get("state_fips"): + national.setdefault(quantity, []).append(spec) + continue + key = ( + quantity, + fiscal_targets._soi_record_set_family( + spec.metadata.get("ledger_layout_record_set_id", "") + ), + spec.metadata["ledger_geography_level"], + ) + families[key] = families.get(key, 0.0) + spec.value + assert set(national) == set(quantities) + for quantity, specs in national.items(): + control = quantities[quantity] + assert { + spec.name: spec.value + for spec in specs + if not math.isclose(spec.value, control.value, rel_tol=1e-12) + } == {}, control.name + assert not [ + spec.name for spec in specs if ".congressional_district_" in spec.name + ] + assert { + key[1:]: total / quantities[key[0]].value + for key, total in families.items() + if total > quantities[key[0]].value * (1 + 1e-12) + } == {} + # Both the district file's states and its districts, and HT2's states, + # are compared for each of the 11 carried quantities. + assert len(families) == 14 + 11 * 2 + 11 + + def test_reviewed_zero_support_facts_are_not_active_targets() -> None: excluded_source_record_id = ( "hhs_acf_tanf.fy2024.cash_assistance.ar." diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_soi_district_national_controls.py b/packages/microcosm-build/tests/engine_free/us/test_us_soi_district_national_controls.py new file mode 100644 index 0000000000..013695aae9 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/us/test_us_soi_district_national_controls.py @@ -0,0 +1,907 @@ +"""Congressional-district rows rebased onto the national controls (#1250). + +The congressional-district file (``22incd.csv``) publishes the columns of +``_SOI_HT2_NATIONAL_CONTROLS`` at the same Form 1040 lines, so its state and +district rows are shares of the nation and its US row is a second, narrower +national level beside the Table 1.x control. +``_rebase_stale_soi_district_distributions`` takes the rows as shares of the +file's own US row times that control and retires the US row. The synthetic +tests below pin the rule and its invariants; the pinned-feed checks sit +beside the release fixture in ``test_us_fiscal_targets.py``. +""" + +from __future__ import annotations + +import math + +import pytest + +from microcosm.build.us_runtime import compile_us_fiscal_target_registry, fiscal_targets +from test_support.microcosm_build.us_fiscal_targets import ( + _dynamic_ledger_fact, + packaged_reference_facts, +) + +REGISTER = fiscal_targets._SOI_HT2_NATIONAL_CONTROLS +#: The measures the TY2022 district package carries: the register less wages +#: and taxable income, which it does not publish. +CD_MEASURES = sorted( + set(REGISTER) + - {"wages_salaries_amount", "wages_salaries_returns", "taxable_income_amount"} +) +#: States with two districts in the 117th and 119th plans, so a state row +#: and both its districts make a complete hierarchy block. +STATES = ( + ("hi", "15", ("1501", "1502")), + ("nh", "33", ("3301", "3302")), + ("me", "23", ("2301", "2302")), +) +_REVIEWED = object() + + +def _record_set(stamp: int) -> str: + return f"irs_soi.ty{stamp}.congressional_district_2022.all_returns" + + +def _us_id(measure_id: str, stamp: int = 2023, row: str = "us") -> str: + return f"{_record_set(stamp)}.{row}.{measure_id}" + + +def _cd_fact( + measure_id: str, + value: float, + *, + row: str, + level: str, + geography_id: str, + stamp: int = 2023, + column: object = _REVIEWED, +) -> dict[str, object]: + fact = _dynamic_ledger_fact( + source_record_id=f"{_record_set(stamp)}.{row}.{measure_id}", + source_name="irs_soi", + measure_id=measure_id, + value=value, + period_value=stamp, + geography_level=level, + geography_id=geography_id, + dimensions={"income_range": "all", "filing_status": "all"}, + layout_record_set_id=_record_set(stamp), + groupby_dimension="irs_soi.congressional_district", + groupby_value_id=row, + ) + if column is _REVIEWED: + column = REGISTER[measure_id].source_column + if column is not None: + fact["layout"]["source_column_id"] = column + return fact + + +def _cd_facts( + measure_id: str, + *, + stamp: int = 2023, + us: float | None = 1_000.0, + districts: tuple[tuple[float, float], ...] = ((300.0, 200.0), (150.0, 100.0)), + column: object = _REVIEWED, +) -> list[dict[str, object]]: + """The district file's US row and, per state, its total and two districts.""" + facts = [] + if us is not None: + facts.append( + _cd_fact( + measure_id, + us, + row="us", + level="country", + geography_id="0100000US", + stamp=stamp, + column=column, + ) + ) + for (postal, fips, geoids), values in zip(STATES, districts, strict=False): + facts.append( + _cd_fact( + measure_id, + math.fsum(values), + row=f"{postal}_total", + level="state", + geography_id=f"0400000US{fips}", + stamp=stamp, + column=column, + ) + ) + for geoid, value in zip(geoids, values, strict=True): + facts.append( + _cd_fact( + measure_id, + value, + row=f"{postal}_{geoid[2:]}", + level="congressional_district", + geography_id=f"5001700US{geoid}", + stamp=stamp, + column=column, + ) + ) + return facts + + +def _control_fact( + measure_id: str, *, period: int = 2023, value: float = 1_100.0 +) -> dict[str, object]: + rule = REGISTER[measure_id] + suffix = ".all_returns" if rule.control_family == "table_1_2" else "" + record_set_id = f"irs_soi.ty{period}.{rule.control_family}{suffix}" + return _dynamic_ledger_fact( + source_record_id=f"{record_set_id}.all.{rule.control_measure_id}", + source_name="irs_soi", + measure_id=rule.control_measure_id, + value=value, + period_value=period, + dimensions={"income_range": "all", "filing_status": "all"}, + layout_record_set_id=record_set_id, + groupby_dimension="us:statutes/26/62#adjusted_gross_income", + groupby_value_id="all", + ) + + +def _control_id(measure_id: str, period: int = 2023) -> str: + return _control_fact(measure_id, period=period)["lineage"]["source_record_id"] + + +def _ht2_facts( + measure_id: str, *, us: float, states: tuple[float, ...], period: int = 2022 +) -> list[dict[str, object]]: + """An HT2 US row and state rows for the first ``len(states)`` STATES.""" + facts = [ + _dynamic_ledger_fact( + source_record_id=f"irs_soi.ty{period}.historic_table_2.us.all.{measure_id}", + source_name="irs_soi", + measure_id=measure_id, + value=us, + period_value=period, + dimensions={"income_range": "all", "filing_status": "all"}, + layout_record_set_id=f"irs_soi.ty{period}.historic_table_2.us", + groupby_dimension="us:statutes/26/62#adjusted_gross_income", + groupby_value_id="all", + ) + ] + for (postal, fips, _), value in zip(STATES, states, strict=False): + record_set_id = f"irs_soi.ty{period}.historic_table_2.state_broad.{postal}" + facts.append( + _dynamic_ledger_fact( + source_record_id=f"{record_set_id}.all.{measure_id}", + source_name="irs_soi", + measure_id=measure_id, + value=value, + period_value=period, + geography_level="state", + geography_id=f"0400000US{fips}", + dimensions={"income_range": "all", "filing_status": "all"}, + layout_record_set_id=record_set_id, + groupby_dimension="us:statutes/26/62#adjusted_gross_income", + groupby_value_id="all", + ) + ) + return facts + + +def _ht2_state_id(measure_id: str, postal: str, period: int = 2022) -> str: + return f"irs_soi.ty{period}.historic_table_2.state_broad.{postal}.all.{measure_id}" + + +def _compile(facts, *, crosswalk=None): + return compile_us_fiscal_target_registry( + [*packaged_reference_facts(), *facts], + target_period=2024, + congressional_district_vintage_crosswalk=crosswalk, + allow_unaged_dollar_targets=True, + ) + + +def _by_record(registry) -> dict[str, object]: + return {spec.metadata["ledger_source_record_id"]: spec for spec in registry.specs} + + +def _cd_rows(registry, measure_id: str) -> dict[str, object]: + """The district-file rows of one measure, keyed by their row id.""" + return { + spec.metadata["ledger_source_record_id"].split(".")[-2]: spec + for spec in registry.specs + if spec.metadata.get("source_measure_id") == measure_id + and fiscal_targets._soi_record_set_family( + spec.metadata.get("ledger_layout_record_set_id", "") + ) + == "congressional_district" + } + + +def _national_rows(registry, quantity) -> list[object]: + return [ + spec + for spec in registry.specs + if spec.family == "irs_soi" + and not spec.metadata.get("state_fips") + and fiscal_targets._soi_national_quantity(spec) == quantity + ] + + +# -------------------------------------------------------------------------- +# The register + + +#: Each registered measure's IRS column in ``22incddocguide.docx`` (TY2022), +#: with the Form 1040 line it gives; the HT2 guides give the same column at +#: the same line (``docs/us-soi-national-controls.md``). +_DISTRICT_GUIDE_COLUMNS = { + "return_count": "N1", # number of returns + "adjusted_gross_income": "A00100", # 1040:11 + "wages_salaries_amount": "A00200", # 1040:1z + "wages_salaries_returns": "N00200", # 1040:1z + "taxable_ira_distributions_amount": "A01400", # 1040:4b + "taxable_ira_distributions_returns": "N01400", # 1040:4b + "taxable_pension_income_amount": "A01700", # 1040:5b + "taxable_pension_income_returns": "N01700", # 1040:5b + "unemployment_compensation_amount": "A02300", # Schedule 1:7 + "unemployment_compensation_returns": "N02300", # Schedule 1:7 + "taxable_social_security_amount": "A02500", # 1040:6b + "taxable_social_security_returns": "N02500", # 1040:6b + "itemized_deductions_amount": "A04470", # 1040:12 + "taxable_income_amount": "A04800", # 1040:15 +} + + +def test_the_register_names_each_columns_reviewed_irs_id() -> None: + assert { + measure_id: rule.source_column for measure_id, rule in REGISTER.items() + } == _DISTRICT_GUIDE_COLUMNS + assert len(CD_MEASURES) == 11 + + +# -------------------------------------------------------------------------- +# The rebase + + +@pytest.mark.parametrize("measure_id", CD_MEASURES) +def test_district_rows_become_shares_of_their_national_control( + measure_id: str, +) -> None: + """A US row of 1,000 with HI 300 + 200 and NH 150 + 100 under an 1,100 + control: every row gains x1.1, the US row retires, and the control's row + is the quantity's only national level.""" + registry = _compile([*_cd_facts(measure_id), _control_fact(measure_id)]) + rows = _cd_rows(registry, measure_id) + + assert "us" not in rows + expected = { + "hi_total": 550.0, + "hi_01": 330.0, + "hi_02": 220.0, + "nh_total": 275.0, + "nh_01": 165.0, + "nh_02": 110.0, + } + assert {row: spec.value for row, spec in rows.items()} == pytest.approx( + expected, rel=1e-12 + ) + concept = REGISTER[measure_id].concept + for row, spec in rows.items(): + metadata = spec.metadata + assert metadata["uprating_factor"] == "1.1" + assert metadata["uprating_index"] == f"total_{measure_id}" + assert metadata["uprating_from_period"] == "2023" + assert metadata["uprating_to_period"] == "2023" + assert metadata["uprating_index_source_record_id"] == _control_id(measure_id) + assert metadata["soi_share_total_source_record_id"] == _us_id(measure_id) + assert metadata["soi_source_concept"] == concept + assert metadata["soi_control_concept"] == concept + assert metadata["stale_distribution_rebased_to_active_total"] == "true" + if row.endswith(("_01", "_02")): + # One factor scales the states and their districts alike. + assert metadata["hierarchy_reconciliation_factor"] == "1" + assert [ + spec.metadata["ledger_source_record_id"] + for spec in _national_rows( + registry, fiscal_targets._soi_national_quantity(rows["hi_total"]) + ) + ] == [_control_id(measure_id)] + + +def test_each_family_divides_by_its_own_us_row() -> None: + """HT2 rows divide by the HT2 US row and district rows by the district + US row, never by the other's; both US rows retire.""" + measure_id = "taxable_social_security_amount" + registry = _compile( + [ + *_cd_facts(measure_id), + *_ht2_facts(measure_id, us=900.0, states=(400.0, 200.0)), + _control_fact(measure_id), + ] + ) + specs = _by_record(registry) + rows = _cd_rows(registry, measure_id) + + assert specs[_ht2_state_id(measure_id, "hi")].value == pytest.approx( + 400.0 * 1_100.0 / 900.0, rel=1e-12 + ) + assert rows["hi_total"].value == pytest.approx(550.0, rel=1e-12) + assert rows["hi_total"].metadata["soi_share_total_source_record_id"] == _us_id( + measure_id + ) + assert [ + spec.metadata["ledger_source_record_id"] + for spec in _national_rows( + registry, fiscal_targets._soi_national_quantity(rows["hi_total"]) + ) + ] == [_control_id(measure_id)] + + +@pytest.mark.parametrize( + "control", + [ + None, + # Older than the row's (stamped) period: the share would run backwards. + {"period": 2022}, + # After the build period: not yet a level for a 2024 build. + {"period": 2025}, + ], + ids=["no-control", "older-control", "future-control"], +) +def test_district_rows_keep_their_level_without_an_eligible_control(control) -> None: + measure_id = "adjusted_gross_income" + facts = _cd_facts(measure_id) + if control is not None: + facts.append(_control_fact(measure_id, **control)) + rows = _cd_rows(_compile(facts), measure_id) + + assert rows["us"].value == 1_000.0 + assert rows["hi_total"].value == 500.0 + assert rows["hi_01"].value == 300.0 + assert "uprating_factor" not in rows["hi_total"].metadata + + +@pytest.mark.parametrize("us", [None, 0.0], ids=["no-us-row", "zero-us-row"]) +def test_district_rows_without_a_denominator_are_dropped_beside_a_control(us) -> None: + """A row with an eligible control but no nonzero US row of its own file + cannot be a share, and as a level it would be the stale second value the + pass removes, so it is dropped (as #1229 and #1040 do).""" + measure_id = "return_count" + registry = _compile([*_cd_facts(measure_id, us=us), _control_fact(measure_id)]) + assert _cd_rows(registry, measure_id) == {} + assert _control_id(measure_id) in _by_record(registry) + + +def test_the_share_ignores_the_package_stamp() -> None: + """The feed stamps 22incd.csv ty2023; stamped truthfully at 2022 the + shares and their landing period are the same, because the stamp cancels + in row / US row.""" + measure_id = "unemployment_compensation_amount" + stamped, truthful = ( + _cd_rows( + _compile([*_cd_facts(measure_id, stamp=stamp), _control_fact(measure_id)]), + measure_id, + ) + for stamp in (2023, 2022) + ) + assert set(stamped) == set(truthful) + for row, spec in stamped.items(): + assert spec.value == truthful[row].value + assert spec.metadata["uprating_to_period"] == "2023" + assert truthful[row].metadata["uprating_to_period"] == "2023" + assert truthful[row].metadata["uprating_from_period"] == "2022" + assert truthful[row].metadata["soi_share_total_source_record_id"] == _us_id( + measure_id, stamp=2022 + ) + + +# -------------------------------------------------------------------------- +# The column guard + + +@pytest.mark.parametrize("column", ["N2", "A00101", None], ids=str) +def test_a_district_fact_under_an_unreviewed_column_refuses(column) -> None: + """Rebasing a measure id that reads another IRS column (or none) would + bind the wrong concept to the control, so the compile refuses.""" + measure_id = "return_count" + facts = _cd_facts(measure_id) + facts[1]["layout"].pop("source_column_id") + if column is not None: + facts[1]["layout"]["source_column_id"] = column + with pytest.raises( + fiscal_targets.UnreviewedSoiNationalControlColumnError, + match=facts[1]["lineage"]["source_record_id"], + ): + _compile([*facts, _control_fact(measure_id)]) + # Without a control nothing is rebased, so nothing new is claimed. + rows = _cd_rows(_compile(facts), measure_id) + assert rows["hi_total"].value == 500.0 + + +@pytest.mark.parametrize("control_period", [2022, 2025], ids=["older", "future"]) +def test_the_column_guard_ignores_rows_without_an_eligible_control( + control_period: int, +) -> None: + """The guard checks exactly the rows the pass would rebase: beside an + older or a future control the rows keep their levels, so an unreviewed + column is not refused.""" + measure_id = "taxable_social_security_returns" + facts = _cd_facts(measure_id, column="N02510") + rows = _cd_rows( + _compile([*facts, _control_fact(measure_id, period=control_period)]), + measure_id, + ) + assert rows["us"].value == 1_000.0 + assert rows["hi_total"].value == 500.0 + + +def test_the_column_guard_sees_each_source_district_before_the_crosswalk() -> None: + """The crosswalk merges source districts into one fact that keeps the + first contributor's layout, so a mislabeled second contributor would hide + behind a reviewed column; the guard runs on the facts as fed.""" + measure_id = "taxable_pension_income_amount" + facts = _cd_facts(measure_id, districts=((300.0, 200.0),)) + hi_02 = next( + fact + for fact in facts + if fact["lineage"]["source_record_id"].endswith(f"hi_02.{measure_id}") + ) + hi_02["layout"]["source_column_id"] = "A01750" + crosswalk = [ + { + "source_geography_id": "5001700US1501", + "target_geography_id": "5001900US1501", + "weight": 1.0, + }, + { + "source_geography_id": "5001700US1502", + "target_geography_id": "5001900US1501", + "weight": 1.0, + }, + { + "source_geography_id": "5001700US1502", + "target_geography_id": "5001900US1502", + "weight": 1.0, + }, + ] + with pytest.raises( + fiscal_targets.UnreviewedSoiNationalControlColumnError, match="hi_02" + ): + _compile([*facts, _control_fact(measure_id)], crosswalk=crosswalk) + # With both contributors reviewed, the merged district is rebased. + hi_02["layout"]["source_column_id"] = "A01700" + specs = _compile([*facts, _control_fact(measure_id)], crosswalk=crosswalk).specs + (merged,) = [ + spec + for spec in specs + if spec.metadata.get("congressional_district_geoid") == "1501" + and spec.metadata.get("source_measure_id") == measure_id + ] + # 300 + 200 / 2 of the file's districts, times 1,100 / 1,000. + assert merged.value == pytest.approx(440.0, rel=1e-12) + + +def _identity_crosswalk() -> list[dict[str, object]]: + return [ + { + "source_geography_id": f"5001700US{geoid}", + "target_geography_id": f"5001900US{geoid}", + "weight": 1.0, + } + for _, _, geoids in STATES[:2] + for geoid in geoids + ] + + +@pytest.mark.parametrize("duplicate_first", [True, False], ids=["first", "last"]) +def test_a_duplicate_record_id_cannot_hide_another_column(duplicate_first) -> None: + """Two facts with one record id both count (#1250 review). Without a + crosswalk the compile already refuses the duplicate id; the crosswalk + merges the copies into one district, and a copy of Hawaii's first district + read from N2 then refuses in either feed order.""" + measure_id = "return_count" + facts = _cd_facts(measure_id) + original = next( + fact + for fact in facts + if fact["lineage"]["source_record_id"].endswith(f"hi_01.{measure_id}") + ) + duplicate = {**original, "layout": {**original["layout"], "source_column_id": "N2"}} + index = facts.index(original) + facts.insert(index if duplicate_first else index + 1, duplicate) + with pytest.raises(ValueError, match="matched multiple Ledger facts"): + _compile([*facts, _control_fact(measure_id)]) + with pytest.raises( + fiscal_targets.UnreviewedSoiNationalControlColumnError, match="'N2'" + ): + _compile([*facts, _control_fact(measure_id)], crosswalk=_identity_crosswalk()) + + +def test_translated_facts_compile_as_the_crosswalk_compile_does() -> None: + """The translator's output is itself a valid feed: each merged district + carries its contributors' columns, so compiling translated facts gives + the same registry as compiling the source facts with the crosswalk.""" + from microcosm.build.us_runtime.congressional_district_vintage import ( + translate_congressional_district_facts_to_current_vintage, + ) + + measure_id = "taxable_social_security_returns" + facts = [*_cd_facts(measure_id), _control_fact(measure_id)] + crosswalk = _identity_crosswalk() + direct = _compile(facts, crosswalk=crosswalk) + translated = _compile( + translate_congressional_district_facts_to_current_vintage(facts, crosswalk) + ) + assert translated.version == direct.version + assert _cd_rows(translated, measure_id)["hi_total"].value == 550.0 + + +def test_the_column_guard_checks_the_denominator() -> None: + """A share divides by the US row, so a US row read from another column + refuses even when every state and district row is reviewed.""" + measure_id = "unemployment_compensation_amount" + facts = _cd_facts(measure_id) + facts[0]["layout"]["source_column_id"] = "A02310" + with pytest.raises( + fiscal_targets.UnreviewedSoiNationalControlColumnError, match=r"\.us\." + ): + _compile([*facts, _control_fact(measure_id)]) + + +def _cd_agi_slice_fact( + measure_id: str, + value: float, + *, + stamp: int, + column: object = _REVIEWED, + national: bool = False, +) -> dict[str, object]: + """The district file's $50,000-$75,000 AGI slice (agi_stub 5), for Hawaii + or for the US row.""" + row = "us_50k_to_75k" if national else "hi_total_50k_to_75k" + fact = _dynamic_ledger_fact( + source_record_id=f"{_record_set(stamp)}.{row}.{measure_id}", + source_name="irs_soi", + measure_id=measure_id, + value=value, + period_value=stamp, + geography_level="country" if national else "state", + geography_id="0100000US" if national else "0400000US15", + dimensions={"income_range": "50k_to_75k", "filing_status": "all"}, + universe_constraints=[ + {"variable": "adjusted_gross_income", "operator": ">=", "value": 50_000}, + {"variable": "adjusted_gross_income", "operator": "<", "value": 75_000}, + ], + layout_record_set_id=_record_set(stamp), + groupby_dimension="irs_soi.congressional_district", + groupby_value_id=row, + ) + fact["layout"]["source_column_id"] = ( + REGISTER[measure_id].source_column if column is _REVIEWED else column + ) + return fact + + +def test_an_agi_slice_of_the_file_takes_the_same_factor() -> None: + """A district-file AGI slice is a share of the same all-AGI US row, so it + takes the measure's one factor (as #1229 and #1040 rebase district + slices) and the column guard checks it. The pinned package has no + slices, and a slice compiles only at its own period (here 2024).""" + measure_id = "adjusted_gross_income" + + def facts(column: object = _REVIEWED) -> list[dict[str, object]]: + return [ + *_cd_facts(measure_id, stamp=2024), + _cd_agi_slice_fact(measure_id, 120.0, stamp=2024, column=column), + _cd_agi_slice_fact(measure_id, 240.0, stamp=2024, national=True), + _control_fact(measure_id, period=2024), + ] + + specs = _by_record(_compile(facts())) + state_slice = specs[f"{_record_set(2024)}.hi_total_50k_to_75k.{measure_id}"] + us_slice = specs[f"{_record_set(2024)}.us_50k_to_75k.{measure_id}"] + assert state_slice.metadata["agi_lower_bound"] == "50000.0" + # The US slice is a share too, never a second denominator. + assert (state_slice.value, us_slice.value) == pytest.approx( + (132.0, 264.0), rel=1e-12 + ) + for spec in (state_slice, us_slice): + assert spec.metadata["soi_share_total_source_record_id"] == _us_id( + measure_id, stamp=2024 + ) + assert _us_id(measure_id, stamp=2024) not in specs + with pytest.raises(fiscal_targets.UnreviewedSoiNationalControlColumnError): + _compile(facts(column="A00101")) + + +def test_the_column_guard_skips_facts_that_compile_to_no_target() -> None: + """The guard checks the rows the pass rebases (#1250 review). A stale AGI + slice compiles to no target at another period, and a superseded vintage + loses latest-vintage selection, so neither is checked whatever its + column.""" + measure_id = "adjusted_gross_income" + stale_slice = _cd_agi_slice_fact(measure_id, 120.0, stamp=2023, column="A00101") + specs = _by_record( + _compile([*_cd_facts(measure_id), stale_slice, _control_fact(measure_id)]) + ) + assert stale_slice["lineage"]["source_record_id"] not in specs + assert specs[f"{_record_set(2023)}.hi_total.{measure_id}"].value == 550.0 + + measure_id = "return_count" + superseded = _cd_facts(measure_id, stamp=2022, column="N2") + current = _cd_facts(measure_id, stamp=2024) + rows = _cd_rows( + _compile([*superseded, *current, _control_fact(measure_id, period=2024)]), + measure_id, + ) + assert rows["hi_total"].metadata["ledger_source_record_id"] == ( + f"{_record_set(2024)}.hi_total.{measure_id}" + ) + assert rows["hi_total"].value == 550.0 + + +# -------------------------------------------------------------------------- +# The denominator + + +def test_conflicting_district_us_totals_refuse_the_compile() -> None: + """Two US rows of one key with different values would let feed order pick + the denominator.""" + measure_id = "taxable_ira_distributions_amount" + us, *rest = _cd_facts(measure_id) + rival = _cd_fact( + measure_id, 900.0, row="us_v2", level="country", geography_id="0100000US" + ) + for ordered in ((us, rival), (rival, us)): + with pytest.raises( + fiscal_targets.AmbiguousSoiNationalControlError, + match="congressional-district US total", + ): + fiscal_targets._soi_district_us_totals(ordered) + with pytest.raises(fiscal_targets.AmbiguousSoiNationalControlError): + _compile([rival, us, *rest, _control_fact(measure_id)]) + + +def test_rows_never_borrow_another_district_files_us_row() -> None: + """Each package divides by its own US row (#1250 review). A 22incd + package without one keeps no rows beside the control, even when another + district file at the same stamp carries a US row of the measure.""" + measure_id = "return_count" + other = _cd_fact( + measure_id, 2_000.0, row="us", level="country", geography_id="0100000US" + ) + other_set = "irs_soi.ty2023.congressional_district_2023.all_returns" + other["layout"]["record_set_id"] = other_set + other["lineage"]["source_record_id"] = f"{other_set}.us.{measure_id}" + registry = _compile( + [*_cd_facts(measure_id, us=None), other, _control_fact(measure_id)] + ) + assert _cd_rows(registry, measure_id) == {} + + +def test_equal_district_us_totals_name_the_smaller_record_id() -> None: + """Equal twins are one total; the named denominator is the smaller record + id in either feed order, and both twins retire.""" + measure_id = "taxable_ira_distributions_returns" + us, *rest = _cd_facts(measure_id) + twin = _cd_fact( + measure_id, 1_000.0, row="us_2", level="country", geography_id="0100000US" + ) + named = set() + for ordered in ((us, twin), (twin, us)): + registry = _compile([*ordered, *rest, _control_fact(measure_id)]) + rows = _cd_rows(registry, measure_id) + assert not {"us", "us_2"} & set(rows) + named |= { + spec.metadata["soi_share_total_source_record_id"] for spec in rows.values() + } + assert named == {min(_us_id(measure_id), _us_id(measure_id, row="us_2"))} + + +# -------------------------------------------------------------------------- +# Invariants over compiles + + +def test_district_shares_conserve_the_control_and_leave_one_national_level() -> None: + """Property over compiles, for every carried column, either package + stamp, any control vintage (or none), any states, other-area residual, + HT2 rows alongside or not, an equal twin US row or not, and any feed + order: + + * **Shares conserved.** With an eligible control (not after the build + period, not before the row's period), each district-file row is its + value times control / the file's US row, and names the (smallest) US + row id as its denominator. The states sum to the control times their + share of the US row, so never past the control, and each state's + districts sum to the state with reconciliation factor 1. + * **Own denominator.** HT2 rows divide by the HT2 US row. + * **One national level.** The control's row is the only national + all-AGI row of the quantity: every district US row and the HT2 US row + retire. + * **No eligible control, no change.** The rows and the US row keep + their levels. + * **Order-free.** Shuffling the facts compiles the same registry. + """ + pytest.importorskip("hypothesis") + from hypothesis import given, settings + from hypothesis import strategies as st + + pairs = st.tuples( + st.integers(min_value=1, max_value=5_000_000), + st.integers(min_value=1, max_value=5_000_000), + ) + + @settings(max_examples=40, deadline=None) + @given( + st.sampled_from(CD_MEASURES), + st.sampled_from((2022, 2023)), + st.one_of(st.none(), st.integers(min_value=2020, max_value=2026)), + st.integers(min_value=1, max_value=50_000_000), + st.lists(pairs, min_size=1, max_size=3), + st.integers(min_value=0, max_value=5_000_000), + st.booleans(), + st.sampled_from((None, "aa_us", "zz_us")), + st.randoms(use_true_random=False), + ) + def check( + measure_id, + stamp, + control_period, + control_value, + districts, + other_areas, + with_ht2, + twin_row, + rng, + ) -> None: + state_totals = [math.fsum(pair) for pair in districts] + us_value = float(sum(state_totals) + other_areas) + facts = _cd_facts( + measure_id, + stamp=stamp, + us=us_value, + districts=tuple(tuple(map(float, pair)) for pair in districts), + ) + if twin_row is not None: + facts.append( + _cd_fact( + measure_id, + us_value, + row=twin_row, + level="country", + geography_id="0100000US", + stamp=stamp, + ) + ) + ht2_states = tuple(2.0 * total + 1.0 for total in state_totals) + ht2_us = math.fsum(ht2_states) + 7.0 + if with_ht2: + facts += _ht2_facts(measure_id, us=ht2_us, states=ht2_states) + if control_period is not None: + facts.append( + _control_fact( + measure_id, period=control_period, value=float(control_value) + ) + ) + shuffled = list(facts) + rng.shuffle(shuffled) + registry = _compile(facts) + assert sorted( + (spec.name, spec.value, tuple(sorted(spec.metadata.items()))) + for spec in registry.specs + ) == sorted( + (spec.name, spec.value, tuple(sorted(spec.metadata.items()))) + for spec in _compile(shuffled).specs + ) + + rows = _cd_rows(registry, measure_id) + raw = { + fact["layout"]["groupby_value_id"]: float(fact["value"]) + for fact in facts + if fact["layout"]["record_set_id"] == _record_set(stamp) + } + eligible = control_period is not None and stamp <= control_period <= 2024 + us_rows = {"us", *([twin_row] if twin_row else [])} + if not eligible: + assert {row: spec.value for row, spec in rows.items()} == raw + assert not any("uprating_factor" in s.metadata for s in rows.values()) + return + + factor = control_value / us_value + assert not us_rows & set(rows) + assert set(rows) == set(raw) - us_rows + denominator = min(_us_id(measure_id, stamp, row) for row in us_rows) + for row, spec in rows.items(): + assert math.isclose(spec.value, raw[row] * factor, rel_tol=1e-12) + assert spec.metadata["soi_share_total_source_record_id"] == denominator + assert spec.metadata["uprating_to_period"] == str(control_period) + state_sum = math.fsum( + spec.value for row, spec in rows.items() if row.endswith("_total") + ) + assert math.isclose( + state_sum, + control_value * sum(state_totals) / us_value, + rel_tol=1e-12, + abs_tol=1e-9, + ) + assert state_sum <= control_value * (1 + 1e-12) + for postal, _, _ in STATES[: len(districts)]: + children = [rows[f"{postal}_01"], rows[f"{postal}_02"]] + assert math.isclose( + math.fsum(spec.value for spec in children), + rows[f"{postal}_total"].value, + rel_tol=1e-12, + abs_tol=1e-9, + ) + for spec in children: + assert spec.metadata.get("hierarchy_reconciliation_factor") in ( + None, + "1", + ) + if with_ht2: + specs = _by_record(registry) + for (postal, _, _), value in zip(STATES, ht2_states, strict=False): + assert math.isclose( + specs[_ht2_state_id(measure_id, postal)].value, + value * control_value / ht2_us, + rel_tol=1e-12, + ) + quantity = fiscal_targets._soi_national_quantity(rows[f"{STATES[0][0]}_total"]) + assert [ + spec.metadata["ledger_source_record_id"] + for spec in _national_rows(registry, quantity) + ] == [_control_id(measure_id, control_period)] + + check() + + +def test_the_district_and_hts_passes_agree_on_the_same_shares() -> None: + """Differential: the HT2 pass (#1247) and the district pass implement one + share rule. Fed the same state values and US total under both families, + at the same period, they rebase each state to the same value with the + same factor.""" + pytest.importorskip("hypothesis") + from hypothesis import given, settings + from hypothesis import strategies as st + + @settings(max_examples=40, deadline=None) + @given( + st.sampled_from(CD_MEASURES), + st.sampled_from((2022, 2023)), + st.lists( + st.integers(min_value=1, max_value=10_000_000), min_size=1, max_size=3 + ), + st.integers(min_value=0, max_value=5_000_000), + st.integers(min_value=1, max_value=50_000_000), + ) + def check(measure_id, period, state_values, other_areas, control_value) -> None: + us = float(2 * sum(state_values) + other_areas) + # Each district-file state is two equal districts, so its total is + # the HT2 state value. + facts = [ + *_cd_facts( + measure_id, + stamp=period, + us=us, + districts=tuple((float(value), float(value)) for value in state_values), + ), + *_ht2_facts( + measure_id, + us=us, + states=tuple(2.0 * value for value in state_values), + period=period, + ), + _control_fact(measure_id, period=2023, value=float(control_value)), + ] + registry = _compile(facts) + specs = _by_record(registry) + rows = _cd_rows(registry, measure_id) + for postal, _, _ in STATES[: len(state_values)]: + ht2 = specs[_ht2_state_id(measure_id, postal, period)] + cd = rows[f"{postal}_total"] + assert cd.value == ht2.value + assert cd.metadata["uprating_factor"] == ht2.metadata["uprating_factor"] + assert ( + cd.metadata["uprating_to_period"] == ht2.metadata["uprating_to_period"] + ) + + check() diff --git a/tools/us_acs_local_cd_surface.py b/tools/us_acs_local_cd_surface.py index edd37055fa..d2dff4709c 100644 --- a/tools/us_acs_local_cd_surface.py +++ b/tools/us_acs_local_cd_surface.py @@ -86,7 +86,11 @@ #: counts bridge by ``return_count`` (counts are not aged), amounts by the #: closest amount Historic Table 2 carries. The factor band below applies to #: the sibling. The bridge is stamp-invariant: if the stamp is corrected -#: (#1030) both levels move together. +#: (#1030) both levels move together. A sibling the compiler binds as a share +#: of a national control (``return_count``, ``adjusted_gross_income`` and +#: ``itemized_deductions_amount`` are ``_SOI_HT2_NATIONAL_CONTROLS`` columns) +#: is read at the file's own level (:func:`cd_file_level`), so the bridge +#: still measures the file's gap and not the control's. STATE_CD_LEVEL_BRIDGES: Mapping[str, str] = { "charitable_amount": "itemized_deductions_amount", "charitable_returns": "return_count", @@ -196,6 +200,65 @@ def is_cd_file_spec(spec) -> bool: return record_set_spec_id(spec).startswith(CD_FILE_RECORD_SET_SPEC_PREFIX) +def cd_file_level(spec, *, aged_like=None) -> float: + """A district-file spec's value on the district file's own basis. + + The compiler binds some district-file rows as shares of a national + control: such a row names its denominator + (``soi_share_total_source_record_id``) and its ``uprating_factor`` is + control / the file's US row (microcosm#1229, #1040, and the + ``_SOI_HT2_NATIONAL_CONTROLS`` columns). Dividing the factor out gives the + file's own level: what a district block's shares, its receipt and a level + bridge read. Other specs return their value. + + A share ages from its control's period, and so does its file level. That + is the file's own aging while the file is stamped at the control's period + (the pinned feed); under #1030 the file ages from its data year, one link + earlier, so a block's recorded file values sit that link apart (its + shares, and so its bound values, do not move). With ``aged_like`` (an unrebased + district-file row), the level is re-aged to that row's aging factor, so + the two sit on one basis; they differ once #1030 ages the file from its + data year. That stands in for the share's own unrebased aging only when + both age on the same series, so a pair on different series refuses. + + Raises: + ValueError: ``aged_like`` ages on a different series from ``spec``. + """ + + metadata = _metadata(spec) + value = float(spec.value) + if not metadata.get("soi_share_total_source_record_id"): + return value + value /= float(metadata["uprating_factor"]) + if aged_like is not None: + if _aging_series(aged_like) != _aging_series(spec): + raise ValueError( + f"{spec.name} ages on {_aging_series(spec)!r} and " + f"{aged_like.name} on {_aging_series(aged_like)!r}; a share's " + "file level can be re-aged only like a row on its own series." + ) + value *= float(_metadata(aged_like).get("aging_factor", "1")) / float( + metadata.get("aging_factor", "1") + ) + return value + + +def _aging_series(spec) -> str: + """The growth series target aging ages ``spec`` on, from the spec alone. + + Read from the compiler's rule, not the receipt: a receipt can be a no-op + (``source_equals_build`` for a share landed at the build period) that + names no series. Rows that never age return why. + """ + + from microcosm.build.us_runtime import target_aging + + reason = target_aging._not_ageable_reason(spec) + if reason is not None: + return reason + return target_aging._cbo_income_source_for_spec(spec) + + def soi_materializer_semantics(spec) -> tuple: """Everything the SOI slice reads from a spec, except its geography. @@ -382,11 +445,12 @@ def drop(spec, reason: str) -> None: cd_file_state_by_key[key] = spec # Each state concept both files carry: its Historic Table 2 / district-file - # ratio, judged against the measure's median ratio across states. + # ratio (the file's own level, ``cd_file_level``), judged against the + # measure's median ratio across states. vintage_ratio: dict[tuple, float] = {} for key in ht2_by_key.keys() & cd_file_state_by_key.keys(): ht2_value = float(ht2_by_key[key].value) - cd_value = float(cd_file_state_by_key[key].value) + cd_value = cd_file_level(cd_file_state_by_key[key]) vintage_ratio[key] = ( ht2_value / cd_value if cd_value != 0.0 and (ht2_value > 0) == (cd_value > 0) @@ -445,11 +509,12 @@ def within_band( kept_cd_by_key[(state, soi_concept_identity(spec))].append(spec) # Each Historic-Table-2-parented block's rebase factor (parent over its - # district rows' sum), judged against its concept's median across states. + # district rows' sum at the file's level), judged against its concept's + # median across states. block_factor: dict[tuple, float] = {} for key, block in kept_cd_by_key.items(): parent = ht2_by_key.get(key) - child_sum = math.fsum(float(spec.value) for spec in block) + child_sum = math.fsum(cd_file_level(spec) for spec in block) if parent is not None and child_sum != 0.0: block_factor[key] = float(parent.value) / child_sum factors_by_identity: dict[tuple, list[float]] = defaultdict(list) @@ -480,12 +545,18 @@ def within_band( sibling_key = (state, (sibling, *identity[1:])) ht2_sibling = ht2_by_key.get(sibling_key) cd_sibling = cd_file_state_by_key.get(sibling_key) - if ht2_sibling is None or cd_sibling is None or float(cd_sibling.value) == 0: + # The bridged row is a district-file level, so the sibling must be + # too: a sibling the compiler bound as a national share is read at + # its own file level, aged as the bridged row is. + cd_sibling_level = ( + None if cd_sibling is None else cd_file_level(cd_sibling, aged_like=spec) + ) + if ht2_sibling is None or cd_sibling_level is None or cd_sibling_level == 0: raise ValueError( f"Level bridge {measure} -> {sibling} in state {state} needs " "the sibling's state total in both files." ) - factor = float(ht2_sibling.value) / float(cd_sibling.value) + factor = float(ht2_sibling.value) / cd_sibling_level # A sign conflict is a data defect, not a level disagreement: refuse # it before the band, which only judges finite positive ratios. if not (math.isfinite(factor) and factor > 0.0): @@ -495,11 +566,13 @@ def within_band( ) # The sibling is trusted for bridging only if both of the band's # verdicts on it pass: its two state levels, and (where it has district - # rows) the rebase of its own district block. + # rows) the rebase of its own district block. Re-aging scales every + # state's ratio alike, so the median is re-aged by the same constant. + reaging = factor / vintage_ratio[sibling_key] trusted = within_band( sibling_key, factor, - median_ratio.get(sibling_key[1], math.nan), + median_ratio.get(sibling_key[1], math.nan) * reaging, f"level_bridge:{measure}", measure, verdict="sibling_vintage_ratio", @@ -581,7 +654,11 @@ def within_band( f"State parent {parent.name} materializes differently from " f"its district rows ({identity}, state {state})." ) - child_sum = math.fsum(float(spec.value) for spec in children) + # Shares and the receipt read the district file's own levels, so a + # district row the compiler bound as a national share records the + # file's value, as every other district row does. + levels = {spec.name: cd_file_level(spec) for spec in children} + child_sum = math.fsum(levels.values()) parent_value = float(parent.value) if child_sum == 0.0: if parent_value != 0.0: @@ -601,7 +678,7 @@ def within_band( # The district file must agree with itself before its shares are used. cd_state = cd_file_state_by_key.get(key) if cd_state is not None: - cd_state_value = float(cd_state.value) + cd_state_value = cd_file_level(cd_state) gap = abs(cd_state_value - child_sum) internal_gaps.append(gap / max(abs(cd_state_value), 1.0)) if gap > STATE_CD_INTERNAL_RTOL * max(abs(cd_state_value), 1.0): @@ -637,12 +714,12 @@ def within_band( "state_cd_parent_target_name": parent.name, "state_cd_parent_basis": basis, "state_cd_parent_value": repr(parent_value), - "state_cd_cd_file_value": repr(float(child.value)), + "state_cd_cd_file_value": repr(levels[child.name]), "state_cd_cd_file_state_sum": repr(child_sum), "state_cd_rebase_factor": repr(factor), } ) - value = float(child.value) * factor + value = levels[child.name] * factor rebased[child.name] = replace( child, value=value,