From a90ad379b08f0eaf6a84b0df1a97e4036541431e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 10:29:47 -0400 Subject: [PATCH 1/4] Read IRS SOI CD and HT2 measures from the columns their guides define The IRS documentation guides define each geographic-file column. Four mappings read a column the guide defines as something else: - congressional_district_2022 limited_state_local_taxes_* read N18425/A18425 (state and local income taxes, Schedule A line 5a); now N18460/A18460 (limited state and local taxes, line 5e), as Historic Table 2 reads. - premium_tax_credit_* read N85530/A85530 (additional Medicare tax, Form 8959 line 24); now N85770/A85770 (total premium tax credit). - interest_paid_deduction_* (Table 2.1's line 10 total) read N19300/A19300, line 8a mortgage interest. Renamed to Table 2.1's line 8a ids and concepts (mortgage_interest_paid_*), with new line 8b, 8c and 9 measures. - historic_table_2 and historic_table_2_state_broad_2022 qbi_* read N03270/A03270 (self-employed health insurance); now N04475/A04475. tests/test_chronicle_soi_doc_guide_columns.py pins every measure id these packages declare to its guide definition (extracts in tests/data/irs_soi_doc_guides), with differential checks against Historic Table 2 and Table 2.1 and the $10,000 line 5e cap. Co-Authored-By: Claude Opus 5.5 --- .../irs-soi-doc-guide-columns.fixed.md | 9 + docs/concept-migrations.md | 71 ++ .../source_package.yaml | 105 ++- .../historic_table_2/source_package.yaml | 12 +- .../source_package.yaml | 12 +- .../irs_soi_doc_guides/22incddocguide.tsv | 153 ++++ .../irs_soi_doc_guides/22incmdocguide.tsv | 153 ++++ .../irs_soi_doc_guides/22incydocguide.tsv | 153 ++++ .../irs_soi_doc_guides/23incmdocguide.tsv | 155 ++++ tests/test_chronicle_bundle.py | 27 +- tests/test_chronicle_soi_doc_guide_columns.py | 707 ++++++++++++++++++ tests/test_chronicle_source_package.py | 37 +- 12 files changed, 1552 insertions(+), 42 deletions(-) create mode 100644 changelog.d/irs-soi-doc-guide-columns.fixed.md create mode 100644 tests/data/irs_soi_doc_guides/22incddocguide.tsv create mode 100644 tests/data/irs_soi_doc_guides/22incmdocguide.tsv create mode 100644 tests/data/irs_soi_doc_guides/22incydocguide.tsv create mode 100644 tests/data/irs_soi_doc_guides/23incmdocguide.tsv create mode 100644 tests/test_chronicle_soi_doc_guide_columns.py diff --git a/changelog.d/irs-soi-doc-guide-columns.fixed.md b/changelog.d/irs-soi-doc-guide-columns.fixed.md new file mode 100644 index 00000000..c163a8d0 --- /dev/null +++ b/changelog.d/irs-soi-doc-guide-columns.fixed.md @@ -0,0 +1,9 @@ +Read four IRS SOI measures from the columns the IRS documentation guides +define. In the congressional-district package, limited SALT now reads +`N18460`/`A18460` (Schedule A line 5e), not 5a's `N18425`/`A18425`. The +premium tax credit now reads `N85770`/`A85770`, not the additional Medicare +tax `N85530`/`A85530`. The line 8a mortgage-interest columns are renamed from +`interest_paid_deduction_*` to Table 2.1's `mortgage_interest_paid_*`, next to +new line 8b, 8c and 9 measures. Historic Table 2 `qbi_*` now reads the QBI +deduction `N04475`/`A04475`, not `N03270`/`A03270`. A regression test pins +every measure id in these packages to its guide definition. diff --git a/docs/concept-migrations.md b/docs/concept-migrations.md index ed4b43c5..4d54d58c 100644 --- a/docs/concept-migrations.md +++ b/docs/concept-migrations.md @@ -74,3 +74,74 @@ and the congressional-district file still label with different concepts. concept, only Table 1.4 may declare the Schedule D gain concept, the retired ids may not reappear, and the consumer selector (`source_measure_id`) and published values are unchanged. + +## 2026-10-12: IRS SOI geographic-file columns follow their documentation guides + +Four IRS SOI mappings read a column that the file's documentation guide defines +as a different item. Extracts of the guides are in +`tests/data/irs_soi_doc_guides/`, each with its irs.gov URL and sha256. + +### Column corrections + +Measure ids, concepts and labels stay. Values, `layout.source_column_id` and +the cell lineage change. + +| Packages | Measure ids | Old columns | New columns | TY2022 US, old → new | +|---|---|---|---|---| +| `soi-congressional-district-2022` | `limited_state_local_taxes_returns` / `_amount` | `N18425`/`A18425`, "State and local income taxes" (Schedule A line 5a) | `N18460`/`A18460`, "Limited state and local taxes" (line 5e) | 10,842,580 → 14,777,040 returns; $250.4B → $121.2B | +| same | `premium_tax_credit_returns` / `_amount` | `N85530`/`A85530`, "Additional Medicare tax" (Form 8959 line 24) | `N85770`/`A85770`, "Total premium tax credit" | 6,848,330 → 7,713,000 returns; $14.2B → $53.1B | +| `soi-historic-table-2`, `soi-historic-table-2-state-broad-2022` | `qbi_claims` / `qbi_amount` | `N03270`/`A03270`, "Self-employment health insurance deduction" (Schedule 1 line 17) | `N04475`/`A04475`, "Qualified business income deduction" (Form 1040 line 13) | 3,711,990 → 25,430,690 returns; $31.3B → $215.7B | + +Historic Table 2 already read `N18460` and `N85770` for the same measure ids. +The TY2022 guides print the premium tax credit's form as "8926"; the TY2023 +Historic Table 2 guide prints 8962 for the same variables. + +### Schedule A line 8a relabel + +| Packages | Columns | Old measure id / concept | New measure id / concept | +|---|---|---|---| +| `soi-congressional-district-2022` | `N19300` | `interest_paid_deduction_returns` / `irs_soi.returns_with_interest_paid_deduction` | `mortgage_interest_paid_returns` / `irs_soi.returns_with_home_mortgage_interest_paid_to_financial_institutions` | +| same | `A19300` | `interest_paid_deduction_amount` / `irs_soi.interest_paid_deduction` | `mortgage_interest_paid_amount` / `irs_soi.home_mortgage_interest_paid_to_financial_institutions` | + +Labels become Table 2.1's: "Returns with home mortgage interest paid to +financial institutions" and "Home mortgage interest paid to financial +institutions amount". Unlike the 2026-09-27 rename, `measure_id` and +`source_record_id` move too, so the consumer selector moves. + +**Why.** The guide defines `N19300`/`A19300` as mortgage interest paid, +Schedule A line 8a. `interest_paid_deduction_*` is Table 2.1's line 10 total, +and the 2022 Schedule A computes line 10 as lines 8a, 8b, 8c and 9 combined +(line 8d is reserved). No geographic file publishes line 10. For TY2022 the +district file's US `A19300` is $140.1B, 0.984 of Table 2.1's line 8a and 0.822 +of its line 10 ($170.5B). + +**Added.** The line 8b, 8c and 9 columns, with Table 2.1's ids, concepts and +labels: `home_mortgage_personal_seller_*` (`N19500`/`A19500`), +`deductible_points_*` (`N19530`/`A19530`) and `investment_interest_paid_*` +(`N19570`/`A19570`). The four amounts sum to line 10: $163.7B for the TY2022 +US row, 0.960 of Table 2.1's. Return counts do not add, because one return can +report more than one line. + +**Consumer effect.** Microcosm's SOI compiler selects facts by `measure_id` and +skips ids it does not map. On its next Chronicle feed re-pin: + +- The district line 8a rows stop mapping to `interest_deduction`. Microcosm + leaves `mortgage_interest_paid_*` unmapped on purpose, so those district + targets drop, and its `state_cd` bridge entries for `interest_paid_deduction_*` + stop matching. The 8b, 8c and 9 ids are unmapped too. Whether to build a + district line 10 target from the four parts is Microcosm's decision. +- The district SALT and PTC rows carry the corrected columns. + microcosm#1224's register keyed on `N18425`/`A18425` and `N85530`/`A85530` + stops matching them, and its pinned receipt test needs updating. +- Microcosm maps no Historic Table 2 `qbi_*` measure, so the QBI fix moves no + Microcosm target. + +**Pinned by** `tests/test_chronicle_soi_doc_guide_columns.py`. It maps every +`measure_id` declared by a package that reads an IRS-variable file (Historic +Table 2, congressional-district or county, any vintage) to one guide +definition, and checks each declaration reads the column that definition names. +Measure ids shared across packages must read one column. The line 10 ids may +not be read from a geographic file. Two differential checks compare published +values: the district US row against Historic Table 2's, and the line 8a column +and four-part sum against Table 2.1. A property check holds every limited SALT +row to the $10,000 line 5e cap. diff --git a/packages/irs_soi/congressional_district_2022/source_package.yaml b/packages/irs_soi/congressional_district_2022/source_package.yaml index b53fed9b..540610b7 100644 --- a/packages/irs_soi/congressional_district_2022/source_package.yaml +++ b/packages/irs_soi/congressional_district_2022/source_package.yaml @@ -14322,10 +14322,10 @@ record_sets: - measure_id: limited_state_local_taxes_returns label: Returns with limited state and local taxes ordinal: 30 - column: BX - source_column_id: N18425 + column: CF + source_column_id: N18460 expected_column_header_row: 1 - expected_column_header: N18425 + expected_column_header: N18460 concept: irs_soi.returns_with_limited_state_local_taxes unit: count aggregation: sum @@ -14333,10 +14333,10 @@ record_sets: - measure_id: limited_state_local_taxes_amount label: Limited state and local taxes ordinal: 31 - column: BY - source_column_id: A18425 + column: CG + source_column_id: A18460 expected_column_header_row: 1 - expected_column_header: A18425 + expected_column_header: A18460 concept: irs_soi.limited_state_local_taxes unit: usd aggregation: sum @@ -14388,25 +14388,25 @@ record_sets: aggregation: sum value_scale: 1000 expected_cell_type: number - - measure_id: interest_paid_deduction_returns - label: Returns with deductible interest paid + - measure_id: mortgage_interest_paid_returns + label: Returns with home mortgage interest paid to financial institutions ordinal: 36 column: CJ source_column_id: N19300 expected_column_header_row: 1 expected_column_header: N19300 - concept: irs_soi.returns_with_interest_paid_deduction + concept: irs_soi.returns_with_home_mortgage_interest_paid_to_financial_institutions unit: count aggregation: sum expected_cell_type: number - - measure_id: interest_paid_deduction_amount - label: Deductible interest paid + - measure_id: mortgage_interest_paid_amount + label: Home mortgage interest paid to financial institutions amount ordinal: 37 column: CK source_column_id: A19300 expected_column_header_row: 1 expected_column_header: A19300 - concept: irs_soi.interest_paid_deduction + concept: irs_soi.home_mortgage_interest_paid_to_financial_institutions unit: usd aggregation: sum value_scale: 1000 @@ -14662,10 +14662,10 @@ record_sets: - measure_id: premium_tax_credit_returns label: Returns with premium tax credit ordinal: 54 - column: EX - source_column_id: N85530 + column: DV + source_column_id: N85770 expected_column_header_row: 1 - expected_column_header: N85530 + expected_column_header: N85770 concept: irs_soi.returns_with_premium_tax_credit unit: count aggregation: sum @@ -14673,12 +14673,81 @@ record_sets: - measure_id: premium_tax_credit_amount label: Premium tax credit ordinal: 55 - column: EY - source_column_id: A85530 + column: DW + source_column_id: A85770 expected_column_header_row: 1 - expected_column_header: A85530 + expected_column_header: A85770 concept: irs_soi.premium_tax_credit unit: usd aggregation: sum value_scale: 1000 expected_cell_type: number + - measure_id: home_mortgage_personal_seller_returns + label: Returns with home mortgage interest paid to individuals + ordinal: 56 + column: CL + source_column_id: N19500 + expected_column_header_row: 1 + expected_column_header: N19500 + concept: irs_soi.returns_with_home_mortgage_interest_paid_to_individuals + unit: count + aggregation: sum + expected_cell_type: number + - measure_id: home_mortgage_personal_seller_amount + label: Home mortgage interest paid to individuals amount + ordinal: 57 + column: CM + source_column_id: A19500 + expected_column_header_row: 1 + expected_column_header: A19500 + concept: irs_soi.home_mortgage_interest_paid_to_individuals + unit: usd + aggregation: sum + value_scale: 1000 + expected_cell_type: number + - measure_id: deductible_points_returns + label: Returns with deductible points + ordinal: 58 + column: CN + source_column_id: N19530 + expected_column_header_row: 1 + expected_column_header: N19530 + concept: irs_soi.returns_with_deductible_points + unit: count + aggregation: sum + expected_cell_type: number + - measure_id: deductible_points_amount + label: Deductible points amount + ordinal: 59 + column: CO + source_column_id: A19530 + expected_column_header_row: 1 + expected_column_header: A19530 + concept: irs_soi.deductible_points + unit: usd + aggregation: sum + value_scale: 1000 + expected_cell_type: number + - measure_id: investment_interest_paid_returns + label: Returns with investment interest expense deduction + ordinal: 60 + column: CP + source_column_id: N19570 + expected_column_header_row: 1 + expected_column_header: N19570 + concept: irs_soi.returns_with_investment_interest_expense_deduction + unit: count + aggregation: sum + expected_cell_type: number + - measure_id: investment_interest_paid_amount + label: Investment interest expense deduction amount + ordinal: 61 + column: CQ + source_column_id: A19570 + expected_column_header_row: 1 + expected_column_header: A19570 + concept: irs_soi.investment_interest_expense_deduction + unit: usd + aggregation: sum + value_scale: 1000 + expected_cell_type: number diff --git a/packages/irs_soi/historic_table_2/source_package.yaml b/packages/irs_soi/historic_table_2/source_package.yaml index fee51ec4..bfb5cfe8 100644 --- a/packages/irs_soi/historic_table_2/source_package.yaml +++ b/packages/irs_soi/historic_table_2/source_package.yaml @@ -810,25 +810,25 @@ record_sets: - measure_id: qbi_claims label: Returns with qualified business income deduction ordinal: 47 - column: BF - source_column_id: N03270 + column: CT + source_column_id: N04475 concept: irs_soi.returns_with_qualified_business_income_deduction unit: count aggregation: sum expected_cell_type: number expected_column_header_row: 1 - expected_column_header: N03270 + expected_column_header: N04475 - measure_id: qbi_amount label: Qualified business income deduction ordinal: 48 - column: BG - source_column_id: A03270 + column: CU + source_column_id: A04475 concept: irs_soi.qualified_business_income_deduction unit: usd aggregation: sum expected_cell_type: number expected_column_header_row: 1 - expected_column_header: A03270 + expected_column_header: A04475 value_scale: 1000 - measure_id: rental_royalty_income_returns label: Returns with rental and royalty net income less loss diff --git a/packages/irs_soi/historic_table_2_state_broad_2022/source_package.yaml b/packages/irs_soi/historic_table_2_state_broad_2022/source_package.yaml index dea8999e..c0f1cfdb 100644 --- a/packages/irs_soi/historic_table_2_state_broad_2022/source_package.yaml +++ b/packages/irs_soi/historic_table_2_state_broad_2022/source_package.yaml @@ -692,25 +692,25 @@ record_sets: - measure_id: qbi_claims label: Returns with qualified business income deduction ordinal: 45 - column: BF - source_column_id: N03270 + column: CT + source_column_id: N04475 concept: irs_soi.returns_with_qualified_business_income_deduction unit: count aggregation: sum expected_cell_type: number expected_column_header_row: 1 - expected_column_header: N03270 + expected_column_header: N04475 - measure_id: qbi_amount label: Qualified business income deduction ordinal: 46 - column: BG - source_column_id: A03270 + column: CU + source_column_id: A04475 concept: irs_soi.qualified_business_income_deduction unit: usd aggregation: sum expected_cell_type: number expected_column_header_row: 1 - expected_column_header: A03270 + expected_column_header: A04475 value_scale: 1000 - measure_id: rental_royalty_income_returns label: Returns with rental and royalty net income less loss diff --git a/tests/data/irs_soi_doc_guides/22incddocguide.tsv b/tests/data/irs_soi_doc_guides/22incddocguide.tsv new file mode 100644 index 00000000..4f2881a9 --- /dev/null +++ b/tests/data/irs_soi_doc_guides/22incddocguide.tsv @@ -0,0 +1,153 @@ +# IRS SOI Congressional District Data, Tax Year 2022, Documentation Guide, section G (Selected Income and Tax Items). +# Source: https://www.irs.gov/pub/irs-soi/22incddocguide.docx +# sha256: 4e265d3c2499d87428e01bfe2c9e581b444cb6289c7a45397cbfc14a681636ef +# Retrieved 2026-10-12. Documents the columns of 22incd.csv. +# Extracted with macOS textutil -convert txt; one row per N/A variable. +# Endnote markers such as [4] are dropped; wording, capitalization and +# line references are the guide's own, typos included. +variable description line_reference +N1 Number of returns +N2 Number of individuals +A00100 Adjust gross income (AGI) 1040:11 +N02650 Number of returns with total income 1040:9 +A02650 Total income amount 1040:9 +N00200 Number of returns with salaries and wages 1040:1z +A00200 Salaries and wages amount 1040:1z +N00300 Number of returns with taxable interest 1040:2b +A00300 Taxable interest amount 1040:2b +N00400 Number of returns tax-exempt interest 1040:2a +A00400 Tax-exempt interest amount 1040:2a +N00600 Number of returns with ordinary dividends 1040:3b +A00600 Ordinary dividends amount 1040:3b +N00650 Number of returns with qualified dividends 1040:3a +A00650 Qualified dividends amount 1040:3a +N00700 Number of returns with state and local income tax refunds Schedule 1:1 +A00700 State and local income tax refunds amount Schedule 1:1 +N00900 Number of returns with business or professional net income (less loss) Schedule 1:3 +A00900 Business or professional net income (less loss) amount Schedule 1:3 +N01000 Number of returns with net capital gain (less loss) 1040:7 +A01000 Net capital gain (less loss) amount 1040:7 +N01400 Number of returns with taxable individual retirement arrangements distributions 1040:4b +A01400 Taxable individual retirement arrangements distributions amount 1040:4b +N01700 Number of returns with taxable pensions and annuities 1040:5b +A01700 Taxable pensions and annuities amount 1040:5b +N02300 Number of returns with unemployment compensation Schedule 1:7 +A02300 Unemployment compensation amount Schedule 1:7 +N02500 Number of returns with taxable Social Security benefits 1040:6b +A02500 Taxable Social Security benefits amount 1040:6b +N26270 Number of returns with partnership/S-corp net income (less loss) Schedule E:32 +A26270 Partnership/S-corp net income (less loss) amount Schedule E:32 +N25870 Number of returns with total rental and royalty net income (less loss) Schedule E:26 +A25870 Total rental and royalty net income (less loss) Schedule E:26 +N02900 Number of returns with total statutory adjustments Schedule 1:26 +A02900 Total statutory adjustments amount Schedule 1:26 +N03220 Number of returns with educator expenses Schedule 1:11 +A03220 Educator expenses amount Schedule 1:11 +N03300 Number of returns with self-employment retirement plans Schedule 1:16 +A03300 Self-employment retirement plans amount Schedule 1:16 +N03270 Number of returns with self-employment health insurance deduction Schedule 1:17 +A03270 Self-employment health insurance deduction amount Schedule 1:17 +N03150 Number of returns with IRA payments Schedule 1:20 +A03150 IRA payments amount Schedule 1:20 +N03210 Number of returns with student loan interest deduction Schedule 1:21 +A03210 Student loan interest deduction amount Schedule 1:21 +N04450 Number of returns with total standard deduction 1040:12 +A04450 Total standard deduction amount 1040:12 +N04100 Number of returns with basic standard deduction 1040:12 +A04100 Basic standard deduction amount 1040:12 +N04200 Number of returns with additional standard deduction 1040:12 +A04200 Additional standard deduction amount 1040:12 +N04470 Number of returns with itemized deductions 1040:12 +A04470 Total itemized deductions amount 1040:12 +N17000 Number of returns with Total medical and dental expense deduction Schedule A:4 +A17000 Total medical and dental expense deduction amount Schedule A:4 +N18425 Number of returns with State and local income taxes Schedule A:5a +A18425 State and local income taxes amount Schedule A:5a +N18450 Number of returns with State and local general sales tax Schedule A:5a +A18450 State and local general sales tax amount Schedule A:5a +N18500 Number of returns with real estate taxes Schedule A:5b +A18500 Real estate taxes amount Schedule A:5b +N18800 Number of returns with Personal property taxes Schedule A:5c +A18800 Personal property taxes amount Schedule A:5c +N18460 Number of returns with Limited state and local taxes Schedule A:5e +A18460 Limited state and local taxes Schedule A:5e +N18300 Number of returns with taxes paid Schedule A:7 +A18300 Taxes paid amount Schedule A:7 +N19300 Number of returns with mortgage interest paid Schedule A:8a +A19300 Mortgage interest paid amount Schedule A:8a +N19500 Number of returns with Home mortgage from personal seller Schedule A:8b +A19500 Home mortgage from personal seller amount Schedule A:8b +N19530 Number of returns with Deductible points Schedule A:8c +A19530 Deductible points amount Schedule A:8c +N19570 Number of returns with Investment interest paid Schedule A:9 +A19570 Investment interest paid amount Schedule A:9 +N19700 Number of returns with Total charitable contributions Schedule A:14 +A19700 Total charitable contributions amount Schedule A:14 +N20950 Number of returns with Other non-limited miscellaneous deductions Schedule A:16 +A20950 Other non-limited miscellaneous deductions amount Schedule A:16 +N04475 Number of returns with Qualified business income deduction 1040:13 +A04475 Qualified business income deduction amount 1040:13 +N04800 Number of returns with taxable income 1040:15 +A04800 Taxable income amount 1040:15 +N05800 Number of returns with income tax before credits 1040:16 +A05800 Income tax before credits amount 1040:16 +N09600 Number of returns with alternative minimum tax Schedule 2:1 +A09600 Alternative minimum tax amount Schedule 2:1 +N05780 Number of returns with excess advance premium tax credit repayment Schedule 2:2 +A05780 Excess advance premium tax credit repayment amount Schedule 2:2 +N07100 Number of returns with total tax credits 1040:21 +A07100 Total tax credits amount 1040:21 +N07300 Number of returns with foreign tax credit Schedule 3:1 +A07300 Foreign tax credit amount Schedule 3:1 +N07180 Number of returns with child and dependent care credit Schedule 3:2 +A07180 Child and dependent care credit amount Schedule 3:2 +N07230 Number of returns with nonrefundable education credit Schedule 3:3 +A07230 Nonrefundable education credit amount Schedule 3:3 +N07240 Number of returns with retirement savings contribution credit Schedule 3:4 +A07240 Retirement savings contribution credit amount Schedule 3:4 +N07225 Number of returns with nonrefundable child and other dependent tax credit 1040:19 +A07225 Nonrefundable child and other dependent tax credit amount 1040:19 +N07260 Number of returns with residential energy tax credit Schedule 3:5 +A07260 Residential energy tax credit amount Schedule 3:5 +N09400 Number of returns with self-employment tax Schedule 2:4 +A09400 Self-employment tax amount Schedule 2:4 +N85770 Number of returns with total premium tax credit 8926:24 +A85770 Total premium tax credit amount 8926:24 +N85775 Number of returns with advance premium tax credit 8926:25 +A85775 Advance premium tax credit amount 8926:25 +N10600 Number of returns with total tax payments 1040:33 +A10600 Total tax payments amount 1040:33 +N59660 Number of returns with earned income credit 1040:27 +A59660 Earned income credit amount 1040:27 +N59661 Number of returns with earned income credit with no qualifying children 1040:27 +A59661 Earned income credit with no qualifying children amount 1040:27 +N59662 Number of returns with earned income credit with one qualifying child 1040:27 +A59662 Earned income credit with one qualifying child amount 1040:27 +N59663 Number of returns with earned income credit with two qualifying children 1040:27 +A59663 Earned income credit with two qualifying children amount 1040:27 +N59664 Number of returns with earned income credit with three qualifying children 1040:27 +A59664 Earned income credit with three qualifying children amount 1040:27 +N59720 Number of returns with excess earned income credit 1040:27 +A59720 Excess earned income credit (refundable) amount 1040:27 +N11070 Number of returns with refundable child tax credit or additional child tax credit 1040:28 +A11070 Refundable child tax credit or additional child tax credit amount 1040:28 +N10960 Number of returns with refundable education credit 1040:29 +A10960 Refundable education credit amount 1040:29 +N11560 Number of returns with net premium tax credit Schedule 3:9 +A11560 Net premium tax credit amount Schedule 3:9 +N06500 Number of returns with income tax after credits 1040:22 +A06500 Income tax after credits amount 1040:22 +N10300 Number of returns with tax liability 1040:24 +A10300 Total tax liability amount 1040:24 +N85530 Number of returns with additional Medicare tax 8959:24 +A85530 Additional Medicare tax 8959:24 +N85300 Number of returns with net investment income tax 8960:17 +A85300 Net investment income tax 8960:17 +N11901 Number of returns with tax due at time of filing 1040:37 +A11901 Tax due at time of filing amount 1040:37 +N11900 Number of returns with total overpayments 1040:34 +A11900 Total overpayments amount 1040:34 +N11902 Number of returns with overpayments refunded 1040:35a +A11902 Overpayments refunded amount 1040:35a +N12000 Number of returns with credit to next year’s estimated tax 1040:36 +A12000 Credited to next year’s estimated tax amount 1040:36 diff --git a/tests/data/irs_soi_doc_guides/22incmdocguide.tsv b/tests/data/irs_soi_doc_guides/22incmdocguide.tsv new file mode 100644 index 00000000..31262aab --- /dev/null +++ b/tests/data/irs_soi_doc_guides/22incmdocguide.tsv @@ -0,0 +1,153 @@ +# IRS SOI Historic Table 2 (State Data), Tax Year 2022, Documentation Guide, section G (Selected Income and Tax Items). +# Source: https://www.irs.gov/pub/irs-soi/22incmdocguide.doc +# sha256: d99db44bc6ca75302dfcf2a0b14a31eb0995709de2e7b39e30e34e306a4c9fa8 +# Retrieved 2026-10-12. Documents the columns of 22in55cmcsv.csv. +# Extracted with macOS textutil -convert txt; one row per N/A variable. +# Endnote markers such as [4] are dropped; wording, capitalization and +# line references are the guide's own, typos included. +variable description line_reference +N1 Number of returns +N2 Number of individuals +A00100 Adjust gross income (AGI) 1040:11 +N02650 Number of returns with total income 1040:9 +A02650 Total income amount 1040:9 +N00200 Number of returns with salaries and wages 1040:1z +A00200 Salaries and wages amount 1040:1z +N00300 Number of returns with taxable interest 1040:2b +A00300 Taxable interest amount 1040:2b +N00400 Number of returns tax-exempt interest 1040:2a +A00400 Tax-exempt interest amount 1040:2a +N00600 Number of returns with ordinary dividends 1040:3b +A00600 Ordinary dividends amount 1040:3b +N00650 Number of returns with qualified dividends 1040:3a +A00650 Qualified dividends amount 1040:3a +N00700 Number of returns with state and local income tax refunds Schedule 1:1 +A00700 State and local income tax refunds amount Schedule 1:1 +N00900 Number of returns with business or professional net income (less loss) Schedule 1:3 +A00900 Business or professional net income (less loss) amount Schedule 1:3 +N01000 Number of returns with net capital gain (less loss) 1040:7 +A01000 Net capital gain (less loss) amount 1040:7 +N01400 Number of returns with taxable individual retirement arrangements distributions 1040:4b +A01400 Taxable individual retirement arrangements distributions amount 1040:4b +N01700 Number of returns with taxable pensions and annuities 1040:5b +A01700 Taxable pensions and annuities amount 1040:5b +N02300 Number of returns with unemployment compensation Schedule 1:7 +A02300 Unemployment compensation amount Schedule 1:7 +N02500 Number of returns with taxable Social Security benefits 1040:6b +A02500 Taxable Social Security benefits amount 1040:6b +N26270 Number of returns with partnership/S-corp net income (less loss) Schedule E:32 +A26270 Partnership/S-corp net income (less loss) amount Schedule E:32 +N25870 Number of returns with total rental and royalty net income (less loss) Schedule E:26 +A25870 Total rental and royalty net income (less loss) Schedule E:26 +N02900 Number of returns with total statutory adjustments Schedule 1:26 +A02900 Total statutory adjustments amount Schedule 1:26 +N03220 Number of returns with educator expenses Schedule 1:11 +A03220 Educator expenses amount Schedule 1:11 +N03300 Number of returns with self-employment retirement plans Schedule 1:16 +A03300 Self-employment retirement plans amount Schedule 1:16 +N03270 Number of returns with self-employment health insurance deduction Schedule 1:17 +A03270 Self-employment health insurance deduction amount Schedule 1:17 +N03150 Number of returns with IRA payments Schedule 1:20 +A03150 IRA payments amount Schedule 1:20 +N03210 Number of returns with student loan interest deduction Schedule 1:21 +A03210 Student loan interest deduction amount Schedule 1:21 +N04450 Number of returns with total standard deduction 1040:12 +A04450 Total standard deduction amount 1040:12 +N04100 Number of returns with basic standard deduction 1040:12 +A04100 Basic standard deduction amount 1040:12 +N04200 Number of returns with additional standard deduction 1040:12 +A04200 Additional standard deduction amount 1040:12 +N04470 Number of returns with itemized deductions 1040:12 +A04470 Total itemized deductions amount 1040:12 +N17000 Number of returns with Total medical and dental expense deduction Schedule A:4 +A17000 Total medical and dental expense deduction amount Schedule A:4 +N18425 Number of returns with State and local income taxes Schedule A:5a +A18425 State and local income taxes amount Schedule A:5a +N18450 Number of returns with State and local general sales tax Schedule A:5a +A18450 State and local general sales tax amount Schedule A:5a +N18500 Number of returns with real estate taxes Schedule A:5b +A18500 Real estate taxes amount Schedule A:5b +N18800 Number of returns with Personal property taxes Schedule A:5c +A18800 Personal property taxes amount Schedule A:5c +N18460 Number of returns with Limited state and local taxes Schedule A:5e +A18460 Limited state and local taxes Schedule A:5e +N18300 Number of returns with taxes paid Schedule A:7 +A18300 Taxes paid amount Schedule A:7 +N19300 Number of returns with mortgage interest paid Schedule A:8a +A19300 Mortgage interest paid amount Schedule A:8a +N19500 Number of returns with Home mortgage from personal seller Schedule A:8b +A19500 Home mortgage from personal seller amount Schedule A:8b +N19530 Number of returns with Deductible points Schedule A:8c +A19530 Deductible points amount Schedule A:8c +N19570 Number of returns with Investment interest paid Schedule A:9 +A19570 Investment interest paid amount Schedule A:9 +N19700 Number of returns with Total charitable contributions Schedule A:14 +A19700 Total charitable contributions amount Schedule A:14 +N20950 Number of returns with Other non-limited miscellaneous deductions Schedule A:16 +A20950 Other non-limited miscellaneous deductions amount Schedule A:16 +N04475 Number of returns with Qualified business income deduction 1040:13 +A04475 Qualified business income deduction amount 1040:13 +N04800 Number of returns with taxable income 1040:15 +A04800 Taxable income amount 1040:15 +N05800 Number of returns with income tax before credits 1040:16 +A05800 Income tax before credits amount 1040:16 +N09600 Number of returns with alternative minimum tax Schedule 2:1 +A09600 Alternative minimum tax amount Schedule 2:1 +N05780 Number of returns with excess advance premium tax credit repayment Schedule 2:2 +A05780 Excess advance premium tax credit repayment amount Schedule 2:2 +N07100 Number of returns with total tax credits 1040:21 +A07100 Total tax credits amount 1040:21 +N07300 Number of returns with foreign tax credit Schedule 3:1 +A07300 Foreign tax credit amount Schedule 3:1 +N07180 Number of returns with child and dependent care credit Schedule 3:2 +A07180 Child and dependent care credit amount Schedule 3:2 +N07230 Number of returns with nonrefundable education credit Schedule 3:3 +A07230 Nonrefundable education credit amount Schedule 3:3 +N07240 Number of returns with retirement savings contribution credit Schedule 3:4 +A07240 Retirement savings contribution credit amount Schedule 3:4 +N07225 Number of returns with nonrefundable child and other dependent tax credit 1040:19 +A07225 Nonrefundable child and other dependent tax credit amount 1040:19 +N07260 Number of returns with residential energy tax credit Schedule 3:5 +A07260 Residential energy tax credit amount Schedule 3:5 +N09400 Number of returns with self-employment tax Schedule 2:4 +A09400 Self-employment tax amount Schedule 2:4 +N85770 Number of returns with total premium tax credit 8926:24 +A85770 Total premium tax credit amount 8926:24 +N85775 Number of returns with advance premium tax credit 8926:25 +A85775 Advance premium tax credit amount 8926:25 +N10600 Number of returns with total tax payments 1040:33 +A10600 Total tax payments amount 1040:33 +N59660 Number of returns with earned income credit 1040:27 +A59660 Earned income credit amount 1040:27 +N59661 Number of returns with earned income credit with no qualifying children 1040:27 +A59661 Earned income credit with no qualifying children amount 1040:27 +N59662 Number of returns with earned income credit with one qualifying child 1040:27 +A59662 Earned income credit with one qualifying child amount 1040:27 +N59663 Number of returns with earned income credit with two qualifying children 1040:27 +A59663 Earned income credit with two qualifying children amount 1040:27 +N59664 Number of returns with earned income credit with three qualifying children 1040:27 +A59664 Earned income credit with three qualifying children amount 1040:27 +N59720 Number of returns with excess earned income credit 1040:27 +A59720 Excess earned income credit (refundable) amount 1040:27 +N11070 Number of returns with refundable child tax credit or additional child tax credit 1040:28 +A11070 Refundable child tax credit or additional child tax credit amount 1040:28 +N10960 Number of returns with refundable education credit 1040:29 +A10960 Refundable education credit amount 1040:29 +N11560 Number of returns with net premium tax credit Schedule 3:9 +A11560 Net premium tax credit amount Schedule 3:9 +N06500 Number of returns with income tax after credits 1040:22 +A06500 Income tax after credits amount 1040:22 +N10300 Number of returns with tax liability 1040:24 +A10300 Total tax liability amount 1040:24 +N85530 Number of returns with additional Medicare tax 8959:24 +A85530 Additional Medicare tax 8959:24 +N85300 Number of returns with net investment income tax 8960:17 +A85300 Net investment income tax 8960:17 +N11901 Number of returns with tax due at time of filing 1040:37 +A11901 Tax due at time of filing amount 1040:37 +N11900 Number of returns with total overpayments 1040:34 +A11900 Total overpayments amount 1040:34 +N11902 Number of returns with overpayments refunded 1040:35a +A11902 Overpayments refunded amount 1040:35a +N12000 Number of returns with credit to next year’s estimated tax 1040:36 +A12000 Credited to next year’s estimated tax amount 1040:36 diff --git a/tests/data/irs_soi_doc_guides/22incydocguide.tsv b/tests/data/irs_soi_doc_guides/22incydocguide.tsv new file mode 100644 index 00000000..0b6a3d88 --- /dev/null +++ b/tests/data/irs_soi_doc_guides/22incydocguide.tsv @@ -0,0 +1,153 @@ +# IRS SOI County Data, Tax Year 2022, Documentation Guide, section G (Selected Income and Tax Items). +# Source: https://www.irs.gov/pub/irs-soi/22incydocguide.docx +# sha256: a6acb748d4155614ee4a95d3eb55b7f3d07bf010be269d09d9f06781fbc48787 +# Retrieved 2026-10-12. Documents the columns of 22incyallnoagi.csv. +# Extracted with macOS textutil -convert txt; one row per N/A variable. +# Endnote markers such as [4] are dropped; wording, capitalization and +# line references are the guide's own, typos included. +variable description line_reference +N1 Number of returns +N2 Number of individuals +A00100 Adjust gross income (AGI) 1040:11 +N02650 Number of returns with total income 1040:9 +A02650 Total income amount 1040:9 +N00200 Number of returns with salaries and wages 1040:1z +A00200 Salaries and wages amount 1040:1z +N00300 Number of returns with taxable interest 1040:2b +A00300 Taxable interest amount 1040:2b +N00400 Number of returns tax-exempt interest 1040:2a +A00400 Tax-exempt interest amount 1040:2a +N00600 Number of returns with ordinary dividends 1040:3b +A00600 Ordinary dividends amount 1040:3b +N00650 Number of returns with qualified dividends 1040:3a +A00650 Qualified dividends amount 1040:3a +N00700 Number of returns with state and local income tax refunds Schedule 1:1 +A00700 State and local income tax refunds amount Schedule 1:1 +N00900 Number of returns with business or professional net income (less loss) Schedule 1:3 +A00900 Business or professional net income (less loss) amount Schedule 1:3 +N01000 Number of returns with net capital gain (less loss) 1040:7 +A01000 Net capital gain (less loss) amount 1040:7 +N01400 Number of returns with taxable individual retirement arrangements distributions 1040:4b +A01400 Taxable individual retirement arrangements distributions amount 1040:4b +N01700 Number of returns with taxable pensions and annuities 1040:5b +A01700 Taxable pensions and annuities amount 1040:5b +N02300 Number of returns with unemployment compensation Schedule 1:7 +A02300 Unemployment compensation amount Schedule 1:7 +N02500 Number of returns with taxable Social Security benefits 1040:6b +A02500 Taxable Social Security benefits amount 1040:6b +N26270 Number of returns with partnership/S-corp net income (less loss) Schedule E:32 +A26270 Partnership/S-corp net income (less loss) amount Schedule E:32 +N25870 Number of returns with total rental and royalty net income (less loss) Schedule E:26 +A25870 Total rental and royalty net income (less loss) Schedule E:26 +N02900 Number of returns with total statutory adjustments Schedule 1:26 +A02900 Total statutory adjustments amount Schedule 1:26 +N03220 Number of returns with educator expenses Schedule 1:11 +A03220 Educator expenses amount Schedule 1:11 +N03300 Number of returns with self-employment retirement plans Schedule 1:16 +A03300 Self-employment retirement plans amount Schedule 1:16 +N03270 Number of returns with self-employment health insurance deduction Schedule 1:17 +A03270 Self-employment health insurance deduction amount Schedule 1:17 +N03150 Number of returns with IRA payments Schedule 1:20 +A03150 IRA payments amount Schedule 1:20 +N03210 Number of returns with student loan interest deduction Schedule 1:21 +A03210 Student loan interest deduction amount Schedule 1:21 +N04450 Number of returns with total standard deduction 1040:12 +A04450 Total standard deduction amount 1040:12 +N04100 Number of returns with basic standard deduction 1040:12 +A04100 Basic standard deduction amount 1040:12 +N04200 Number of returns with additional standard deduction 1040:12 +A04200 Additional standard deduction amount 1040:12 +N04470 Number of returns with itemized deductions 1040:12 +A04470 Total itemized deductions amount 1040:12 +N17000 Number of returns with Total medical and dental expense deduction Schedule A:4 +A17000 Total medical and dental expense deduction amount Schedule A:4 +N18425 Number of returns with State and local income taxes Schedule A:5a +A18425 State and local income taxes amount Schedule A:5a +N18450 Number of returns with State and local general sales tax Schedule A:5a +A18450 State and local general sales tax amount Schedule A:5a +N18500 Number of returns with real estate taxes Schedule A:5b +A18500 Real estate taxes amount Schedule A:5b +N18800 Number of returns with Personal property taxes Schedule A:5c +A18800 Personal property taxes amount Schedule A:5c +N18460 Number of returns with Limited state and local taxes Schedule A:5e +A18460 Limited state and local taxes Schedule A:5e +N18300 Number of returns with taxes paid Schedule A:7 +A18300 Taxes paid amount Schedule A:7 +N19300 Number of returns with mortgage interest paid Schedule A:8a +A19300 Mortgage interest paid amount Schedule A:8a +N19500 Number of returns with Home mortgage from personal seller Schedule A:8b +A19500 Home mortgage from personal seller amount Schedule A:8b +N19530 Number of returns with Deductible points Schedule A:8c +A19530 Deductible points amount Schedule A:8c +N19570 Number of returns with Investment interest paid Schedule A:9 +A19570 Investment interest paid amount Schedule A:9 +N19700 Number of returns with Total charitable contributions Schedule A:14 +A19700 Total charitable contributions amount Schedule A:14 +N20950 Number of returns with Other non-limited miscellaneous deductions Schedule A:16 +A20950 Other non-limited miscellaneous deductions amount Schedule A:16 +N04475 Number of returns with Qualified business income deduction 1040:13 +A04475 Qualified business income deduction amount 1040:13 +N04800 Number of returns with taxable income 1040:15 +A04800 Taxable income amount 1040:15 +N05800 Number of returns with income tax before credits 1040:16 +A05800 Income tax before credits amount 1040:16 +N09600 Number of returns with alternative minimum tax Schedule 2:1 +A09600 Alternative minimum tax amount Schedule 2:1 +N05780 Number of returns with excess advance premium tax credit repayment Schedule 2:2 +A05780 Excess advance premium tax credit repayment amount Schedule 2:2 +N07100 Number of returns with total tax credits 1040:21 +A07100 Total tax credits amount 1040:21 +N07300 Number of returns with foreign tax credit Schedule 3:1 +A07300 Foreign tax credit amount Schedule 3:1 +N07180 Number of returns with child and dependent care credit Schedule 3:2 +A07180 Child and dependent care credit amount Schedule 3:2 +N07230 Number of returns with nonrefundable education credit Schedule 3:3 +A07230 Nonrefundable education credit amount Schedule 3:3 +N07240 Number of returns with retirement savings contribution credit Schedule 3:4 +A07240 Retirement savings contribution credit amount Schedule 3:4 +N07225 Number of returns with nonrefundable child and other dependent tax credit 1040:19 +A07225 Nonrefundable child and other dependent tax credit amount 1040:19 +N07260 Number of returns with residential energy tax credit Schedule 3:5 +A07260 Residential energy tax credit amount Schedule 3:5 +N09400 Number of returns with self-employment tax Schedule 2:4 +A09400 Self-employment tax amount Schedule 2:4 +N85770 Number of returns with total premium tax credit 8926:24 +A85770 Total premium tax credit amount 8926:24 +N85775 Number of returns with advance premium tax credit 8926:25 +A85775 Advance premium tax credit amount 8926:25 +N10600 Number of returns with total tax payments 1040:33 +A10600 Total tax payments amount 1040:33 +N59660 Number of returns with earned income credit 1040:27 +A59660 Earned income credit amount 1040:27 +N59661 Number of returns with earned income credit with no qualifying children 1040:27 +A59661 Earned income credit with no qualifying children amount 1040:27 +N59662 Number of returns with earned income credit with one qualifying child 1040:27 +A59662 Earned income credit with one qualifying child amount 1040:27 +N59663 Number of returns with earned income credit with two qualifying children 1040:27 +A59663 Earned income credit with two qualifying children amount 1040:27 +N59664 Number of returns with earned income credit with three qualifying children 1040:27 +A59664 Earned income credit with three qualifying children amount 1040:27 +N59720 Number of returns with excess earned income credit 1040:27 +A59720 Excess earned income credit (refundable) amount 1040:27 +N11070 Number of returns with refundable child tax credit or additional child tax credit 1040:28 +A11070 Refundable child tax credit or additional child tax credit amount 1040:28 +N10960 Number of returns with refundable education credit 1040:29 +A10960 Refundable education credit amount 1040:29 +N11560 Number of returns with net premium tax credit Schedule 3:9 +A11560 Net premium tax credit amount Schedule 3:9 +N06500 Number of returns with income tax after credits 1040:22 +A06500 Income tax after credits amount 1040:22 +N10300 Number of returns with tax liability 1040:24 +A10300 Total tax liability amount 1040:24 +N85530 Number of returns with additional Medicare tax 8959:24 +A85530 Additional Medicare tax 8959:24 +N85300 Number of returns with net investment income tax 8960:17 +A85300 Net investment income tax 8960:17 +N11901 Number of returns with tax due at time of filing 1040:37 +A11901 Tax due at time of filing amount 1040:37 +N11900 Number of returns with total overpayments 1040:34 +A11900 Total overpayments amount 1040:34 +N11902 Number of returns with overpayments refunded 1040:35a +A11902 Overpayments refunded amount 1040:35a +N12000 Number of returns with credit to next year’s estimated tax 1040:36 +A12000 Credited to next year’s estimated tax amount 1040:36 diff --git a/tests/data/irs_soi_doc_guides/23incmdocguide.tsv b/tests/data/irs_soi_doc_guides/23incmdocguide.tsv new file mode 100644 index 00000000..f22fc13d --- /dev/null +++ b/tests/data/irs_soi_doc_guides/23incmdocguide.tsv @@ -0,0 +1,155 @@ +# IRS SOI Historic Table 2 (State Data), Tax Year 2023, Documentation Guide, section G (Selected Income and Tax Items). +# Source: https://www.irs.gov/pub/irs-soi/23incmdocguide.doc +# sha256: 942790690b0027b465019ec5d3f5d8f23124882ab85535f06129f74c12021206 +# Retrieved 2026-10-12. Documents the columns of 23in55cmcsv.csv. +# Extracted with macOS textutil -convert txt; one row per N/A variable. +# Endnote markers such as [4] are dropped; wording, capitalization and +# line references are the guide's own, typos included. +variable description line_reference +N1 Number of returns +N2 Number of individuals +A00100 Adjust gross income (AGI) 1040:11 +N02650 Number of returns with total income 1040:9 +A02650 Total income amount 1040:9 +N00200 Number of returns with salaries and wages 1040:1z +A00200 Salaries and wages amount 1040:1z +N00300 Number of returns with taxable interest 1040:2b +A00300 Taxable interest amount 1040:2b +N00400 Number of returns tax-exempt interest 1040:2a +A00400 Tax-exempt interest amount 1040:2a +N00600 Number of returns with ordinary dividends 1040:3b +A00600 Ordinary dividends amount 1040:3b +N00650 Number of returns with qualified dividends 1040:3a +A00650 Qualified dividends amount 1040:3a +N00700 Number of returns with state and local income tax refunds Schedule 1:1 +A00700 State and local income tax refunds amount Schedule 1:1 +N00900 Number of returns with business or professional net income (less loss) Schedule 1:3 +A00900 Business or professional net income (less loss) amount Schedule 1:3 +N01000 Number of returns with net capital gain (less loss) 1040:7 +A01000 Net capital gain (less loss) amount 1040:7 +N01400 Number of returns with taxable individual retirement arrangements distributions 1040:4b +A01400 Taxable individual retirement arrangements distributions amount 1040:4b +N01700 Number of returns with taxable pensions and annuities 1040:5b +A01700 Taxable pensions and annuities amount 1040:5b +N02300 Number of returns with unemployment compensation Schedule 1:7 +A02300 Unemployment compensation amount Schedule 1:7 +N02500 Number of returns with taxable Social Security benefits 1040:6b +A02500 Taxable Social Security benefits amount 1040:6b +N26270 Number of returns with partnership/S-corp net income (less loss) Schedule E:32 +A26270 Partnership/S-corp net income (less loss) amount Schedule E:32 +N25870 Number of returns with total rental and royalty net income (less loss) Schedule E:26 +A25870 Total rental and royalty net income (less loss) Schedule E:26 +N02900 Number of returns with total statutory adjustments Schedule 1:26 +A02900 Total statutory adjustments amount Schedule 1:26 +N03220 Number of returns with educator expenses Schedule 1:11 +A03220 Educator expenses amount Schedule 1:11 +N03300 Number of returns with self-employment retirement plans Schedule 1:16 +A03300 Self-employment retirement plans amount Schedule 1:16 +N03270 Number of returns with self-employment health insurance deduction Schedule 1:17 +A03270 Self-employment health insurance deduction amount Schedule 1:17 +N03150 Number of returns with IRA payments Schedule 1:20 +A03150 IRA payments amount Schedule 1:20 +N03210 Number of returns with student loan interest deduction Schedule 1:21 +A03210 Student loan interest deduction amount Schedule 1:21 +N04450 Number of returns with total standard deduction 1040:12 +A04450 Total standard deduction amount 1040:12 +N04100 Number of returns with basic standard deduction 1040:12 +A04100 Basic standard deduction amount 1040:12 +N04200 Number of returns with additional standard deduction 1040:12 +A04200 Additional standard deduction amount 1040:12 +N04470 Number of returns with itemized deductions 1040:12 +A04470 Total itemized deductions amount 1040:12 +N17000 Number of returns with Total medical and dental expense deduction Schedule A:4 +A17000 Total medical and dental expense deduction amount Schedule A:4 +N18425 Number of returns with state and local income taxes Schedule A:5a +A18425 State and local income taxes amount Schedule A:5a +N18450 Number of returns with state and local general sales tax Schedule A:5a +A18450 State and local general sales tax amount Schedule A:5a +N18500 Number of returns with real estate taxes Schedule A:5b +A18500 Real estate taxes amount Schedule A:5b +N18800 Number of returns with Personal property taxes Schedule A:5c +A18800 Personal property taxes amount Schedule A:5c +N18460 Number of returns with Limited state and local taxes Schedule A:5e +A18460 Limited state and local taxes Schedule A:5e +N18300 Number of returns with taxes paid Schedule A:7 +A18300 Taxes paid amount Schedule A:7 +N19300 Number of returns with mortgage interest paid Schedule A:8a +A19300 Mortgage interest paid amount Schedule A:8a +N19500 Number of returns with Home mortgage from personal seller Schedule A:8b +A19500 Home mortgage from personal seller amount Schedule A:8b +N19530 Number of returns with Deductible points Schedule A:8c +A19530 Deductible points amount Schedule A:8c +N19570 Number of returns with Investment interest paid Schedule A:9 +A19570 Investment interest paid amount Schedule A:9 +N19700 Number of returns with Total charitable contributions Schedule A:14 +A19700 Total charitable contributions amount Schedule A:14 +N20950 Number of returns with Other non-limited miscellaneous deductions Schedule A:16 +A20950 Other non-limited miscellaneous deductions amount Schedule A:16 +N04475 Number of returns with Qualified business income deduction 1040:13 +A04475 Qualified business income deduction amount 1040:13 +N04800 Number of returns with taxable income 1040:15 +A04800 Taxable income amount 1040:15 +N05800 Number of returns with income tax before credits 1040:16 +A05800 Income tax before credits amount 1040:16 +N09600 Number of returns with alternative minimum tax Schedule 2:1 +A09600 Alternative minimum tax amount Schedule 2:1 +N05780 Number of returns with excess advance premium tax credit repayment Schedule 2:2 +A05780 Excess advance premium tax credit repayment amount Schedule 2:2 +N07100 Number of returns with total tax credits 1040:21 +A07100 Total tax credits amount 1040:21 +N07300 Number of returns with foreign tax credit Schedule 3:1 +A07300 Foreign tax credit amount Schedule 3:1 +N07180 Number of returns with child and dependent care credit Schedule 3:2 +A07180 Child and dependent care credit amount Schedule 3:2 +N07230 Number of returns with nonrefundable education credit Schedule 3:3 +A07230 Nonrefundable education credit amount Schedule 3:3 +N07240 Number of returns with retirement savings contribution credit Schedule 3:4 +A07240 Retirement savings contribution credit amount Schedule 3:4 +N07225 Number of returns with nonrefundable child and other dependent tax credit 1040:19 +A07225 Nonrefundable child and other dependent tax credit amount 1040:19 +N07262 Number of returns with residential clean energy tax credit Schedule 3:5a +A07262 Residential clean energy tax credit amount Schedule 3:5a +N07265 Number of returns with energy efficient home improvement tax credit Schedule 3:5b +A07265 Energy efficient home improvement tax credit amount Schedule 3:5b +N09400 Number of returns with self-employment tax Schedule 2:4 +A09400 Self-employment tax amount Schedule 2:4 +N85770 Number of returns with total premium tax credit 8962:24 +A85770 Total premium tax credit amount 8962:24 +N85775 Number of returns with advance premium tax credit 8962:25 +A85775 Advance premium tax credit amount 8962:25 +N10600 Number of returns with total tax payments 1040:33 +A10600 Total tax payments amount 1040:33 +N59660 Number of returns with earned income credit 1040:27 +A59660 Earned income credit amount 1040:27 +N59661 Number of returns with earned income credit with no qualifying children 1040:27 +A59661 Earned income credit with no qualifying children amount 1040:27 +N59662 Number of returns with earned income credit with one qualifying child 1040:27 +A59662 Earned income credit with one qualifying child amount 1040:27 +N59663 Number of returns with earned income credit with two qualifying children 1040:27 +A59663 Earned income credit with two qualifying children amount 1040:27 +N59664 Number of returns with earned income credit with three qualifying children 1040:27 +A59664 Earned income credit with three qualifying children amount 1040:27 +N59720 Number of returns with excess earned income credit 1040:27 +A59720 Excess earned income credit (refundable) amount 1040:27 +N11070 Number of returns with additional child tax credit 1040:28 +A11070 Additional child tax credit amount 1040:28 +N10960 Number of returns with refundable education credit 1040:29 +A10960 Refundable education credit amount 1040:29 +N11560 Number of returns with net premium tax credit Schedule 3:9 +A11560 Net premium tax credit amount Schedule 3:9 +N06500 Number of returns with income tax after credits 1040:22 +A06500 Income tax after credits amount 1040:22 +N10300 Number of returns with tax liability 1040:24 +A10300 Total tax liability amount 1040:24 +N85530 Number of returns with additional Medicare tax 8959:24 +A85530 Additional Medicare tax 8959:24 +N85300 Number of returns with net investment income tax 8960:17 +A85300 Net investment income tax 8960:17 +N11901 Number of returns with tax due at time of filing 1040:37 +A11901 Tax due at time of filing amount 1040:37 +N11900 Number of returns with total overpayments 1040:34 +A11900 Total overpayments amount 1040:34 +N11902 Number of returns with overpayments refunded 1040:35a +A11902 Overpayments refunded amount 1040:35a +N12000 Number of returns with credit to next year’s estimated tax 1040:36 +A12000 Credited to next year’s estimated tax amount 1040:36 diff --git a/tests/test_chronicle_bundle.py b/tests/test_chronicle_bundle.py index e8c75695..5171ea84 100644 --- a/tests/test_chronicle_bundle.py +++ b/tests/test_chronicle_bundle.py @@ -377,13 +377,19 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): # source table; the publisher, period, geography and entity keys already # exist. Measured with only bea-nipa-private-group-health-insurance at # bundle year 2023: no warnings and no duplicate keys. + # The congressional-district package adds its Schedule A line 8b, 8c and 9 + # columns: 2,880 TY2022 tax-unit facts (six measures at 480 rows), six at + # each of its geographies. Measured with the 20 IRS SOI packages at bundle + # year 2023: no new geography, period or entity keys, duplicate keys or + # warnings. Its SALT and PTC column fixes, its line 8a relabel and the + # Historic Table 2 QBI column fix change no count. assert summary["counts"] == { "aggregate_duplicate_key_count": 0, "entity_count": 12, "error_count": 0, # chronicle#322 adds 4,947 ONS observations and 3,200 DESNZ ones. # Inner/Outer London also add two region-level geography keys. - "fact_count": 1004254, + "fact_count": 1007134, "geography_count": 20548, "period_count": 511, # 467 before chronicle#292 moved the congressional-district and @@ -441,7 +447,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): if row_count == 1: assert row["aggregate_fact_key"].startswith("ledger.aggregate_fact.v2:") assert row["semantic_fact_key"].startswith("ledger.semantic_fact.v2:") - assert row_count == 1004254 + assert row_count == 1007134 assert provenance_classes <= { "administrative", "census", @@ -462,7 +468,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): "jct-obbba-revenue-estimates-2025", "jct-tax-expenditures-2024", ] - assert coverage["fact_count"] == 1004254 + assert coverage["fact_count"] == 1007134 assert coverage["counts"]["by_source"] == { "bea": 446, "bfp_economic_outlook": 5, @@ -488,7 +494,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): "hmrc": 31515, "ici": 12, "ird": 4362, - "irs_soi": 41083, + "irs_soi": 43963, "isc": 4, "jrc_euromod_be": 90, "kff": 52, @@ -893,7 +899,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): ] == 7 ) - assert table_counts["irs_soi:Congressional District Data 2022"] == 26880 + assert table_counts["irs_soi:Congressional District Data 2022"] == 29760 assert table_counts["irs_soi:IRS SOI County Data 2022"] == 6286 assert table_counts["census_pep:Vintage 2024 County Population Totals"] == 3144 assert ( @@ -1762,6 +1768,9 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): expected_period_counts["calendar_year:2023"] += 162 # chronicle#291: IRS SOI Historic Table 2 TY2023 state AGI-band facts. expected_period_counts["tax_year:2023"] += 1020 + # The congressional-district file's Schedule A line 8b, 8c and 9 columns: + # six measures at its 480 rows. + expected_period_counts["tax_year:2022"] += 2880 # BEA NIPA Table 7.8 line 17 (B4923C): the bundle year's own column. expected_period_counts["calendar_year:2023"] += 1 assert coverage["counts"]["by_period"] == expected_period_counts @@ -1790,10 +1799,10 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): assert coverage["counts"]["by_geography"]["nuts1:BE2"] == 3673 assert coverage["counts"]["by_geography"]["nuts1:BE3"] == 3662 assert coverage["counts"]["by_geography"]["commune:11002"] == 1 - assert coverage["counts"]["by_geography"]["country:0100000US"] == 2110 - assert coverage["counts"]["by_geography"]["state:0400000US06"] == 249 + assert coverage["counts"]["by_geography"]["country:0100000US"] == 2116 + assert coverage["counts"]["by_geography"]["state:0400000US06"] == 255 assert ( - coverage["counts"]["by_geography"]["congressional_district:5001700US0601"] == 56 + coverage["counts"]["by_geography"]["congressional_district:5001700US0601"] == 62 ) assert coverage["counts"]["by_geography"]["country:K02000001"] == 31901 assert coverage["counts"]["by_geography"]["country:E92000001"] == 6099 @@ -1832,7 +1841,7 @@ def test_build_bundle_writes_merged_consumer_contract(tmp_path): "person": 496036, "return": 14600, "social_protection_scheme": 36, - "tax_unit": 42388, + "tax_unit": 45268, } assert not coverage["duplicates"]["aggregate_fact_keys"] assert len(coverage["duplicates"]["semantic_fact_keys"]) == 2324 diff --git a/tests/test_chronicle_soi_doc_guide_columns.py b/tests/test_chronicle_soi_doc_guide_columns.py new file mode 100644 index 00000000..dc0477fb --- /dev/null +++ b/tests/test_chronicle_soi_doc_guide_columns.py @@ -0,0 +1,707 @@ +"""IRS SOI geographic-file measures read the column their documentation guide defines. + +Historic Table 2, the congressional-district file and the county file publish +one CSV column per IRS variable (``N18460``, ``A18460``, ...). Each file's +documentation guide defines every variable with a description and a form line. +``GUIDE_DEFINITIONS`` pins every ``measure_id`` those packages declare to one +guide definition, and the tests below check that every package, in any vintage, +reads the one column that definition names. Extracts of the guides live in +``tests/data/irs_soi_doc_guides``; each records its irs.gov URL and sha256. + +Four mappings broke this rule until they were fixed together: + +- ``congressional_district_2022`` ``limited_state_local_taxes_*`` read + ``N18425``/``A18425``, "State and local income taxes" (Schedule A line 5a), + instead of ``N18460``/``A18460``, "Limited state and local taxes" (line 5e). + Historic Table 2 already read line 5e. +- The same package's ``premium_tax_credit_*`` read ``N85530``/``A85530``, + "Additional Medicare tax" (Form 8959 line 24), instead of + ``N85770``/``A85770``, "Total premium tax credit", which Historic Table 2 + already read. +- The same package's ``interest_paid_deduction_*`` read ``N19300``/``A19300``, + "Mortgage interest paid" (Schedule A line 8a). Table 2.1 uses that measure id + for the line 10 total, which no geographic file publishes. The line 8a + columns are now ``mortgage_interest_paid_*``, Table 2.1's line 8a ids, next + to new line 8b, 8c and 9 measures. +- ``historic_table_2`` and ``historic_table_2_state_broad_2022`` ``qbi_*`` read + ``N03270``/``A03270``, "Self-employment health insurance deduction", instead + of ``N04475``/``A04475``, "Qualified business income deduction". + +``docs/concept-migrations.md`` records the change. +""" + +from __future__ import annotations + +import csv +import re +from collections import defaultdict +from dataclasses import dataclass +from functools import cache +from importlib.resources import files +from pathlib import Path +from typing import Any + +import pytest +import yaml + +from chronicle.source_package import load_source_package + +REPO_ROOT = Path(__file__).resolve().parents[1] +IRS_SOI_PACKAGES = REPO_ROOT / "packages" / "irs_soi" +GUIDE_DIR = Path(__file__).resolve().parent / "data" / "irs_soi_doc_guides" +IRS_VARIABLE = re.compile(r"^[NA]\d+$") + +# The guide extract that documents each data file a package reads. +GUIDE_BY_DATA_FILE = { + "22incd.csv": "22incddocguide", + "22incyallnoagi.csv": "22incydocguide", + "22in55cmcsv.csv": "22incmdocguide", + "23in55cmcsv.csv": "23incmdocguide", +} + +# IRS-variable columns a data file carries that its guide does not define. +# No package may read them. +UNDOCUMENTED_COLUMNS = { + "22incd.csv": frozenset({"A00101"}), + "22incyallnoagi.csv": frozenset({"A00101"}), +} + +# The TY2022 guides print form "8926" for these variables; the TY2023 Historic +# Table 2 guide prints "8962" for the same variables and descriptions. +LINE_ERRATA = {"8926:24": "8962:24", "8926:25": "8962:25"} + +# measure_id -> (form line, guide description, later guide wordings ...). +# Descriptions are the TY2022 guide wording and match case-insensitively; each +# must name exactly one variable in every guide a package reads. +GUIDE_DEFINITIONS: dict[str, tuple[str, ...]] = { + "return_count": ("", "Number of returns"), + "tax_filer_individual_count": ("", "Number of individuals"), + "adjusted_gross_income": ("1040:11", "Adjust gross income (AGI)"), + "total_income_returns": ("1040:9", "Number of returns with total income"), + "total_income_amount": ("1040:9", "Total income amount"), + "wages_salaries_returns": ("1040:1z", "Number of returns with salaries and wages"), + "wages_salaries_amount": ("1040:1z", "Salaries and wages amount"), + "taxable_interest_returns": ("1040:2b", "Number of returns with taxable interest"), + "taxable_interest_amount": ("1040:2b", "Taxable interest amount"), + "tax_exempt_interest_returns": ("1040:2a", "Number of returns tax-exempt interest"), + "tax_exempt_interest_amount": ("1040:2a", "Tax-exempt interest amount"), + "ordinary_dividends_returns": ( + "1040:3b", + "Number of returns with ordinary dividends", + ), + "ordinary_dividends_amount": ("1040:3b", "Ordinary dividends amount"), + "qualified_dividends_returns": ( + "1040:3a", + "Number of returns with qualified dividends", + ), + "qualified_dividends_amount": ("1040:3a", "Qualified dividends amount"), + "schedule_c_income_returns": ( + "Schedule 1:3", + "Number of returns with business or professional net income (less loss)", + ), + "schedule_c_income_amount": ( + "Schedule 1:3", + "Business or professional net income (less loss) amount", + ), + "net_capital_gains_returns": ( + "1040:7", + "Number of returns with net capital gain (less loss)", + ), + "net_capital_gains_amount": ("1040:7", "Net capital gain (less loss) amount"), + "taxable_ira_distributions_returns": ( + "1040:4b", + ( + "Number of returns with taxable individual retirement arrangements " + "distributions" + ), + ), + "taxable_ira_distributions_amount": ( + "1040:4b", + "Taxable individual retirement arrangements distributions amount", + ), + "taxable_pension_income_returns": ( + "1040:5b", + "Number of returns with taxable pensions and annuities", + ), + "taxable_pension_income_amount": ( + "1040:5b", + "Taxable pensions and annuities amount", + ), + "unemployment_compensation_returns": ( + "Schedule 1:7", + "Number of returns with unemployment compensation", + ), + "unemployment_compensation_amount": ( + "Schedule 1:7", + "Unemployment compensation amount", + ), + "taxable_social_security_returns": ( + "1040:6b", + "Number of returns with taxable Social Security benefits", + ), + "taxable_social_security_amount": ( + "1040:6b", + "Taxable Social Security benefits amount", + ), + "partnership_scorp_income_returns": ( + "Schedule E:32", + "Number of returns with partnership/S-corp net income (less loss)", + ), + "partnership_scorp_income_amount": ( + "Schedule E:32", + "Partnership/S-corp net income (less loss) amount", + ), + "rental_royalty_income_returns": ( + "Schedule E:26", + "Number of returns with total rental and royalty net income (less loss)", + ), + "rental_royalty_income_amount": ( + "Schedule E:26", + "Total rental and royalty net income (less loss)", + ), + "itemized_deductions_returns": ( + "1040:12", + "Number of returns with itemized deductions", + ), + "itemized_deductions_amount": ("1040:12", "Total itemized deductions amount"), + "medical_dental_expense_returns": ( + "Schedule A:4", + "Number of returns with Total medical and dental expense deduction", + ), + "medical_dental_expense_amount": ( + "Schedule A:4", + "Total medical and dental expense deduction amount", + ), + "real_estate_taxes_claims": ( + "Schedule A:5b", + "Number of returns with real estate taxes", + ), + "real_estate_taxes_amount": ("Schedule A:5b", "Real estate taxes amount"), + "limited_state_local_taxes_returns": ( + "Schedule A:5e", + "Number of returns with Limited state and local taxes", + ), + "limited_state_local_taxes_amount": ( + "Schedule A:5e", + "Limited state and local taxes", + ), + "mortgage_interest_paid_returns": ( + "Schedule A:8a", + "Number of returns with mortgage interest paid", + ), + "mortgage_interest_paid_amount": ("Schedule A:8a", "Mortgage interest paid amount"), + "home_mortgage_personal_seller_returns": ( + "Schedule A:8b", + "Number of returns with Home mortgage from personal seller", + ), + "home_mortgage_personal_seller_amount": ( + "Schedule A:8b", + "Home mortgage from personal seller amount", + ), + "deductible_points_returns": ( + "Schedule A:8c", + "Number of returns with Deductible points", + ), + "deductible_points_amount": ("Schedule A:8c", "Deductible points amount"), + "investment_interest_paid_returns": ( + "Schedule A:9", + "Number of returns with Investment interest paid", + ), + "investment_interest_paid_amount": ( + "Schedule A:9", + "Investment interest paid amount", + ), + "charitable_returns": ( + "Schedule A:14", + "Number of returns with Total charitable contributions", + ), + "charitable_amount": ("Schedule A:14", "Total charitable contributions amount"), + "qbi_claims": ( + "1040:13", + "Number of returns with Qualified business income deduction", + ), + "qbi_amount": ("1040:13", "Qualified business income deduction amount"), + "qualified_business_income_deduction_returns": ( + "1040:13", + "Number of returns with Qualified business income deduction", + ), + "qualified_business_income_deduction_amount": ( + "1040:13", + "Qualified business income deduction amount", + ), + "taxable_income_returns": ("1040:15", "Number of returns with taxable income"), + "taxable_income_amount": ("1040:15", "Taxable income amount"), + "income_tax_before_credits_returns": ( + "1040:16", + "Number of returns with income tax before credits", + ), + "income_tax_before_credits_amount": ("1040:16", "Income tax before credits amount"), + "ctc_claims": ( + "1040:19", + "Number of returns with nonrefundable child and other dependent tax credit", + ), + "ctc_amount": ( + "1040:19", + "Nonrefundable child and other dependent tax credit amount", + ), + "premium_tax_credit_returns": ( + "8962:24", + "Number of returns with total premium tax credit", + ), + "premium_tax_credit_amount": ("8962:24", "Total premium tax credit amount"), + "eitc_claims": ("1040:27", "Number of returns with earned income credit"), + "eitc_amount": ("1040:27", "Earned income credit amount"), + "eitc_no_children_claims": ( + "1040:27", + "Number of returns with earned income credit with no qualifying children", + ), + "eitc_no_children_amount": ( + "1040:27", + "Earned income credit with no qualifying children amount", + ), + "eitc_one_child_claims": ( + "1040:27", + "Number of returns with earned income credit with one qualifying child", + ), + "eitc_one_child_amount": ( + "1040:27", + "Earned income credit with one qualifying child amount", + ), + "eitc_two_children_claims": ( + "1040:27", + "Number of returns with earned income credit with two qualifying children", + ), + "eitc_two_children_amount": ( + "1040:27", + "Earned income credit with two qualifying children amount", + ), + # The guides say "three"; these ids and their labels say "three or more". + "eitc_three_or_more_children_claims": ( + "1040:27", + "Number of returns with earned income credit with three qualifying children", + ), + "eitc_three_or_more_children_amount": ( + "1040:27", + "Earned income credit with three qualifying children amount", + ), + "actc_claims": ( + "1040:28", + ( + "Number of returns with refundable child tax credit or additional " + "child tax credit" + ), + "Number of returns with additional child tax credit", # TY2023 guide + ), + "actc_amount": ( + "1040:28", + "Refundable child tax credit or additional child tax credit amount", + "Additional child tax credit amount", # TY2023 guide + ), + "income_tax_liability_returns": ( + "1040:22", + "Number of returns with income tax after credits", + ), + "income_tax_liability_amount": ("1040:22", "Income tax after credits amount"), +} + +# Measure ids that mean a line no geographic file publishes. No package that +# reads an IRS-variable file may declare them. +NOT_IN_GEOGRAPHIC_FILES = { + "interest_paid_deduction_returns": ( + "Table 2.1's Schedule A line 10 interest paid deduction; the geographic " + "files publish only its parts N19300 (8a), N19500 (8b), N19530 (8c) and " + "N19570 (9)" + ), + "interest_paid_deduction_amount": ( + "Table 2.1's Schedule A line 10 interest paid deduction; the geographic " + "files publish only its parts A19300 (8a), A19500 (8b), A19530 (8c) and " + "A19570 (9)" + ), +} + + +@dataclass(frozen=True) +class ColumnRead: + """One measure declaration in a package that reads an IRS-variable file.""" + + package_dir: str + package_id: str + measure_id: str + data_file: str + data_path: Path + column: str + header: str + declared: tuple[str, ...] + concept: str | None + label: str | None + + +def _normalize(text: str) -> str: + return " ".join(text.casefold().split()) + + +def _column_index(letters: str) -> int: + index = 0 + for letter in letters: + index = index * 26 + ord(letter) - ord("A") + 1 + return index - 1 + + +@cache +def _guide(stem: str) -> dict[str, tuple[str, str]]: + path = GUIDE_DIR / f"{stem}.tsv" + lines = [line for line in path.read_text().splitlines() if not line.startswith("#")] + reader = csv.DictReader(lines, delimiter="\t") + assert reader.fieldnames == ["variable", "description", "line_reference"] + guide: dict[str, tuple[str, str]] = {} + for row in reader: + assert IRS_VARIABLE.match(row["variable"]), (stem, row) + assert row["variable"] not in guide, (stem, row["variable"]) + guide[row["variable"]] = (row["description"], row["line_reference"]) + return guide + + +def _artifact_file(artifact: dict[str, Any]) -> tuple[str, Path]: + root = files(artifact["resource_package"]).joinpath(artifact["resource_directory"]) + manifest = yaml.safe_load(root.joinpath(artifact["manifest"]).read_text()) + spec = manifest["files"][artifact["artifact_year"]] + return spec["filename"], Path(str(root.joinpath(spec["filename"]))) + + +@cache +def _header(path: Path) -> tuple[str, ...]: + with path.open("rb") as file: + first_line = file.readline().decode("utf-8-sig") + return tuple(next(csv.reader([first_line]))) + + +@cache +def _column_reads() -> tuple[ColumnRead, ...]: + reads: dict[tuple[str, str, str], ColumnRead] = {} + for path in sorted(IRS_SOI_PACKAGES.glob("*/source_package.yaml")): + payload = yaml.safe_load(path.read_text()) + artifact = payload["artifact"] + if artifact.get("parser") != "delimited_text_full_rows": + continue + assert artifact.get("artifact_year"), ( + f"{path.parent.name}: a delimited IRS package without artifact_year " + "can read a different file per year; extend this test to check each" + ) + data_file, data_path = _artifact_file(artifact) + header = _header(data_path) + for record_set in payload["record_sets"]: + for measure in record_set["measures"]: + assert "column_by_year" not in measure, (path.parent.name, measure) + column = measure["column"] + read = ColumnRead( + package_dir=path.parent.name, + package_id=payload["package_id"], + measure_id=measure["measure_id"], + data_file=data_file, + data_path=data_path, + column=column, + header=header[_column_index(column)], + declared=tuple( + measure[key] + for key in ("source_column_id", "expected_column_header") + if measure.get(key) is not None + ), + concept=measure.get("concept"), + label=measure.get("label"), + ) + key = (read.package_dir, read.measure_id, read.column) + assert reads.setdefault(key, read) == read, key + return tuple(reads.values()) + + +def _defined_variable(measure_id: str, guide_stem: str) -> str | None: + wordings = {_normalize(text) for text in GUIDE_DEFINITIONS[measure_id][1:]} + matches = [ + variable + for variable, (description, _line) in _guide(guide_stem).items() + if _normalize(description) in wordings + ] + assert len(matches) <= 1, (measure_id, guide_stem, matches) + return matches[0] if matches else None + + +def test_irs_variable_packages_are_found(): + packages = {read.package_dir for read in _column_reads()} + + assert { + "congressional_district_2022", + "county_2022", + "historic_table_2", + "historic_table_2_state_agi_2022", + "historic_table_2_state_agi_2023", + "historic_table_2_state_broad_2022", + "historic_table_2_state_eitc_2022", + } <= packages + assert all(IRS_VARIABLE.match(read.header) for read in _column_reads()) + + +def test_guide_extracts_define_every_column_their_files_publish(): + data_files = {read.data_file for read in _column_reads()} + assert data_files <= set(GUIDE_BY_DATA_FILE), sorted(data_files) + + for data_file in sorted(data_files): + path = next( + read.data_path for read in _column_reads() if read.data_file == data_file + ) + published = {name for name in _header(path) if IRS_VARIABLE.match(name)} + documented = set(_guide(GUIDE_BY_DATA_FILE[data_file])) + undocumented = UNDOCUMENTED_COLUMNS.get(data_file, frozenset()) + + assert published - documented == undocumented, data_file + assert documented <= published, data_file + + +def test_guide_descriptions_name_one_variable_each(): + for stem in sorted(set(GUIDE_BY_DATA_FILE.values())): + by_description: dict[str, list[str]] = defaultdict(list) + for variable, (description, _line) in _guide(stem).items(): + by_description[_normalize(description)].append(variable) + + assert { + description: variables + for description, variables in by_description.items() + if len(variables) > 1 + } == {}, stem + + +def test_every_definition_names_the_same_variable_in_every_guide(): + for measure_id in sorted(GUIDE_DEFINITIONS): + variables = { + stem: _defined_variable(measure_id, stem) + for stem in sorted(set(GUIDE_BY_DATA_FILE.values())) + } + named = set(variables.values()) - {None} + + assert len(named) == 1, (measure_id, variables) + + +def test_every_measure_reads_the_column_its_guide_definition_names(): + failures = [] + for read in _column_reads(): + where = f"{read.package_dir}:{read.measure_id} (column {read.column})" + if read.measure_id in NOT_IN_GEOGRAPHIC_FILES: + failures.append( + f"{where} reads {read.header}, but the id means " + f"{NOT_IN_GEOGRAPHIC_FILES[read.measure_id]}" + ) + continue + if read.measure_id not in GUIDE_DEFINITIONS: + failures.append( + f"{where} reads {read.header} and has no GUIDE_DEFINITIONS entry; " + "pin it to the guide description of the column it should read" + ) + continue + if any(declared != read.header for declared in read.declared): + failures.append( + f"{where} declares {read.declared}, header is {read.header}" + ) + guide_stem = GUIDE_BY_DATA_FILE[read.data_file] + if read.header in UNDOCUMENTED_COLUMNS.get(read.data_file, frozenset()): + failures.append(f"{where} reads {read.header}, which {guide_stem} omits") + continue + expected = _defined_variable(read.measure_id, guide_stem) + if read.header != expected: + description, line = _guide(guide_stem)[read.header] + failures.append( + f"{where} reads {read.header} ({description!r}, {line}); " + f"{guide_stem} defines {read.measure_id} as {expected}" + ) + continue + _description, line = _guide(guide_stem)[read.header] + if LINE_ERRATA.get(line, line) != GUIDE_DEFINITIONS[read.measure_id][0]: + failures.append(f"{where}: {guide_stem} puts {read.header} on line {line}") + + assert failures == [], "\n".join(failures) + + +def test_measure_ids_shared_across_packages_read_one_column(): + headers: dict[str, set[str]] = defaultdict(set) + packages: dict[str, set[str]] = defaultdict(set) + for read in _column_reads(): + headers[read.measure_id].add(read.header) + packages[read.measure_id].add(read.package_dir) + shared = {measure_id for measure_id, dirs in packages.items() if len(dirs) > 1} + + assert { + "limited_state_local_taxes_amount", + "premium_tax_credit_amount", + "return_count", + } <= shared + assert { + measure_id: sorted(headers[measure_id]) + for measure_id in shared + if len(headers[measure_id]) > 1 + } == {} + + +def test_guide_definitions_have_no_stale_entries(): + read_ids = {read.measure_id for read in _column_reads()} + + assert set(GUIDE_DEFINITIONS) - read_ids == set() + assert set(GUIDE_DEFINITIONS) & set(NOT_IN_GEOGRAPHIC_FILES) == set() + + +def test_line_10_interest_paid_ids_keep_their_table_2_1_meaning(): + table_2_1 = yaml.safe_load( + (IRS_SOI_PACKAGES / "table_2_1" / "source_package.yaml").read_text() + ) + table_2_1_ids = { + measure["measure_id"] + for record_set in table_2_1["record_sets"] + for measure in record_set["measures"] + } + + assert set(NOT_IN_GEOGRAPHIC_FILES) <= table_2_1_ids + assert not {read.measure_id for read in _column_reads()} & set( + NOT_IN_GEOGRAPHIC_FILES + ) + + +SCHEDULE_A_INTEREST_PARTS = ( + "mortgage_interest_paid", # line 8a + "home_mortgage_personal_seller", # line 8b + "deductible_points", # line 8c + "investment_interest_paid", # line 9 +) + + +def test_congressional_district_interest_parts_share_table_2_1_ids(): + table_2_1 = { + measure["measure_id"]: measure + for record_set in yaml.safe_load( + (IRS_SOI_PACKAGES / "table_2_1" / "source_package.yaml").read_text() + )["record_sets"] + for measure in record_set["measures"] + } + district = { + read.measure_id: read + for read in _column_reads() + if read.package_dir == "congressional_district_2022" + } + + for part in SCHEDULE_A_INTEREST_PARTS: + for suffix in ("returns", "amount"): + measure_id = f"{part}_{suffix}" + assert district[measure_id].concept == table_2_1[measure_id]["concept"] + assert district[measure_id].label == table_2_1[measure_id]["label"] + + +@cache +def _facts(package_id: str, year: int) -> dict[str, Any]: + package = load_source_package(package_id) + return {fact.source_record_id: fact for fact in package.build_facts(year)} + + +def _us_values(package_id: str, prefix: str) -> dict[str, float]: + return { + record_id.removeprefix(prefix): fact.value + for record_id, fact in _facts(package_id, 2022).items() + if record_id.startswith(prefix) + } + + +# Historic Table 2 and the congressional-district file use different ids for +# the qualified business income deduction columns. +HT2_TO_DISTRICT_IDS = { + "qbi_claims": "qualified_business_income_deduction_returns", + "qbi_amount": "qualified_business_income_deduction_amount", +} + + +def test_congressional_district_us_row_is_a_subset_of_historic_table_2(): + """Differential check of the two TY2022 IRS tabulations of one population. + + The district file drops returns without a ZIP code that matches the state, + and it suppresses small cells, so its US row should not exceed Historic + Table 2's. Every TY2022 ratio lies in [0.92, 0.99]. The columns the + district package used to read gave 2.04 (SALT amount), 0.72 (SALT returns), + 0.87 and 0.26 (PTC returns and amount), and the district QBI deduction was + 6.65 times what Historic Table 2's ``qbi_amount`` read. + """ + district = _us_values( + "soi-congressional-district-2022", + "irs_soi.ty2022.congressional_district_2022.all_returns.us.", + ) + ht2 = _us_values("soi-historic-table-2", "irs_soi.ty2022.historic_table_2.us.all.") + pairs = { + measure_id: (district[HT2_TO_DISTRICT_IDS.get(measure_id, measure_id)], value) + for measure_id, value in ht2.items() + if HT2_TO_DISTRICT_IDS.get(measure_id, measure_id) in district + } + + assert { + "limited_state_local_taxes_returns", + "limited_state_local_taxes_amount", + "premium_tax_credit_returns", + "premium_tax_credit_amount", + "qbi_claims", + "qbi_amount", + } <= set(pairs) + assert len(pairs) == 44 + assert { + measure_id: round(cd / ht2_value, 4) + for measure_id, (cd, ht2_value) in pairs.items() + if not 0.90 <= cd / ht2_value <= 1.0 + } == {} + + +def test_district_interest_parts_match_table_2_1_lines_8a_and_10(): + """Differential check of the line 8a relabel against Publication 1304. + + Schedule A (2022) line 10 adds lines 8e and 9, and line 8e adds lines 8a + through 8c (8d is reserved). The district's line 8a column is close to + Table 2.1's line 8a, well short of its line 10 total, and the four parts + together come close to line 10. + """ + district = _us_values( + "soi-congressional-district-2022", + "irs_soi.ty2022.congressional_district_2022.all_returns.us.", + ) + table_2_1 = _us_values( + "soi-table-2-1", "irs_soi.ty2022.table_2_1.itemized_all_returns.all." + ) + line_8a = district["mortgage_interest_paid_amount"] + parts = sum(district[f"{part}_amount"] for part in SCHEDULE_A_INTEREST_PARTS) + + assert line_8a / table_2_1["mortgage_interest_paid_amount"] == pytest.approx( + 0.984, abs=0.001 + ) + assert line_8a / table_2_1["interest_paid_deduction_amount"] < 0.85 + assert parts / table_2_1["interest_paid_deduction_amount"] == pytest.approx( + 0.960, abs=0.001 + ) + + +@pytest.mark.parametrize( + ("package_id", "year"), + [ + ("soi-congressional-district-2022", 2022), + ("soi-historic-table-2", 2022), + ("soi-historic-table-2-state-broad-2022", 2022), + ("soi-table-2-1", 2022), + ("soi-table-2-1", 2023), + ], +) +def test_limited_salt_amounts_respect_the_10000_cap(package_id, year): + """Schedule A (2022) line 5e is the smaller of line 5d or $10,000. + + So no row's limited SALT amount can average more than $10,000 a return. + Line 5a, which the district package used to read, averaged $23,098 for the + US. Return counts are rounded to tens; the highest average here is $9,820. + """ + by_row: dict[str, dict[str, float]] = defaultdict(dict) + for record_id, fact in _facts(package_id, year).items(): + row, measure_id = record_id.rsplit(".", 1) + if measure_id.startswith("limited_state_local_taxes_"): + by_row[row][measure_id] = fact.value + + assert by_row + for row, values in by_row.items(): + returns = values["limited_state_local_taxes_returns"] + amount = values["limited_state_local_taxes_amount"] + assert amount <= 10_000 * returns, (row, amount, returns) diff --git a/tests/test_chronicle_source_package.py b/tests/test_chronicle_source_package.py index f4112ef7..4c353a49 100644 --- a/tests/test_chronicle_source_package.py +++ b/tests/test_chronicle_source_package.py @@ -3737,7 +3737,7 @@ def test_soi_congressional_district_2022_builds_all_return_facts(): assert package.package_id == "soi-congressional-district-2022" assert len(rows) == 4_791 assert len(cells) == 79_365 - assert len(facts) == 26_880 + assert len(facts) == 29_760 assert validate_source_rows(rows).valid assert validate_source_cells(cells).valid assert validate_facts(facts).valid @@ -3767,6 +3767,33 @@ def test_soi_congressional_district_2022_builds_all_return_facts(): "us.eitc_three_or_more_children_amount" ] assert eitc_three_or_more_amount.value == 13_600_954_503 + # The IRS guide's columns: limited SALT is Schedule A line 5e (not 5a), + # the PTC is the total premium tax credit (not additional Medicare tax), + # and N19300/A19300 is line 8a mortgage interest (not the line 10 total). + us_columns = { + measure_id: (fact.value, fact.layout.source_column_id) + for record_id, fact in values_by_record.items() + if record_id.startswith( + "irs_soi.ty2022.congressional_district_2022.all_returns.us." + ) + for measure_id in [record_id.rsplit(".", 1)[1]] + } + assert us_columns["limited_state_local_taxes_returns"] == (14_777_040, "N18460") + assert us_columns["limited_state_local_taxes_amount"] == ( + 121_189_294_000, + "A18460", + ) + assert us_columns["premium_tax_credit_returns"] == (7_713_000, "N85770") + assert us_columns["premium_tax_credit_amount"] == (53_073_341_000, "A85770") + assert us_columns["mortgage_interest_paid_returns"] == (11_352_500, "N19300") + assert us_columns["mortgage_interest_paid_amount"] == (140_135_155_000, "A19300") + assert us_columns["home_mortgage_personal_seller_amount"] == ( + 2_583_705_000, + "A19500", + ) + assert us_columns["deductible_points_amount"] == (1_014_806_000, "A19530") + assert us_columns["investment_interest_paid_amount"] == (19_977_693_000, "A19570") + assert "interest_paid_deduction_amount" not in us_columns assert eitc_three_or_more_amount.layout.source_column_id == "A59664" assert eitc_three_or_more_amount.filters["eitc_child_count"] == "3plus" assert { @@ -3861,7 +3888,10 @@ def test_soi_historic_table_2_package_builds_2022_national_facts(): assert schedule_c_returns.value == 30_354_680 assert partnership_scorp.value == 1_033_254_282_000 assert medical_dental.value == 80_235_875_000 - assert qbi.value == 31_307_205_000 + # A04475, the QBI deduction; A03270 (self-employed health insurance) is + # 31_307_205_000. + assert qbi.value == 215_679_578_000 + assert qbi.layout.source_column_id == "A04475" assert rental.value == 85_642_801_000 assert ctc.value == 82_862_736_000 assert actc.value == 33_857_987_000 @@ -3922,7 +3952,8 @@ def test_soi_historic_table_2_state_broad_package_builds_2022_state_facts(): assert ca_ptc.value == 6_379_623_000 assert ca_partnership.value == 125_930_370_000 assert ca_medical.value == 11_456_144_000 - assert ca_qbi.value == 4_400_400_000 + assert ca_qbi.value == 25_995_054_000 + assert ca_qbi.layout.source_column_id == "A04475" assert ca_rental.value == 14_331_993_000 assert ca_ctc.value == 9_724_583_000 assert ca_actc.value == 3_605_628_000 From da557c9ce7983a27839c23ea2096c8071c42994c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 10:51:01 -0400 Subject: [PATCH 2/4] Note that the line 8a rows keep their IRS columns for column-keyed consumers Co-Authored-By: Claude Opus 5.5 --- docs/concept-migrations.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/concept-migrations.md b/docs/concept-migrations.md index 4d54d58c..f8749382 100644 --- a/docs/concept-migrations.md +++ b/docs/concept-migrations.md @@ -122,14 +122,17 @@ labels: `home_mortgage_personal_seller_*` (`N19500`/`A19500`), US row, 0.960 of Table 2.1's. Return counts do not add, because one return can report more than one line. -**Consumer effect.** Microcosm's SOI compiler selects facts by `measure_id` and -skips ids it does not map. On its next Chronicle feed re-pin: +**Consumer effect.** Microcosm's SOI compiler (main, 2026-10-12) selects facts +by `measure_id` and skips ids it does not map. On its next Chronicle feed +re-pin: - The district line 8a rows stop mapping to `interest_deduction`. Microcosm leaves `mortgage_interest_paid_*` unmapped on purpose, so those district targets drop, and its `state_cd` bridge entries for `interest_paid_deduction_*` - stop matching. The 8b, 8c and 9 ids are unmapped too. Whether to build a - district line 10 target from the four parts is Microcosm's decision. + stop matching. The 8b, 8c and 9 ids are unmapped too. How to use the + district rows is Microcosm's decision; the line 8a rows keep + `layout.source_column_id` `N19300`/`A19300`, so a selector keyed on the IRS + column still finds them. - The district SALT and PTC rows carry the corrected columns. microcosm#1224's register keyed on `N18425`/`A18425` and `N85530`/`A85530` stops matching them, and its pinned receipt test needs updating. From d61c069a6cefea1df59f837736e0a622ae5d6925 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 14:28:20 -0400 Subject: [PATCH 3/4] Check every compiled IRS SOI selector against its guide definition Review follow-ups (GPT-6.1 Sol, APPROVE with one P2 and two P3s): - The guide check read only measure.column. The build reads row.column or measure.column and row-level header guards, so the test now walks the compiled source-record specs. Row ranges must stay in one column, and a header guard must sit on the file's header row. A new mutation test moves one Historic Table 2 row to column T and expects the check to name it. - The $10,000 line 5e bound now allows the guides' rounding: counts to the nearest ten, amounts to thousands. - The guide extracts name the "Selected Income and Tax Items" table by title. The Historic Table 2 and county guides call it section F in their contents and G in their body. Co-Authored-By: Claude Opus 5.5 --- docs/concept-migrations.md | 13 +- .../irs_soi_doc_guides/22incddocguide.tsv | 2 +- .../irs_soi_doc_guides/22incmdocguide.tsv | 2 +- .../irs_soi_doc_guides/22incydocguide.tsv | 2 +- .../irs_soi_doc_guides/23incmdocguide.tsv | 2 +- tests/test_chronicle_soi_doc_guide_columns.py | 189 +++++++++++++----- 6 files changed, 150 insertions(+), 60 deletions(-) diff --git a/docs/concept-migrations.md b/docs/concept-migrations.md index f8749382..b87fadfc 100644 --- a/docs/concept-migrations.md +++ b/docs/concept-migrations.md @@ -142,9 +142,10 @@ re-pin: **Pinned by** `tests/test_chronicle_soi_doc_guide_columns.py`. It maps every `measure_id` declared by a package that reads an IRS-variable file (Historic Table 2, congressional-district or county, any vintage) to one guide -definition, and checks each declaration reads the column that definition names. -Measure ids shared across packages must read one column. The line 10 ids may -not be read from a geographic file. Two differential checks compare published -values: the district US row against Historic Table 2's, and the line 8a column -and four-part sum against Table 2.1. A property check holds every limited SALT -row to the $10,000 line 5e cap. +definition. It then checks that every compiled cell selector reads the column +that definition names. Row-level column overrides and header guards count, as +the build reads them. Measure ids shared across packages must read one column. +The line 10 ids may not be read from a geographic file. Two differential checks +compare published values: the district US row against Historic Table 2's, and +the line 8a column and four-part sum against Table 2.1. A property check holds +every limited SALT row to the $10,000 line 5e cap, within publisher rounding. diff --git a/tests/data/irs_soi_doc_guides/22incddocguide.tsv b/tests/data/irs_soi_doc_guides/22incddocguide.tsv index 4f2881a9..b901a74f 100644 --- a/tests/data/irs_soi_doc_guides/22incddocguide.tsv +++ b/tests/data/irs_soi_doc_guides/22incddocguide.tsv @@ -1,4 +1,4 @@ -# IRS SOI Congressional District Data, Tax Year 2022, Documentation Guide, section G (Selected Income and Tax Items). +# IRS SOI Congressional District Data, Tax Year 2022, Documentation Guide, table "Selected Income and Tax Items". # Source: https://www.irs.gov/pub/irs-soi/22incddocguide.docx # sha256: 4e265d3c2499d87428e01bfe2c9e581b444cb6289c7a45397cbfc14a681636ef # Retrieved 2026-10-12. Documents the columns of 22incd.csv. diff --git a/tests/data/irs_soi_doc_guides/22incmdocguide.tsv b/tests/data/irs_soi_doc_guides/22incmdocguide.tsv index 31262aab..0032b495 100644 --- a/tests/data/irs_soi_doc_guides/22incmdocguide.tsv +++ b/tests/data/irs_soi_doc_guides/22incmdocguide.tsv @@ -1,4 +1,4 @@ -# IRS SOI Historic Table 2 (State Data), Tax Year 2022, Documentation Guide, section G (Selected Income and Tax Items). +# IRS SOI Historic Table 2 (State Data), Tax Year 2022, Documentation Guide, table "Selected Income and Tax Items". # Source: https://www.irs.gov/pub/irs-soi/22incmdocguide.doc # sha256: d99db44bc6ca75302dfcf2a0b14a31eb0995709de2e7b39e30e34e306a4c9fa8 # Retrieved 2026-10-12. Documents the columns of 22in55cmcsv.csv. diff --git a/tests/data/irs_soi_doc_guides/22incydocguide.tsv b/tests/data/irs_soi_doc_guides/22incydocguide.tsv index 0b6a3d88..aa0b660e 100644 --- a/tests/data/irs_soi_doc_guides/22incydocguide.tsv +++ b/tests/data/irs_soi_doc_guides/22incydocguide.tsv @@ -1,4 +1,4 @@ -# IRS SOI County Data, Tax Year 2022, Documentation Guide, section G (Selected Income and Tax Items). +# IRS SOI County Data, Tax Year 2022, Documentation Guide, table "Selected Income and Tax Items". # Source: https://www.irs.gov/pub/irs-soi/22incydocguide.docx # sha256: a6acb748d4155614ee4a95d3eb55b7f3d07bf010be269d09d9f06781fbc48787 # Retrieved 2026-10-12. Documents the columns of 22incyallnoagi.csv. diff --git a/tests/data/irs_soi_doc_guides/23incmdocguide.tsv b/tests/data/irs_soi_doc_guides/23incmdocguide.tsv index f22fc13d..0f117dff 100644 --- a/tests/data/irs_soi_doc_guides/23incmdocguide.tsv +++ b/tests/data/irs_soi_doc_guides/23incmdocguide.tsv @@ -1,4 +1,4 @@ -# IRS SOI Historic Table 2 (State Data), Tax Year 2023, Documentation Guide, section G (Selected Income and Tax Items). +# IRS SOI Historic Table 2 (State Data), Tax Year 2023, Documentation Guide, table "Selected Income and Tax Items". # Source: https://www.irs.gov/pub/irs-soi/23incmdocguide.doc # sha256: 942790690b0027b465019ec5d3f5d8f23124882ab85535f06129f74c12021206 # Retrieved 2026-10-12. Documents the columns of 23in55cmcsv.csv. diff --git a/tests/test_chronicle_soi_doc_guide_columns.py b/tests/test_chronicle_soi_doc_guide_columns.py index dc0477fb..2fd96994 100644 --- a/tests/test_chronicle_soi_doc_guide_columns.py +++ b/tests/test_chronicle_soi_doc_guide_columns.py @@ -322,18 +322,25 @@ @dataclass(frozen=True) class ColumnRead: - """One measure declaration in a package that reads an IRS-variable file.""" + """One distinct cell selector of a package that reads an IRS-variable file. + + Built from the compiled source-record specs, so a row-level ``column``, + ``source_column_id`` or header guard counts the way the build reads it + (``row.column or measure.column``). + """ package_dir: str - package_id: str measure_id: str data_file: str data_path: Path column: str header: str - declared: tuple[str, ...] - concept: str | None - label: str | None + declared: tuple[Any, ...] + header_guard_address: str | None + expected_header_address: str + concept: str + label: str + example_record: str def _normalize(text: str) -> str: @@ -375,43 +382,76 @@ def _header(path: Path) -> tuple[str, ...]: return tuple(next(csv.reader([first_line]))) +CELL_ADDRESS = re.compile(r"^([A-Z]+)(\d+)$") + + +def _package_reads(path: Path, package_dir: str) -> tuple[ColumnRead, ...] | None: + """Every distinct selector a package compiles, or None for other parsers.""" + payload = yaml.safe_load(path.read_text()) + artifact = payload["artifact"] + if artifact.get("parser") != "delimited_text_full_rows": + return None + assert artifact.get("artifact_year"), ( + f"{package_dir}: a delimited IRS package without artifact_year can read " + "a different file per year; extend this test to check each" + ) + data_file, data_path = _artifact_file(artifact) + header = _header(data_path) + header_row = artifact.get("header_row", 1) + package = load_source_package(path) + reads: dict[tuple[Any, ...], ColumnRead] = {} + for spec in package.build_source_record_specs(artifact["artifact_year"]): + selector, layout = spec.selector, spec.layout + match = CELL_ADDRESS.match(selector.address) + assert match, spec.source_record_id + column = match.group(1) + # A row range (Historic Table 2 state AGI bands sum adjacent stubs) + # must stay in one column. + if selector.end_address is not None: + end = CELL_ADDRESS.match(selector.end_address) + assert end and end.group(1) == column, spec.source_record_id + declared = tuple( + value + for value in ( + # Without a declared column id the layout repeats the measure id. + None + if layout.source_column_id == layout.measure_id + else layout.source_column_id, + selector.expected_column_header, + ) + if value is not None + ) + read = ColumnRead( + package_dir=package_dir, + measure_id=layout.measure_id, + data_file=data_file, + data_path=data_path, + column=column, + header=header[_column_index(column)], + declared=declared, + header_guard_address=( + selector.expected_column_header_address + if selector.expected_column_header is not None + else None + ), + expected_header_address=f"{column}{header_row}", + concept=spec.concept, + label=layout.measure_label, + example_record=spec.source_record_id, + ) + key = (read.measure_id, read.column, read.declared, read.header_guard_address) + reads.setdefault(key, read) + return tuple(reads.values()) + + @cache def _column_reads() -> tuple[ColumnRead, ...]: - reads: dict[tuple[str, str, str], ColumnRead] = {} + reads: list[ColumnRead] = [] for path in sorted(IRS_SOI_PACKAGES.glob("*/source_package.yaml")): - payload = yaml.safe_load(path.read_text()) - artifact = payload["artifact"] - if artifact.get("parser") != "delimited_text_full_rows": - continue - assert artifact.get("artifact_year"), ( - f"{path.parent.name}: a delimited IRS package without artifact_year " - "can read a different file per year; extend this test to check each" - ) - data_file, data_path = _artifact_file(artifact) - header = _header(data_path) - for record_set in payload["record_sets"]: - for measure in record_set["measures"]: - assert "column_by_year" not in measure, (path.parent.name, measure) - column = measure["column"] - read = ColumnRead( - package_dir=path.parent.name, - package_id=payload["package_id"], - measure_id=measure["measure_id"], - data_file=data_file, - data_path=data_path, - column=column, - header=header[_column_index(column)], - declared=tuple( - measure[key] - for key in ("source_column_id", "expected_column_header") - if measure.get(key) is not None - ), - concept=measure.get("concept"), - label=measure.get("label"), - ) - key = (read.package_dir, read.measure_id, read.column) - assert reads.setdefault(key, read) == read, key - return tuple(reads.values()) + package_reads = _package_reads(path, path.parent.name) + if package_reads is not None: + reads.extend(package_reads) + return tuple(reads) def _defined_variable(measure_id: str, guide_stem: str) -> str | None: @@ -480,10 +520,13 @@ def test_every_definition_names_the_same_variable_in_every_guide(): assert len(named) == 1, (measure_id, variables) -def test_every_measure_reads_the_column_its_guide_definition_names(): +def _column_read_failures(reads: tuple[ColumnRead, ...]) -> list[str]: failures = [] - for read in _column_reads(): - where = f"{read.package_dir}:{read.measure_id} (column {read.column})" + for read in reads: + where = ( + f"{read.package_dir}:{read.measure_id} (column {read.column}, " + f"e.g. {read.example_record})" + ) if read.measure_id in NOT_IN_GEOGRAPHIC_FILES: failures.append( f"{where} reads {read.header}, but the id means " @@ -500,23 +543,66 @@ def test_every_measure_reads_the_column_its_guide_definition_names(): failures.append( f"{where} declares {read.declared}, header is {read.header}" ) + if read.header_guard_address not in {None, read.expected_header_address}: + failures.append( + f"{where} guards header cell {read.header_guard_address}, not " + f"{read.expected_header_address}" + ) guide_stem = GUIDE_BY_DATA_FILE[read.data_file] - if read.header in UNDOCUMENTED_COLUMNS.get(read.data_file, frozenset()): + guide = _guide(guide_stem) + if read.header not in guide: failures.append(f"{where} reads {read.header}, which {guide_stem} omits") continue expected = _defined_variable(read.measure_id, guide_stem) + description, line = guide[read.header] if read.header != expected: - description, line = _guide(guide_stem)[read.header] failures.append( f"{where} reads {read.header} ({description!r}, {line}); " f"{guide_stem} defines {read.measure_id} as {expected}" ) continue - _description, line = _guide(guide_stem)[read.header] if LINE_ERRATA.get(line, line) != GUIDE_DEFINITIONS[read.measure_id][0]: failures.append(f"{where}: {guide_stem} puts {read.header} on line {line}") + return failures + - assert failures == [], "\n".join(failures) +def test_every_measure_reads_the_column_its_guide_definition_names(): + assert _column_read_failures(_column_reads()) == [] + + +def test_a_row_level_column_override_is_checked(tmp_path): + """The build reads ``row.column or measure.column``, so one row can move + every measure in it to another column. The check sees each such cell.""" + payload = yaml.safe_load( + (IRS_SOI_PACKAGES / "historic_table_2" / "source_package.yaml").read_text() + ) + rows = payload["record_sets"][0]["rows"] + assert rows[1]["value_id"] == "under_1" + rows[1]["column"] = "T" # A00100, adjusted gross income + path = tmp_path / "historic_table_2" / "source_package.yaml" + path.parent.mkdir() + path.write_text(yaml.safe_dump(payload, sort_keys=False)) + + reads = _package_reads(path, "historic_table_2") + failures = _column_read_failures(reads) + + assert ( + _column_read_failures( + _package_reads( + IRS_SOI_PACKAGES / "historic_table_2" / "source_package.yaml", + "historic_table_2", + ) + ) + == [] + ) + assert any( + failure.startswith("historic_table_2:return_count (column T, e.g. ") + and ".under_1.return_count" in failure + and "reads A00100" in failure + for failure in failures + ), failures + # The other ten rows still read their own columns and still pass. + assert all(".under_1." in failure for failure in failures) def test_measure_ids_shared_across_packages_read_one_column(): @@ -690,9 +776,12 @@ def test_district_interest_parts_match_table_2_1_lines_8a_and_10(): def test_limited_salt_amounts_respect_the_10000_cap(package_id, year): """Schedule A (2022) line 5e is the smaller of line 5d or $10,000. - So no row's limited SALT amount can average more than $10,000 a return. - Line 5a, which the district package used to read, averaged $23,098 for the - US. Return counts are rounded to tens; the highest average here is $9,820. + Each return's limited SALT is at most $10,000, so a row's amount is at most + $10,000 times its returns. The geographic files round return counts to the + nearest ten and every package publishes amounts in thousands, so the bound + allows five more returns and $500. Line 5a, which the district package used + to read, averaged $23,098 a return for the US; the highest average on the + corrected columns is $9,820. """ by_row: dict[str, dict[str, float]] = defaultdict(dict) for record_id, fact in _facts(package_id, year).items(): @@ -704,4 +793,4 @@ def test_limited_salt_amounts_respect_the_10000_cap(package_id, year): for row, values in by_row.items(): returns = values["limited_state_local_taxes_returns"] amount = values["limited_state_local_taxes_amount"] - assert amount <= 10_000 * returns, (row, amount, returns) + assert amount <= 10_000 * (returns + 5) + 500, (row, amount, returns) From c542be2583c9b9f2f22816c0db3dc0c8452f5921 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 14:45:50 -0400 Subject: [PATCH 4/4] Close the re-review's coverage gaps in the IRS guide-column check Re-review of d61c069a (GPT-6.1 Sol, APPROVE, three P3s): - An explicit source_column_id equal to the measure id looked like the layout's fallback. The check now takes the declared id from the compiled record set (row before measure), so it is checked against the header. - Deduplication could drop a selector whose concept or label differed. Both are now in the key, and a measure id with more than one concept or label in a package fails. - Divisor selectors were not inspected. These packages publish levels, so a divisor column now fails until the test checks divisors. The mutation test is parametrized over all four cases on a temp copy of historic_table_2. Co-Authored-By: Claude Opus 5.5 --- tests/test_chronicle_soi_doc_guide_columns.py | 212 ++++++++++++------ 1 file changed, 145 insertions(+), 67 deletions(-) diff --git a/tests/test_chronicle_soi_doc_guide_columns.py b/tests/test_chronicle_soi_doc_guide_columns.py index 2fd96994..b4188daf 100644 --- a/tests/test_chronicle_soi_doc_guide_columns.py +++ b/tests/test_chronicle_soi_doc_guide_columns.py @@ -35,6 +35,7 @@ import csv import re from collections import defaultdict +from copy import deepcopy from dataclasses import dataclass from functools import cache from importlib.resources import files @@ -45,6 +46,7 @@ import yaml from chronicle.source_package import load_source_package +from chronicle.sources.specs import compile_source_record_set_specs REPO_ROOT = Path(__file__).resolve().parents[1] IRS_SOI_PACKAGES = REPO_ROOT / "packages" / "irs_soi" @@ -338,6 +340,7 @@ class ColumnRead: declared: tuple[Any, ...] header_guard_address: str | None expected_header_address: str + divisor_column: str | None concept: str label: str example_record: str @@ -400,47 +403,68 @@ def _package_reads(path: Path, package_dir: str) -> tuple[ColumnRead, ...] | Non header_row = artifact.get("header_row", 1) package = load_source_package(path) reads: dict[tuple[Any, ...], ColumnRead] = {} - for spec in package.build_source_record_specs(artifact["artifact_year"]): - selector, layout = spec.selector, spec.layout - match = CELL_ADDRESS.match(selector.address) - assert match, spec.source_record_id - column = match.group(1) - # A row range (Historic Table 2 state AGI bands sum adjacent stubs) - # must stay in one column. - if selector.end_address is not None: - end = CELL_ADDRESS.match(selector.end_address) - assert end and end.group(1) == column, spec.source_record_id - declared = tuple( - value - for value in ( - # Without a declared column id the layout repeats the measure id. - None - if layout.source_column_id == layout.measure_id - else layout.source_column_id, - selector.expected_column_header, + year = artifact["artifact_year"] + for record_set in package.build_source_record_set_specs(year): + rows = {row.value_id: row for row in record_set.rows} + measures = {measure.measure_id: measure for measure in record_set.measures} + for spec in compile_source_record_set_specs(record_set): + selector, layout = spec.selector, spec.layout + match = CELL_ADDRESS.match(selector.address) + assert match, spec.source_record_id + column = match.group(1) + # A row range (Historic Table 2 state AGI bands sum adjacent stubs) + # must stay in one column. + if selector.end_address is not None: + end = CELL_ADDRESS.match(selector.end_address) + assert end and end.group(1) == column, spec.source_record_id + # The column id the package declares, row before measure, as the + # build resolves it. Without one the layout repeats the measure id, + # which is not a declaration. + declared_id = ( + rows[layout.groupby_value_id].source_column_id + or measures[layout.measure_id].source_column_id ) - if value is not None - ) - read = ColumnRead( - package_dir=package_dir, - measure_id=layout.measure_id, - data_file=data_file, - data_path=data_path, - column=column, - header=header[_column_index(column)], - declared=declared, - header_guard_address=( - selector.expected_column_header_address - if selector.expected_column_header is not None - else None - ), - expected_header_address=f"{column}{header_row}", - concept=spec.concept, - label=layout.measure_label, - example_record=spec.source_record_id, - ) - key = (read.measure_id, read.column, read.declared, read.header_guard_address) - reads.setdefault(key, read) + declared = tuple( + value + for value in (declared_id, selector.expected_column_header) + if value is not None + ) + divisor = spec.divisor_selector + read = ColumnRead( + package_dir=package_dir, + measure_id=layout.measure_id, + data_file=data_file, + data_path=data_path, + column=column, + header=header[_column_index(column)], + declared=declared, + header_guard_address=( + selector.expected_column_header_address + if selector.expected_column_header is not None + else None + ), + expected_header_address=f"{column}{header_row}", + divisor_column=( + CELL_ADDRESS.match(divisor.address).group(1) + if divisor is not None + else None + ), + concept=spec.concept, + label=layout.measure_label, + example_record=spec.source_record_id, + ) + # Every field but the example record: two selectors that differ in + # anything checked below both stay. + key = ( + read.measure_id, + read.column, + read.declared, + read.header_guard_address, + read.divisor_column, + read.concept, + read.label, + ) + reads.setdefault(key, read) return tuple(reads.values()) @@ -522,6 +546,15 @@ def test_every_definition_names_the_same_variable_in_every_guide(): def _column_read_failures(reads: tuple[ColumnRead, ...]) -> list[str]: failures = [] + meanings: dict[tuple[str, str], set[tuple[str, str]]] = defaultdict(set) + for read in reads: + meanings[(read.package_dir, read.measure_id)].add((read.concept, read.label)) + for (package_dir, measure_id), declared in sorted(meanings.items()): + if len(declared) > 1: + failures.append( + f"{package_dir}:{measure_id} declares more than one concept or " + f"label: {sorted(declared)}" + ) for read in reads: where = ( f"{read.package_dir}:{read.measure_id} (column {read.column}, " @@ -543,6 +576,11 @@ def _column_read_failures(reads: tuple[ColumnRead, ...]) -> list[str]: failures.append( f"{where} declares {read.declared}, header is {read.header}" ) + if read.divisor_column is not None: + failures.append( + f"{where} divides by column {read.divisor_column}; these packages " + "publish levels, so extend this test before adding a divisor" + ) if read.header_guard_address not in {None, read.expected_header_address}: failures.append( f"{where} guards header cell {read.header_guard_address}, not " @@ -570,39 +608,79 @@ def test_every_measure_reads_the_column_its_guide_definition_names(): assert _column_read_failures(_column_reads()) == [] -def test_a_row_level_column_override_is_checked(tmp_path): - """The build reads ``row.column or measure.column``, so one row can move - every measure in it to another column. The check sees each such cell.""" - payload = yaml.safe_load( - (IRS_SOI_PACKAGES / "historic_table_2" / "source_package.yaml").read_text() +def _move_a_row_to_agi(payload: dict[str, Any]) -> None: + # The build reads ``row.column or measure.column``, so one row can move + # every measure in it to another column. + row = payload["record_sets"][0]["rows"][1] + assert row["value_id"] == "under_1" + row["column"] = "T" # A00100, adjusted gross income + + +def _declare_the_measure_id_as_column_id(payload: dict[str, Any]) -> None: + # Indistinguishable in the layout from the fallback for no declaration. + _ht2_measure(payload, "return_count")["source_column_id"] = "return_count" + + +def _divide_by_agi(payload: dict[str, Any]) -> None: + _ht2_measure(payload, "return_count")["divisor_column"] = "T" + + +def _relabel_in_a_second_record_set(payload: dict[str, Any]) -> None: + record_set = deepcopy(payload["record_sets"][0]) + for key in ("record_set_id", "record_set_spec_id", "source_record_id_prefix"): + record_set[key] += "_copy" + _ht2_measure({"record_sets": [record_set]}, "qbi_amount")["concept"] = ( + "irs_soi.self_employed_health_insurance_deduction" + ) + payload["record_sets"].append(record_set) + + +def _ht2_measure(payload: dict[str, Any], measure_id: str) -> dict[str, Any]: + return next( + measure + for measure in payload["record_sets"][0]["measures"] + if measure["measure_id"] == measure_id ) - rows = payload["record_sets"][0]["rows"] - assert rows[1]["value_id"] == "under_1" - rows[1]["column"] = "T" # A00100, adjusted gross income + + +@pytest.mark.parametrize( + ("mutate", "expected"), + [ + ( + _move_a_row_to_agi, + "historic_table_2:return_count (column T, e.g. " + "irs_soi.ty2022.historic_table_2.us.under_1.return_count) reads A00100", + ), + ( + _declare_the_measure_id_as_column_id, + "historic_table_2:return_count (column C, e.g. " + "irs_soi.ty2022.historic_table_2.us.all.return_count) declares " + "('return_count',), header is N1", + ), + ( + _divide_by_agi, + "historic_table_2:return_count (column C, e.g. " + "irs_soi.ty2022.historic_table_2.us.all.return_count) divides by column T", + ), + ( + _relabel_in_a_second_record_set, + "historic_table_2:qbi_amount declares more than one concept or label", + ), + ], +) +def test_package_mutations_are_caught(tmp_path, mutate, expected): + """Each mutation compiles, so only this module's checks can catch it.""" + source = IRS_SOI_PACKAGES / "historic_table_2" / "source_package.yaml" + payload = yaml.safe_load(source.read_text()) + mutate(payload) path = tmp_path / "historic_table_2" / "source_package.yaml" path.parent.mkdir() path.write_text(yaml.safe_dump(payload, sort_keys=False)) - reads = _package_reads(path, "historic_table_2") - failures = _column_read_failures(reads) + failures = _column_read_failures(_package_reads(path, "historic_table_2")) - assert ( - _column_read_failures( - _package_reads( - IRS_SOI_PACKAGES / "historic_table_2" / "source_package.yaml", - "historic_table_2", - ) - ) - == [] - ) - assert any( - failure.startswith("historic_table_2:return_count (column T, e.g. ") - and ".under_1.return_count" in failure - and "reads A00100" in failure - for failure in failures - ), failures - # The other ten rows still read their own columns and still pass. - assert all(".under_1." in failure for failure in failures) + assert _column_read_failures(_package_reads(source, "historic_table_2")) == [] + assert any(failure.startswith(expected) for failure in failures), failures def test_measure_ids_shared_across_packages_read_one_column():