diff --git a/changelog.d/us-humanitarian-immigration-statuses-767.added.md b/changelog.d/us-humanitarian-immigration-statuses-767.added.md new file mode 100644 index 0000000000..d2717d8472 --- /dev/null +++ b/changelog.d/us-humanitarian-immigration-statuses-767.added.md @@ -0,0 +1 @@ +Impute humanitarian and temporary-protection immigration statuses (REFUGEE, ASYLEE, PAROLED_ONE_YEAR per program origin, TPS per country) in the US immigration stage, drawn to cited DHS/OHSS/CBP/CRS stocks, so the enacted H.R.1 §71109/§71301/§71302 and SNAP §10108 eligibility channels stop computing $0; gate each category against its target and add four release-blocking reform-coverage probes. In a stacked build, a draw whose ACS candidates cannot cover its residual takes every candidate, as the ASEC stage's draws do, and its receipt records the shortfall; the transfer refuses only a draw that ends below 0.35 of its target; Operation Allies Welcome parole takes exact-year ACS arrivals from 2021 and 2022 only. Attested schema-9 legacy scoring takes the immigration stage's and its transfer group's contracts from a frozen schema-16 schedule; its other contracts follow today's registry, as on main. diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index 5a20b58ca7..933c68d56a 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -4,9 +4,9 @@ "version": 1 }, "country": "us", - "documentation_sha256": "fda6ec0dd925077f4d84d30a1c206dd81d82c5e16bd94506e81fe1b9f7e2f466", + "documentation_sha256": "8b9068a9bf9887d7f383ff08851bddac05e4c5771332c71819ff3a180e748823", "field_usage": { - "authored_normative_field_count": 32609, + "authored_normative_field_count": 32749, "claim_count": 48, "claims": [ { @@ -16,8 +16,8 @@ "legacy_sinks": [], "mode": "front_end_validation", "pointer_class": "all", - "pointer_count": 104, - "pointer_sha256": "8a186065f5b8ffc59bc3f62fe927975e6f36bc1e913aa61761652c9a8aa67988", + "pointer_count": 107, + "pointer_sha256": "2c2099d3d5ef3d664f74a2e732870391022a852cf49d5cf96810c9a984d2b20b", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/country_package.json", @@ -206,8 +206,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 110, - "pointer_sha256": "4e1cdee017270e10a123df1e37be3d7c3a88056684c09eace90bd102c7d9ec83", + "pointer_count": 128, + "pointer_sha256": "072e12dcb7b0ec4af2d954c3280cca7ebad3496826b20de3bbc713ec1e9344e9", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/spec~1imputation.yaml/transfer_execution", @@ -260,8 +260,8 @@ ], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 24511, - "pointer_sha256": "5869ddc4afa6eb0c09f290b709e75467decc6929e7f79c98341878928e6169fc", + "pointer_count": 24577, + "pointer_sha256": "cd6f3f0bb0cca334bec3c133060352b1b99a4e7be118924de8741a9da748325c", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/spec~1imputation.yaml/producer_graph", @@ -500,8 +500,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 1926, - "pointer_sha256": "8bd6122267288e2df25b2880a43191865707a240892bec484cb830ee72404964", + "pointer_count": 1939, + "pointer_sha256": "0e43f91c8617181bdceeef7570dd5a7e871f77e1acd3961f9b7203f825f20882", "rationale": null, "relative_sink_prefix": "/source_manifest/stages", "source_prefix": "/authored/spec~1sources.yaml/stages", @@ -516,8 +516,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 88, - "pointer_sha256": "ae011455154bae0df3913ca9a056058a909d913439b292cb46c11f63c7d0d9a3", + "pointer_count": 89, + "pointer_sha256": "8d281f8f1e2f9914684150b0424ff816d0ec09ed0d9053ea5a64e877a86cc35b", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/spec~1spine.yaml/pipeline_contract", @@ -530,8 +530,8 @@ "legacy_sinks": [], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 277, - "pointer_sha256": "dfa7ae701f62f7b1f06954798d29cfba3fc53a0be21e1751646d1f5bafd6071f", + "pointer_count": 316, + "pointer_sha256": "bb848f38e068858ac23e964a4ffbbf5542253da50515c72fe7753d8c56912486", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/spec~1spine.yaml/seed_site_bindings", @@ -722,8 +722,8 @@ "legacy_sinks": [], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 827, - "pointer_sha256": "525ed8621baa2c6e0c0970a6cec85b3b250f8aadfe93750a08deb0d9f240f15e", + "pointer_count": 1035, + "pointer_sha256": "0da36c3d74127193e82c128071fb687028d4e6cf096f045c8d96d3c52391bc41", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/resolved/seed_protocol", @@ -736,8 +736,8 @@ "legacy_sinks": [], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 277, - "pointer_sha256": "368fc19a8f07c4abf38ebcf2fbc4414d33c403c0f1f9881c2d2e3f5d6160feb6", + "pointer_count": 316, + "pointer_sha256": "199ec98045871070860164b522892327ab840b48c0c6a53972ebc0ff0d7d091f", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/resolved/seed_site_bindings", @@ -758,21 +758,21 @@ "verifier": "vintages" } ], - "configuration_field_count": 42394, - "consumed_field_count": 42394, + "configuration_field_count": 42781, + "consumed_field_count": 42781, "generation0_effect_counts": { - "legacy_behavior": 38780, - "no_generation0_effect": 3614 + "legacy_behavior": 38878, + "no_generation0_effect": 3903 }, "mode_counts": { - "compiler_semantic": 27673, - "front_end_validation": 349, + "compiler_semantic": 28025, + "front_end_validation": 352, "identity_only": 103, - "legacy_behavior": 14269 + "legacy_behavior": 14301 }, "multiple_primary_use_field_count": 0, - "pointer_inventory_sha256": "f6832e8d2892af22e41afd0b0bea0cba354bbff8b84bebcc652e765a3ae7df92", - "resolved_binding_field_count": 9785, + "pointer_inventory_sha256": "439d805c20da730deaa1aec21b94b635d2667075dda4aec542815d4a84e8a5b3", + "resolved_binding_field_count": 10032, "unused_field_count": 0 }, "inventory_coverage": { @@ -796,14 +796,14 @@ "primary_targets": 66, "producer_authored_outputs": 92, "producer_compiled_outputs": 230, - "producer_inputs": 2747, + "producer_inputs": 2753, "producer_nodes": 38, "producer_virtual_resources": 75, "release_rungs": 5, "resolved_references": 337, - "seed_owner_bindings": 112, + "seed_owner_bindings": 125, "seed_owner_rows": 54, - "seed_sites": 53, + "seed_sites": 66, "seed_streams": 14, "source_operators": 16, "source_stages": 38, @@ -1060,11 +1060,11 @@ "legacy_adapter.stacked_checkpoint_static_components" ], "expected": { - "schedule_sha256": "f1fb754e6ade602be394837bc0d7d091f2fba4b4b83fb00f59a0258fe56eb5da" + "schedule_sha256": "46cea046d2fb8223d32772bd6a7807c52791640943949154982dcc52d93fa9eb" }, "failures": [], "observed": { - "schedule_sha256": "f1fb754e6ade602be394837bc0d7d091f2fba4b4b83fb00f59a0258fe56eb5da" + "schedule_sha256": "46cea046d2fb8223d32772bd6a7807c52791640943949154982dcc52d93fa9eb" }, "status": "covered" }, @@ -1236,14 +1236,14 @@ "expected": { "edges": 71, "nodes": 38, - "schedule_sha256": "f1fb754e6ade602be394837bc0d7d091f2fba4b4b83fb00f59a0258fe56eb5da", + "schedule_sha256": "46cea046d2fb8223d32772bd6a7807c52791640943949154982dcc52d93fa9eb", "waves": 6 }, "failures": [], "observed": { "edges": 71, "nodes": 38, - "schedule_sha256": "f1fb754e6ade602be394837bc0d7d091f2fba4b4b83fb00f59a0258fe56eb5da", + "schedule_sha256": "46cea046d2fb8223d32772bd6a7807c52791640943949154982dcc52d93fa9eb", "waves": 6 }, "status": "covered" @@ -1261,11 +1261,11 @@ ], "expected": { "relation": "source rows preserved exactly", - "rows": 2747 + "rows": 2753 }, "failures": [], "observed": { - "rows": 2747 + "rows": 2753 }, "status": "covered" }, @@ -1307,7 +1307,7 @@ "expected": "complete execution-row and transition-authority object", "failures": [], "observed": { - "sha256": "9a09b49855b3dc4c37ac86c3b91293b6944a50f4448354701b18db8f659ae121" + "sha256": "e7f16e205ae8c7604d5b021f046724441533222c89f42ab7765620e9b8a36091" }, "status": "covered" }, @@ -1324,12 +1324,12 @@ ], "expected": { "nodes": 38, - "sha256": "1b208fff4f6d878c522147fee617177544bda8963c940f98a4bfc37507cb955f" + "sha256": "2606b852c3ecdc7ae0f7ff5b30c24a5ce6834b218db0898b44b18b906daa9558" }, "failures": [], "observed": { "nodes": 38, - "sha256": "1b208fff4f6d878c522147fee617177544bda8963c940f98a4bfc37507cb955f" + "sha256": "2606b852c3ecdc7ae0f7ff5b30c24a5ce6834b218db0898b44b18b906daa9558" }, "status": "covered" }, @@ -1347,12 +1347,12 @@ ], "expected": { "producer_count": 38, - "sha256": "4464b646c45dff39a500e07bf60a8505d45725712fc27dcfc6de1e839fd6b0c3" + "sha256": "5653131d24487b1779e7c0a60deaed10d3a2a0b5a876c65dda40a6df0ef4e436" }, "failures": [], "observed": { "producer_count": 38, - "sha256": "4464b646c45dff39a500e07bf60a8505d45725712fc27dcfc6de1e839fd6b0c3" + "sha256": "5653131d24487b1779e7c0a60deaed10d3a2a0b5a876c65dda40a6df0ef4e436" }, "status": "covered" }, @@ -1519,7 +1519,7 @@ "expected": { "groups": 12, "partition": "disjoint and exhaustive", - "sites": 53 + "sites": 66 }, "failures": [], "observed": { @@ -1536,6 +1536,19 @@ "pregnancy_assignment", "wic_claim_assignment", "snap_discretionary_exemption_assignment", + "immigration_humanitarian_paroled_one_year_afghanistan_assignment", + "immigration_humanitarian_paroled_one_year_ukraine_assignment", + "immigration_humanitarian_paroled_one_year_nicaragua_assignment", + "immigration_humanitarian_paroled_one_year_venezuela_assignment", + "immigration_humanitarian_refugee_assignment", + "immigration_humanitarian_asylee_assignment", + "immigration_humanitarian_deportation_withheld_assignment", + "immigration_humanitarian_tps_venezuela_assignment", + "immigration_humanitarian_tps_el_salvador_assignment", + "immigration_humanitarian_tps_honduras_assignment", + "immigration_humanitarian_tps_nicaragua_assignment", + "immigration_humanitarian_tps_nepal_assignment", + "immigration_humanitarian_tps_other_designated_assignment", "immigration_ead_workers_assignment", "immigration_ead_students_assignment", "ssi_take_up_assignment", @@ -1602,7 +1615,7 @@ "sipp_tip_training_cap" ] }, - "sites": 53 + "sites": 66 }, "status": "covered" }, @@ -1642,13 +1655,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "fa48af060da522bb1edcb80580098f258763757139292602759cbccbfff76b37", - "protocol_sha256": "5840ac6cefd5ce3d2172aca9e3bfc08d2344e97a8e9b42a723a4edcaa9f8ff53" + "map_sha256": "4556389f3383a2d2600857cf7117b9306f8f45a9a6231ce2a8fe62bb6a5e388c", + "protocol_sha256": "0f4550f80efd0273589b533a4a453e74b06aa71b9ed1e08fdc016ef3d9d27fef" }, "failures": [], "observed": { - "map_sha256": "fa48af060da522bb1edcb80580098f258763757139292602759cbccbfff76b37", - "protocol_sha256": "5840ac6cefd5ce3d2172aca9e3bfc08d2344e97a8e9b42a723a4edcaa9f8ff53" + "map_sha256": "4556389f3383a2d2600857cf7117b9306f8f45a9a6231ce2a8fe62bb6a5e388c", + "protocol_sha256": "0f4550f80efd0273589b533a4a453e74b06aa71b9ed1e08fdc016ef3d9d27fef" }, "status": "covered" }, @@ -1663,7 +1676,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "5840ac6cefd5ce3d2172aca9e3bfc08d2344e97a8e9b42a723a4edcaa9f8ff53", + "implementation_sha256": "0f4550f80efd0273589b533a4a453e74b06aa71b9ed1e08fdc016ef3d9d27fef", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1684,7 +1697,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "5840ac6cefd5ce3d2172aca9e3bfc08d2344e97a8e9b42a723a4edcaa9f8ff53", + "implementation_sha256": "0f4550f80efd0273589b533a4a453e74b06aa71b9ed1e08fdc016ef3d9d27fef", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1718,12 +1731,12 @@ ], "expected": { "relation": "all site fields preserved exactly", - "sites": 53 + "sites": 66 }, "failures": [], "observed": { - "sha256": "a0366cd518f42a240b589af3ef6580aac44ad7ed70969bcc669b365d315146ab", - "sites": 53 + "sha256": "5c22dfb6f3cd7c23c527e054ba6f5040ca2eb4235df8659e3f7c61ebe2ae6c4f", + "sites": 66 }, "status": "covered" }, @@ -1738,12 +1751,12 @@ "compiler_ir.seed_stream_map.sites.owners" ], "expected": { - "bindings": 112, - "coverage": "all 53 sites" + "bindings": 125, + "coverage": "all 66 sites" }, "failures": [], "observed": { - "bindings": 112 + "bindings": 125 }, "status": "covered" }, @@ -1761,12 +1774,12 @@ "legacy_adapter.imputation.late_producer_resource_semantics" ], "expected": { - "sha256": "dc3879886d2a69f20fb982a675d7fb322c0e4017afef6471be54b0ee3cbd2af6", + "sha256": "e9a70a60c3db9250066ee607d5e1fb07cd9d6a54424ab9eb9405f5e65b5a59b8", "stages": 38 }, "failures": [], "observed": { - "sha256": "dc3879886d2a69f20fb982a675d7fb322c0e4017afef6471be54b0ee3cbd2af6", + "sha256": "e9a70a60c3db9250066ee607d5e1fb07cd9d6a54424ab9eb9405f5e65b5a59b8", "stages": 38 }, "status": "covered" @@ -1826,11 +1839,11 @@ "legacy_adapter.stacked_checkpoint_static_components" ], "expected": { - "sha256": "bf681aedc859fd6398ef72da68f584069b7d25b3085c549bf99f82e8324c4eef" + "sha256": "7cdedd5bf056f1407910eddcd40ad0660bd7ef2d6defe224612c02b3476cbf15" }, "failures": [], "observed": { - "sha256": "bf681aedc859fd6398ef72da68f584069b7d25b3085c549bf99f82e8324c4eef" + "sha256": "7cdedd5bf056f1407910eddcd40ad0660bd7ef2d6defe224612c02b3476cbf15" }, "status": "covered" }, @@ -1873,7 +1886,7 @@ "alpha", "zeta" ], - "sha256": "ecf64f2bfbb59ffaaa717d8a09d38e56a38d6d43c1aa0a209d57492b038fb19a" + "sha256": "6aefc9bd351e4ca81998012de0eceacab9c802334d1ae2582f29cde4a6cd5036" }, "failures": [], "observed": { @@ -1897,7 +1910,7 @@ "alpha", "zeta" ], - "sha256": "ecf64f2bfbb59ffaaa717d8a09d38e56a38d6d43c1aa0a209d57492b038fb19a" + "sha256": "6aefc9bd351e4ca81998012de0eceacab9c802334d1ae2582f29cde4a6cd5036" }, "status": "covered" }, @@ -1961,7 +1974,7 @@ "take_up_contract", "us_qbi_reconciliation_contract" ], - "sha256": "731c8c33dc511dfc35bc4b37bed9864ee75446e45555b999a01ec9b6ad1289cf" + "sha256": "6e015f560238e801f4ef3263c05069ce4981078b620d260a2d37170e621e3beb" }, "status": "covered" }, @@ -2585,7 +2598,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "4ccbd01720871e59ad8e3c00e1c1296231facef91e83c561076ee499e0a138b2" + "spec_sha256": "783ff9a423a3de4db335c5886e37139ac94b0ae619528433e08a273c17bb111f" } }, "report_schema_version": 3, @@ -2595,7 +2608,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "4ccbd01720871e59ad8e3c00e1c1296231facef91e83c561076ee499e0a138b2" + "spec_sha256": "783ff9a423a3de4db335c5886e37139ac94b0ae619528433e08a273c17bb111f" }, "status": "pass" } diff --git a/docs/evidence/us-immigration-statuses-767/README.md b/docs/evidence/us-immigration-statuses-767/README.md new file mode 100644 index 0000000000..1edea9608d --- /dev/null +++ b/docs/evidence/us-immigration-statuses-767/README.md @@ -0,0 +1,292 @@ +# US immigration statuses: default-pool stage receipts (microcosm#767) + +PR #779 draws humanitarian and temporary-protection statuses in the US +immigration source stage (`us_runtime/immigration.py`). +[`docs/us-release-build-rule.md`](../../us-release-build-rule.md) asks a pull +request that changes how Microcosm builds the dataset to carry a from-scratch +build from its own tree before it merges. These receipts run that one stage +alone, on the inputs the next certified build uses, at this PR's head and at +main. Max decides whether they stand in for the from-scratch run (d2729, on the +terms of d2647). Nothing here changes a build. + +## What the script runs + +`scripts/stage_receipt_pooled.py` runs the stage on the three Census ASEC person +files of main's default pool: the 2024, 2025 and 2026 ASEC, income years 2023 to +2025 (`asec_sources.ASEC_DEFAULT_POOL_INCOME_YEARS`). Max's ruling d2603 moves +the next certified build to that pool. The script: + +- checks each archive against the SHA-256 that + `us_runtime/education_assistance_source.py` pins; +- weights each person by the household's `HSUP_WGT` times 0.01 and rescales each + file to one third of income year 2024's weighted population, as + `asec_pool.pool_asec_sources` does when `build_pooled_asec_unit_frame` pools + for target year 2024; +- gives each person the pool's lineage columns, `source_year` (the income + year), `source_household_id` (`PH_SEQ`) and `source_person_id` (`PERIDNUM`), + and leaves the tree's kernel to choose which of them key its draws; +- records, for each draw call, the key that reproduces its draws; +- runs at seed 0 and period 2024, the `--target-year` of + `tools/route_a/route_a.sh`; +- runs that tree's own composition gate on the output. + +`--draw-keys full` hands all three lineage columns to every draw call that +brings no key of its own, and `--draw-keys legacy` hands it `source_year` and +`source_person_id`. Two trees run on one key share their draws and differ only +in method. + +The receipts hold aggregates only. Each records the tree's commit, the SHA-256 +of its `immigration.py` and of the script, and every draw call's key. + +## Receipts + +| File | Tree | Draw keys | Result | +|---|---|---|---| +| `receipt.head.json` | this PR at `4f94e489a`, which last merged main at `764ae3aa7` | its own: income year, `PH_SEQ`, `PERIDNUM` | its gate passes with no failures | +| `receipt.main.json` | main at `764ae3aa7` | its own: income year, `PERIDNUM` | main draws no humanitarian status; its gate passes | +| `receipt.main_full_keys.json` | main at `764ae3aa7` | this PR's, through `--draw-keys full` | main's method on this PR's draws; its gate passes | +| `receipt.pre_sync_42bb166b.json` | this PR at `42bb166b1`, before it merged main `1acd884fe` | none: the stage refuses before its first draw | the stage refuses the 2026 ASEC's `PEINUSYR` code 29 | +| `receipt.head_medicaid_reporters.json` | this PR at `4f94e489a`, from the head run's labels | its own | 0.391 million of the 0.714 million people in the restored Medicaid statuses report Medicaid at interview | +| `receipt.stacked_supply.json` | this PR at `4f94e489a`, from `receipt.head.json` and the ACS 2024 1-year PUMS | none: it draws no rows | five draws run short of ACS candidates; none ends below half its target in the worst case | + +The script reran main's tree and the pre-sync tree from `git archive +packages`, so those three receipts take their commit from `--source-commit`. +Their `immigration_module_sha256` values equal the SHA-256 of `immigration.py` +at those commits. + +## Results + +Weighted persons, in millions: + +| Status | Main | This PR | Cited target | +|---|---:|---:|---:| +| Citizen | 300.244 | 300.244 | | +| Lawful permanent resident | 11.680 | 10.427 | | +| Undocumented | 13.071 | 12.891 | | +| DACA | 0.376 | 0.277 | | +| Cuban/Haitian entrant | 0.581 | 0.539 | | +| Paroled, one year | 0 | 0.398 | 0.442 | +| Refugee | 0 | 0.160 | 0.160 | +| Asylee | 0 | 0.155 | 0.155 | +| TPS | 0 | 0.860 | 0.858 | +| All | 325.952 | 325.952 | | + +- Ten of the twelve positive-target draws emit within 3% of their targets. + Nicaraguan TPS emits 3,193 against 2,910. The U4U Ukrainian parole draw emits + 112,934 against 158,000: it exhausts its candidates, Ukraine-born non-citizens + in arrival codes 28 and 29, at 0.71. The composition gate's band runs from + 0.35 to 1.5 of each target. Deportation withheld carries a cited zero and + emits none. +- Pew's broad unauthorized universe totals 14.574 million against its 14.0 + million anchor, and that universe's labor force 9.699 million against the 9.7 + million control. +- In both trees `ssn_card_type` is `NONE` exactly on the undocumented rows. + +## Medicaid at interview in the restored statuses + +The reform-coverage probe `hr1_medicaid_humanitarian_eligibility_restoration` +restores the pre-H.R.1 Medicaid status list at 2027 law and counts the people +it enrolls. `scripts/medicaid_reporters.py` joins the head run's labels to the +ASEC's `NOW_MCAID` (Medicaid at interview), which `us_runtime/cps_carried.py` +maps to `has_medicaid_health_coverage_at_interview`, the column the Medicaid +take-up stage anchors on. `receipt.head_medicaid_reporters.json` holds the +aggregates, in millions of weighted people: + +| Status | People | Report Medicaid at interview | +|---|---:|---:| +| Refugee | 0.160 | 0.121 | +| Asylee | 0.155 | 0.107 | +| Paroled, one year | 0.398 | 0.163 | +| All restored | 0.714 | 0.391 | + +The stage draws no deportation-withheld or conditional-entrant rows. The +probe's 10,000-person floor is 2.6% of the 0.391 million reporters. + +## The stacked pool's ACS residuals + +The `--exact-k` arm of `tools/build_us_fiscal_refresh_release.py`, the only arm +that runs the improvement gate, takes a pool from +`tools/build_us_multispine_pool.py`, whose default pipeline stacks the ASEC pool +and the ACS 2024 1-year. Each arm carries half the ASEC pool's household mass +(`stacked_spine.DEFAULT_STACKED_HOUSEHOLD_MASS_SHARES`), and the immigration +stage labels the ASEC rows at the full pool's person mass, so those rows carry +about half of each draw's target. The ACS transfer's +`reconcile_us_immigration_humanitarian_transfer` then asks the ACS rows for the +rest, each draw's residual. + +The reconciliation used to refuse any draw whose ACS candidates held less +weight than its residual. Now such a draw takes every candidate, as the ASEC +stage's draws do, and its receipt records the shortfall. The reconciliation +refuses only a draw that ends below 0.35 of its target, the floor of the +composition gate's band, and the stacked validator replays the same rule. +Operation Allies Welcome (OAW) parole now takes exact-year ACS arrivals from +2021 and 2022 only, the program's arrival years; its window used to stay open so +that the old rule would not refuse it. + +`scripts/stacked_supply.py` projects each draw's ACS supply without building a +stacked pool. It takes the ASEC arm's labelled mass from `receipt.head.json`, +scaled to the stacked weights, and each draw's candidates from the ACS 2024 +1-year PUMS through the kernel's own candidate rule, weighted as the frame +weights them and scaled by the ACS arm's weight scale, 0.478. +`receipt.stacked_supply.json` holds the aggregates, in weighted persons. The +projected share adds to the ASEC arm as much of the ACS supply as the residual +takes, as a share of the target, if no earlier draw takes any of the draw's +candidates. The expected share lets each earlier parole or TPS draw take its +residual from the candidates still free, each with the same chance, as the +reconciliation's person-keyed hash draws do; it leaves out what refugee and +asylee draws take. The worst case lets earlier draws take as much as they can +(below): + +| Draw | Target | ASEC arm | ACS residual | ACS supply | Projected share | Expected share | Worst case | +|---|---:|---:|---:|---:|---:|---:|---:| +| Parole, Afghanistan (OAW) | 73,566 | 37,128 | 36,438 | 34,292 | 0.971 | 0.971 | 0.971 | +| Parole, Ukraine (U4U) | 158,000 | 56,467 | 101,533 | 57,046 | 0.718 | 0.718 | 0.718 | +| Parole, Nicaragua | 93,070 | 46,971 | 46,099 | 27,188 | 0.797 | 0.797 | 0.797 | +| Parole, Venezuela | 117,330 | 59,215 | 58,115 | 152,918 | 1.000 | 1.000 | 1.000 | +| Refugee | 160,000 | 80,750 | 79,250 | 363,809 | 1.000 | n/a | 0.505 | +| Asylee | 155,000 | 78,227 | 76,773 | 2,292,045 | 1.000 | n/a | 0.505 | +| TPS, Venezuela | 605,015 | 305,344 | 299,671 | 338,594 | 1.000 | 0.983 | 0.710 | +| TPS, El Salvador | 170,125 | 85,860 | 84,265 | 114,363 | 1.000 | 1.000 | 1.000 | +| TPS, Honduras | 51,225 | 25,853 | 25,372 | 22,197 | 0.938 | 0.938 | 0.938 | +| TPS, Nicaragua | 2,910 | 1,469 | 1,441 | 6,960 | 1.000 | 1.000 | 1.000 | +| TPS, Nepal | 7,160 | 3,614 | 3,546 | 16,986 | 1.000 | 1.000 | 1.000 | +| TPS, other designated | 21,215 | 10,707 | 10,508 | 141,413 | 1.000 | 1.000 | 1.000 | + +- Five draws run short of candidates. Four of them, U4U, Nicaraguan parole, + Honduran TPS and OAW, run short even if no earlier draw takes theirs. No + earlier draw that takes any rows shares a candidate with them: OAW comes + first, U4U's and Nicaraguan parole's candidates overlap no earlier draw's, + and Honduran TPS's all fall inside the deportation-withheld draw's, whose + zero target takes none. So each of the four ends at its projected share in + the worst case too, 0.718 to 0.971 of its target. Under the old rule, U4U, + the second draw in manifest order, would have stopped every stacked build on + these inputs. +- TPS Venezuela runs short because of an earlier draw. Its supply exceeds its + residual by 38,923, but Venezuelan parole, the fourth draw, takes its 58,115 + from non-citizens born in Venezuela who arrived in 2022 or later, and 0.848 + of that supply, those who arrived in 2022 or 2023, are TPS Venezuela's + candidates too. The parole draw takes an expected 49,300 of them, more than + the margin, so TPS Venezuela ends at an expected 0.983 of its target before + refugee and asylee draws take any of its candidates, and at 0.710 in the + worst case. +- The worst case lets every earlier draw take as much of a draw's candidates as + it can, up to its own residual. It holds for parole and TPS draws, which take + any non-citizen SSN code. Refugee and asylee draws take only rows the transfer + imputes as `OTHER_NON_CITIZEN`, which PUMS cannot show, so their supply column + overstates and their worst case counts the ASEC arm alone. +- The ASEC arm alone carries 0.505 of each target the ASEC stage met, and 0.357 + of U4U's. No draw falls below the 0.35 floor in the worst case: the lowest is + 0.505. +- The projection scales each arm by one factor. A build stands or falls on the + reconciliation itself and on the pool's terminal gates, which run the + composition gate on the stacked pool (`tools/build_us_multispine_pool.py`), + not on this projection. + +## Why the two trees label different rows + +The totals move little, but the two trees label different rows. Two changes in +this PR move rows, and `receipt.main_full_keys.json` separates them: it runs +main's method on this PR's key. Comparing it with main's own run isolates the +key, and comparing it with this PR's run isolates the method. + +**The draw key.** Each draw hashes a string built from the row's lineage. Main +builds it from the income year and `PERIDNUM`. This PR adds `PH_SEQ`, so two +rows that share an income year and a `PERIDNUM` but not a household draw apart +where main's key would tie them. In these +three files the income year and `PERIDNUM` already tell all 421,119 rows apart, +so the added column neither ties nor separates any rows. It still changes every +string the draws hash, and so every row's draw. Main's EAD spill takes +candidates in draw order until it has spilled its excess, so new draws hand the +EADs to other rows. On this PR's key, main's own method moves 2.400 million +people from undocumented to LPR and 2.392 million back, and 0.220 million from +undocumented to DACA and 0.244 million back. No status total moves by more than +0.025 million. + +**The method.** This PR draws the humanitarian statuses after the legal-status +indicators and before the EAD spill. Refugee and asylee draws take only rows +that an indicator marks documented; parole and TPS draws may also take rows from +the residual pool, and those rows take an EAD. Main's spill counts a worker by +prior-year wage or self-employment earnings (`WSAL_VAL`, `SEMP_VAL`), spills +workers and then students, and stops when the workers or students left in the +residual pool match their controls. This PR's spill counts a worker by current +labor-force status (`A_LFSR` 1 to 4, age 16 and over), spills students and then +workers, and stops when the workers or students in Pew's broad unauthorized +universe match the controls. That universe still counts a DACA or Cuban/Haitian +cohort row after it takes an EAD, and every parole and TPS row. In both trees a +spilled row takes an EAD, and a documented non-citizen with no other mark is a +lawful permanent resident. With both runs on this PR's key, the method moves +0.319 million people from undocumented to LPR and 0.706 million back, and 0.031 +million from undocumented to DACA and 0.105 million back. The humanitarian +draws take 0.874 million of main's LPR rows and 0.700 million of its +undocumented rows. + +The two effects do not add, because both reshuffle one pool. Between LPR and +undocumented, the key alone moves 4.793 million people, the method alone 1.025 +million, and the two together 4.436 million. + +Row by row, in millions, with the first run's labels down the side and the +second run's across. Main against this PR, each on its own key: + +| Main \ this PR | LPR | Undocumented | DACA | Cuban/Haitian | Paroled | Refugee | Asylee | TPS | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| LPR | 8.406 | 2.415 | | | 0.228 | 0.160 | 0.155 | 0.315 | +| Undocumented | 2.021 | 10.059 | 0.172 | 0.104 | 0.170 | | | 0.545 | +| DACA | | 0.271 | 0.105 | | | | | | +| Cuban/Haitian | | 0.146 | | 0.435 | | | | | + +The key alone, main on its own key against main on this PR's key: + +| Main \ main on this PR's key | LPR | Undocumented | DACA | Cuban/Haitian | +|---|---:|---:|---:|---:| +| LPR | 9.288 | 2.392 | | | +| Undocumented | 2.400 | 10.314 | 0.220 | 0.136 | +| DACA | | 0.244 | 0.131 | | +| Cuban/Haitian | | 0.123 | | 0.458 | + +The method alone, main on this PR's key against this PR: + +| Main on this PR's key \ this PR | LPR | Undocumented | DACA | Cuban/Haitian | Paroled | Refugee | Asylee | TPS | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| LPR | 10.108 | 0.706 | | | 0.231 | 0.160 | 0.155 | 0.327 | +| Undocumented | 0.319 | 12.013 | 0.031 | 0.012 | 0.167 | | | 0.533 | +| DACA | | 0.105 | 0.246 | | | | | | +| Cuban/Haitian | | 0.067 | | 0.527 | | | | | + +Citizens keep their label in every run (300.244 million). The tables come from +the three runs' `--labels-out` files, which stay out of the repository because +they hold person records; `scripts/xtab_labels.py` rebuilds the tables from +them. + +## Rerunning + +The archives are Census's public-use ASEC CSV releases at the URLs +`education_assistance_source.py` names. Put the three zips in one or more +directories, then run: + +```bash +.venv/bin/python -I docs/evidence/us-immigration-statuses-767/scripts/stage_receipt_pooled.py . --archive-dir --out receipt.json +``` + +For another commit, extract `git archive packages` into a directory and +pass that directory and `--source-commit `; add `--draw-keys full` to +run that tree on this PR's key. Add `--labels-out .pkl` to two runs, then +rebuild a row-by-row table: + +```bash +.venv/bin/python -I docs/evidence/us-immigration-statuses-767/scripts/xtab_labels.py main.pkl head.pkl +``` + +Count the head run's restored-status Medicaid reporters from its labels: + +```bash +.venv/bin/python -I docs/evidence/us-immigration-statuses-767/scripts/medicaid_reporters.py head.pkl --archive-dir --out receipt.head_medicaid_reporters.json +``` + +Project the stacked pool's ACS supply from the head receipt. Put the ACS 2024 +1-year PUMS files `csv_hus.zip` and `csv_pus.zip`, at the URLs +`us_runtime/acs_2024_1yr_sources.json` pins, in one directory; the script also +reads the three ASEC zips for the pool's masses: + +```bash +.venv/bin/python -I docs/evidence/us-immigration-statuses-767/scripts/stacked_supply.py receipt.head.json --acs-dir --archive-dir --out receipt.stacked_supply.json +``` diff --git a/docs/evidence/us-immigration-statuses-767/receipt.head.json b/docs/evidence/us-immigration-statuses-767/receipt.head.json new file mode 100644 index 0000000000..d7b45f6eb3 --- /dev/null +++ b/docs/evidence/us-immigration-statuses-767/receipt.head.json @@ -0,0 +1,379 @@ +{ + "archives": [ + { + "archive": "asecpub24csv.zip", + "household_weighted_persons": 320890854.43000007, + "income_year": 2023, + "marsupwt_persons": 332382485.08, + "peinusyr_max": 28, + "rows": 144265, + "sha256": "cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09" + }, + { + "archive": "asecpub25csv.zip", + "household_weighted_persons": 325952369.06, + "income_year": 2024, + "marsupwt_persons": 337689641.57, + "peinusyr_max": 28, + "rows": 142125, + "sha256": "318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b" + }, + { + "archive": "asecpub26csv.zip", + "household_weighted_persons": 329518151.84999996, + "income_year": 2025, + "marsupwt_persons": 338554893.54, + "peinusyr_max": 29, + "rows": 134729, + "sha256": "fe819d0d2fc4470c282e76c247b8aa8b6054811619730307e9f2fe6186707d93" + } + ], + "composition": { + "ASYLEE": { + "by_income_year": { + "2023": 61212.7587594046, + "2024": 46805.7, + "2025": 47473.44368574968 + }, + "code_29_weighted": 0.0, + "rows": 186, + "under_19_weighted": 46328.961730005605, + "weighted": 155491.90244515426 + }, + "CITIZEN": { + "by_income_year": { + "2023": 100026523.85662985, + "2024": 99788608.02, + "2025": 100429092.3485838 + }, + "code_29_weighted": 174138.23378113355, + "rows": 388829, + "under_19_weighted": 69797402.3449544, + "weighted": 300244224.22521365 + }, + "CUBAN_HAITIAN_ENTRANT": { + "by_income_year": { + "2023": 166021.48593467462, + "2024": 204853.16333333333, + "2025": 168316.9858028873 + }, + "code_29_weighted": 23718.69405929822, + "rows": 630, + "under_19_weighted": 63696.40176895717, + "weighted": 539191.6350708952 + }, + "DACA": { + "by_income_year": { + "2023": 104140.60565893029, + "2024": 95633.99666666666, + "2025": 76955.79573372872 + }, + "code_29_weighted": 0.0, + "rows": 359, + "under_19_weighted": 4168.370981434536, + "weighted": 276730.39805932564 + }, + "LEGAL_PERMANENT_RESIDENT": { + "by_income_year": { + "2023": 3526808.2138986206, + "2024": 3555543.08, + "2025": 3344707.160514318 + }, + "code_29_weighted": 345716.8718042292, + "rows": 13141, + "under_19_weighted": 915650.1381589137, + "weighted": 10427058.454412939 + }, + "PAROLED_ONE_YEAR": { + "by_income_year": { + "2023": 99339.58014190361, + "2024": 149332.66333333333, + "2025": 149216.92251845784 + }, + "code_29_weighted": 26609.018450874144, + "rows": 438, + "under_19_weighted": 100118.51899012196, + "weighted": 397889.1659936948 + }, + "REFUGEE": { + "by_income_year": { + "2023": 47182.47778915634, + "2024": 53504.206666666665, + "2025": 59764.97282362702 + }, + "code_29_weighted": 25519.560004618885, + "rows": 176, + "under_19_weighted": 64043.9924089673, + "weighted": 160451.65727945004 + }, + "TPS": { + "by_income_year": { + "2023": 277924.2046921097, + "2024": 311935.7733333333, + "2025": 270260.13037578424 + }, + "code_29_weighted": 0.0, + "rows": 1081, + "under_19_weighted": 107278.50949508903, + "weighted": 860120.1084012273 + }, + "UNDOCUMENTED": { + "by_income_year": { + "2023": 4341636.503162008, + "2024": 4444573.083333333, + "2025": 4105001.9266283335 + }, + "code_29_weighted": 464004.9826060645, + "rows": 16279, + "under_19_weighted": 1583961.7706165907, + "weighted": 12891211.513123676 + } + }, + "draw_keys": { + "calls": [ + { + "forced": false, + "key": "full", + "rows": 421119, + "salt": "immigration:paroled_one_year:afghanistan" + }, + { + "forced": false, + "key": "full", + "rows": 421119, + "salt": "immigration:paroled_one_year:ukraine" + }, + { + "forced": false, + "key": "full", + "rows": 421119, + "salt": "immigration:paroled_one_year:nicaragua" + }, + { + "forced": false, + "key": "full", + "rows": 421119, + "salt": "immigration:paroled_one_year:venezuela" + }, + { + "forced": false, + "key": "full", + "rows": 421119, + "salt": "immigration:refugee" + }, + { + "forced": false, + "key": "full", + "rows": 421119, + "salt": "immigration:asylee" + }, + { + "forced": false, + "key": "full", + "rows": 421119, + "salt": "immigration:tps:venezuela" + }, + { + "forced": false, + "key": "full", + "rows": 421119, + "salt": "immigration:tps:el_salvador" + }, + { + "forced": false, + "key": "full", + "rows": 421119, + "salt": "immigration:tps:honduras" + }, + { + "forced": false, + "key": "full", + "rows": 421119, + "salt": "immigration:tps:nicaragua" + }, + { + "forced": false, + "key": "full", + "rows": 421119, + "salt": "immigration:tps:nepal" + }, + { + "forced": false, + "key": "full", + "rows": 421119, + "salt": "immigration:tps:other_designated" + }, + { + "forced": false, + "key": "full", + "rows": 421119, + "salt": "immigration:ead_students" + }, + { + "forced": false, + "key": "full", + "rows": 421119, + "salt": "immigration:ead_workers" + } + ], + "requested": "native", + "used": { + "full": "source_year:source_household_id(PH_SEQ):source_person_id(PERIDNUM)" + } + }, + "error": null, + "gate_failures": [], + "gate_passed": true, + "humanitarian_draw_achieved": { + "asylee": { + "category": "asylee", + "origin": null, + "population": 155491.90244515426, + "relative": 1.0031735641622856, + "source": "https://ohss.dhs.gov/topics/immigration/asylees/annual-flow-report/fy24-asylees-flow-report", + "status": "ASYLEE", + "target": 155000.0 + }, + "deportation_withheld": { + "category": "deportation_withheld", + "origin": null, + "population": 0.0, + "relative": null, + "source": "https://crsreports.congress.gov/product/pdf/IN/IN12501", + "status": "DEPORTATION_WITHHELD", + "target": 0.0 + }, + "paroled_one_year:afghanistan": { + "category": "paroled_one_year", + "origin": "afghanistan", + "population": 74154.29223282929, + "relative": 1.007996795161206, + "source": "https://www.dhs.gov/sites/default/files/2023-06/PLCY%20-%20AFG%202503%20Update%202_0.pdf", + "status": "PAROLED_ONE_YEAR", + "target": 73566.0 + }, + "paroled_one_year:nicaragua": { + "category": "paroled_one_year", + "origin": "nicaragua", + "population": 93112.73319928072, + "relative": 1.0004591511688055, + "source": "https://www.cbp.gov/newsroom/national-media-release/cbp-releases-december-2024-monthly-update", + "status": "PAROLED_ONE_YEAR", + "target": 93070.0 + }, + "paroled_one_year:ukraine": { + "category": "paroled_one_year", + "origin": "ukraine", + "population": 112933.72361361884, + "relative": 0.7147704026178407, + "source": "https://www.dhs.gov/sites/default/files/2024-12/2024_1104_dmo_plcy_uniting_for_ukraine_process_overview_and_assessment.pdf", + "status": "PAROLED_ONE_YEAR", + "target": 158000.0 + }, + "paroled_one_year:venezuela": { + "category": "paroled_one_year", + "origin": "venezuela", + "population": 117688.41694796596, + "relative": 1.0030547766808655, + "source": "https://www.cbp.gov/newsroom/national-media-release/cbp-releases-december-2024-monthly-update", + "status": "PAROLED_ONE_YEAR", + "target": 117330.0 + }, + "refugee": { + "category": "refugee", + "origin": null, + "population": 160451.65727945004, + "relative": 1.0028228579965628, + "source": "https://ohss.dhs.gov/topics/immigration/refugees/annual-flow-report/fy-24-refugees-flow-report", + "status": "REFUGEE", + "target": 160000.0 + }, + "tps:el_salvador": { + "category": "tps", + "origin": "el_salvador", + "population": 170410.32409187898, + "relative": 1.0016771438170697, + "source": "https://www.everycrsreport.com/reports/RS20844.html", + "status": "TPS", + "target": 170125.0 + }, + "tps:honduras": { + "category": "tps", + "origin": "honduras", + "population": 52053.03129856825, + "relative": 1.0161645934322743, + "source": "https://www.everycrsreport.com/reports/RS20844.html", + "status": "TPS", + "target": 51225.0 + }, + "tps:nepal": { + "category": "tps", + "origin": "nepal", + "population": 7349.0149911888375, + "relative": 1.0263987417861504, + "source": "https://www.everycrsreport.com/reports/RS20844.html", + "status": "TPS", + "target": 7160.0 + }, + "tps:nicaragua": { + "category": "tps", + "origin": "nicaragua", + "population": 3192.9463618784666, + "relative": 1.0972324267623597, + "source": "https://www.everycrsreport.com/reports/RS20844.html", + "status": "TPS", + "target": 2910.0 + }, + "tps:other_designated": { + "category": "tps", + "origin": "other_designated", + "population": 21484.827706756125, + "relative": 1.0127187229203924, + "source": "https://www.everycrsreport.com/reports/RS20844.html", + "status": "TPS", + "target": 21215.0 + }, + "tps:venezuela": { + "category": "tps", + "origin": "venezuela", + "population": 605629.9639509566, + "relative": 1.0010164441393297, + "source": "https://www.everycrsreport.com/reports/RS20844.html", + "status": "TPS", + "target": 605015.0 + } + }, + "humanitarian_target_relative_band": [ + 0.35, + 1.5 + ], + "humanitarian_targets_by_status": { + "ASYLEE": 155000.0, + "DEPORTATION_WITHHELD": 0.0, + "PAROLED_ONE_YEAR": 441966.0, + "REFUGEE": 160000.0, + "TPS": 857650.0 + }, + "immigration_module_sha256": "ecac0e264521639af5e240fe33edf8f7c3568437497d4eb27dee1862ba7e5b6c", + "none_ssn_iff_undocumented": true, + "pew_labor_force": 9699442.858246539, + "pew_labor_force_control": 9700000.0, + "pew_universe": 14573640.556715064, + "pew_universe_anchor": 14000000.0, + "ratio_to_target": { + "ASYLEE": 1.0031735641622856, + "DEPORTATION_WITHHELD": null, + "PAROLED_ONE_YEAR": 0.9002709846316115, + "REFUGEE": 1.0028228579965628, + "TPS": 1.0028800890820582 + }, + "rows": 421119, + "script_sha256": "7c28e5000282137583839624f97d731b43637d8f2a49cd8bbfb5e25bcb6e47dc", + "seed": 0, + "source_commit": "4f94e489aba5de4c33444742ac8333e736acba03", + "source_commit_from": "git", + "source_tree_clean": true, + "stage_seconds": 10.8, + "time_period": 2024, + "weights": "HSUP_WGT * 0.01 by PH_SEQ, each file rescaled to one third of income year 2024's weighted person population (asec_pool.pool_asec_sources, anchor_year=2024)" +} diff --git a/docs/evidence/us-immigration-statuses-767/receipt.head_medicaid_reporters.json b/docs/evidence/us-immigration-statuses-767/receipt.head_medicaid_reporters.json new file mode 100644 index 0000000000..e5b84d3c78 --- /dev/null +++ b/docs/evidence/us-immigration-statuses-767/receipt.head_medicaid_reporters.json @@ -0,0 +1,56 @@ +{ + "archives": [ + { + "archive": "asecpub24csv.zip", + "income_year": 2023, + "sha256": "cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09" + }, + { + "archive": "asecpub25csv.zip", + "income_year": 2024, + "sha256": "318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b" + }, + { + "archive": "asecpub26csv.zip", + "income_year": 2025, + "sha256": "fe819d0d2fc4470c282e76c247b8aa8b6054811619730307e9f2fe6186707d93" + } + ], + "column": "NOW_MCAID == 1", + "labels_file": "head_4f94e489a.native.labels.pkl", + "labels_rows": 421119, + "labels_sha256": "363451875c6d7842c59a79e037880d9bcf29f266676a9e1a1586310b0779ff0f", + "script_sha256": "ee1efc1a55a511bec5e668f7032e71de68acf58d29e8e996b44609376ae35a72", + "statuses": { + "ASYLEE": { + "now_mcaid_persons_millions": 0.106635, + "persons_millions": 0.155492, + "rows": 186 + }, + "CONDITIONAL_ENTRANT": { + "now_mcaid_persons_millions": 0.0, + "persons_millions": 0.0, + "rows": 0 + }, + "DEPORTATION_WITHHELD": { + "now_mcaid_persons_millions": 0.0, + "persons_millions": 0.0, + "rows": 0 + }, + "PAROLED_ONE_YEAR": { + "now_mcaid_persons_millions": 0.163247, + "persons_millions": 0.397889, + "rows": 438 + }, + "REFUGEE": { + "now_mcaid_persons_millions": 0.121047, + "persons_millions": 0.160452, + "rows": 176 + }, + "all restored": { + "now_mcaid_persons_millions": 0.39093, + "persons_millions": 0.713833, + "rows": 800 + } + } +} diff --git a/docs/evidence/us-immigration-statuses-767/receipt.main.json b/docs/evidence/us-immigration-statuses-767/receipt.main.json new file mode 100644 index 0000000000..c03cce907a --- /dev/null +++ b/docs/evidence/us-immigration-statuses-767/receipt.main.json @@ -0,0 +1,122 @@ +{ + "archives": [ + { + "archive": "asecpub24csv.zip", + "household_weighted_persons": 320890854.43000007, + "income_year": 2023, + "marsupwt_persons": 332382485.08, + "peinusyr_max": 28, + "rows": 144265, + "sha256": "cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09" + }, + { + "archive": "asecpub25csv.zip", + "household_weighted_persons": 325952369.06, + "income_year": 2024, + "marsupwt_persons": 337689641.57, + "peinusyr_max": 28, + "rows": 142125, + "sha256": "318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b" + }, + { + "archive": "asecpub26csv.zip", + "household_weighted_persons": 329518151.84999996, + "income_year": 2025, + "marsupwt_persons": 338554893.54, + "peinusyr_max": 29, + "rows": 134729, + "sha256": "fe819d0d2fc4470c282e76c247b8aa8b6054811619730307e9f2fe6186707d93" + } + ], + "composition": { + "CITIZEN": { + "by_income_year": { + "2023": 100026523.85662985, + "2024": 99788608.02, + "2025": 100429092.3485838 + }, + "code_29_weighted": 174138.23378113355, + "rows": 388829, + "under_19_weighted": 69797402.3449544, + "weighted": 300244224.22521365 + }, + "CUBAN_HAITIAN_ENTRANT": { + "by_income_year": { + "2023": 188218.28306411504, + "2024": 216763.02666666667, + "2025": 176496.12279339816 + }, + "code_29_weighted": 23192.688342167814, + "rows": 676, + "under_19_weighted": 64619.12753444083, + "weighted": 581477.4325241798 + }, + "DACA": { + "by_income_year": { + "2023": 149258.51740021014, + "2024": 127492.23999999999, + "2025": 99016.39532075019 + }, + "code_29_weighted": 0.0, + "rows": 511, + "under_19_weighted": 3745.3214781583056, + "weighted": 375767.1527209604 + }, + "LEGAL_PERMANENT_RESIDENT": { + "by_income_year": { + "2023": 3915174.3808217305, + "2024": 4020124.2700000005, + "2025": 3744677.211858804 + }, + "code_29_weighted": 392355.07269730815, + "rows": 14722, + "under_19_weighted": 1122543.9672262375, + "weighted": 11679975.862680536 + }, + "UNDOCUMENTED": { + "by_income_year": { + "2023": 4371614.648750752, + "2024": 4497802.13, + "2025": 4201507.608109934 + }, + "code_29_weighted": 470021.365885609, + "rows": 16381, + "under_19_weighted": 1694338.2479112435, + "weighted": 13070924.386860687 + } + }, + "draw_keys": { + "calls": [ + { + "forced": false, + "key": "legacy", + "rows": 421119, + "salt": "immigration:ead_workers" + }, + { + "forced": false, + "key": "legacy", + "rows": 421119, + "salt": "immigration:ead_students" + } + ], + "requested": "native", + "used": { + "legacy": "source_year:source_person_id(PERIDNUM)" + } + }, + "error": null, + "gate_failures": [], + "gate_passed": true, + "immigration_module_sha256": "8871df8b5f86f6bbf8541259a69d52b7db214560e5dacf5ad6db15268368f826", + "none_ssn_iff_undocumented": true, + "rows": 421119, + "script_sha256": "7c28e5000282137583839624f97d731b43637d8f2a49cd8bbfb5e25bcb6e47dc", + "seed": 0, + "source_commit": "764ae3aa778cbc9030e286fabc3777e25c18f403", + "source_commit_from": "argument", + "source_tree_clean": null, + "stage_seconds": 1.9, + "time_period": 2024, + "weights": "HSUP_WGT * 0.01 by PH_SEQ, each file rescaled to one third of income year 2024's weighted person population (asec_pool.pool_asec_sources, anchor_year=2024)" +} diff --git a/docs/evidence/us-immigration-statuses-767/receipt.main_full_keys.json b/docs/evidence/us-immigration-statuses-767/receipt.main_full_keys.json new file mode 100644 index 0000000000..3d6b9c1740 --- /dev/null +++ b/docs/evidence/us-immigration-statuses-767/receipt.main_full_keys.json @@ -0,0 +1,122 @@ +{ + "archives": [ + { + "archive": "asecpub24csv.zip", + "household_weighted_persons": 320890854.43000007, + "income_year": 2023, + "marsupwt_persons": 332382485.08, + "peinusyr_max": 28, + "rows": 144265, + "sha256": "cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09" + }, + { + "archive": "asecpub25csv.zip", + "household_weighted_persons": 325952369.06, + "income_year": 2024, + "marsupwt_persons": 337689641.57, + "peinusyr_max": 28, + "rows": 142125, + "sha256": "318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b" + }, + { + "archive": "asecpub26csv.zip", + "household_weighted_persons": 329518151.84999996, + "income_year": 2025, + "marsupwt_persons": 338554893.54, + "peinusyr_max": 29, + "rows": 134729, + "sha256": "fe819d0d2fc4470c282e76c247b8aa8b6054811619730307e9f2fe6186707d93" + } + ], + "composition": { + "CITIZEN": { + "by_income_year": { + "2023": 100026523.85662985, + "2024": 99788608.02, + "2025": 100429092.3485838 + }, + "code_29_weighted": 174138.23378113355, + "rows": 388829, + "under_19_weighted": 69797402.3449544, + "weighted": 300244224.22521365 + }, + "CUBAN_HAITIAN_ENTRANT": { + "by_income_year": { + "2023": 183475.60695258938, + "2024": 227568.58, + "2025": 183211.21581351623 + }, + "code_29_weighted": 24247.812392376745, + "rows": 688, + "under_19_weighted": 64589.79838310768, + "weighted": 594255.4027661056 + }, + "DACA": { + "by_income_year": { + "2023": 141697.3615097301, + "2024": 108160.53333333333, + "2025": 101408.83964565907 + }, + "code_29_weighted": 0.0, + "rows": 455, + "under_19_weighted": 3805.3054248002936, + "weighted": 351266.7344887225 + }, + "LEGAL_PERMANENT_RESIDENT": { + "by_income_year": { + "2023": 3925049.9863831, + "2024": 3972096.273333333, + "2025": 3790862.9623701023 + }, + "code_29_weighted": 387197.4977466433, + "rows": 14680, + "under_19_weighted": 1107430.6397713334, + "weighted": 11688009.222086536 + }, + "UNDOCUMENTED": { + "by_income_year": { + "2023": 4374042.875191389, + "2024": 4554356.279999999, + "2025": 4146214.3202536087 + }, + "code_29_weighted": 474123.816786065, + "rows": 16467, + "under_19_weighted": 1709420.9205708387, + "weighted": 13074613.475444999 + } + }, + "draw_keys": { + "calls": [ + { + "forced": true, + "key": "full", + "rows": 421119, + "salt": "immigration:ead_workers" + }, + { + "forced": true, + "key": "full", + "rows": 421119, + "salt": "immigration:ead_students" + } + ], + "requested": "full", + "used": { + "full": "source_year:source_household_id(PH_SEQ):source_person_id(PERIDNUM)" + } + }, + "error": null, + "gate_failures": [], + "gate_passed": true, + "immigration_module_sha256": "8871df8b5f86f6bbf8541259a69d52b7db214560e5dacf5ad6db15268368f826", + "none_ssn_iff_undocumented": true, + "rows": 421119, + "script_sha256": "7c28e5000282137583839624f97d731b43637d8f2a49cd8bbfb5e25bcb6e47dc", + "seed": 0, + "source_commit": "764ae3aa778cbc9030e286fabc3777e25c18f403", + "source_commit_from": "argument", + "source_tree_clean": null, + "stage_seconds": 2.2, + "time_period": 2024, + "weights": "HSUP_WGT * 0.01 by PH_SEQ, each file rescaled to one third of income year 2024's weighted person population (asec_pool.pool_asec_sources, anchor_year=2024)" +} diff --git a/docs/evidence/us-immigration-statuses-767/receipt.pre_sync_42bb166b.json b/docs/evidence/us-immigration-statuses-767/receipt.pre_sync_42bb166b.json new file mode 100644 index 0000000000..1f9c346e6d --- /dev/null +++ b/docs/evidence/us-immigration-statuses-767/receipt.pre_sync_42bb166b.json @@ -0,0 +1,46 @@ +{ + "archives": [ + { + "archive": "asecpub24csv.zip", + "household_weighted_persons": 320890854.43000007, + "income_year": 2023, + "marsupwt_persons": 332382485.08, + "peinusyr_max": 28, + "rows": 144265, + "sha256": "cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09" + }, + { + "archive": "asecpub25csv.zip", + "household_weighted_persons": 325952369.06, + "income_year": 2024, + "marsupwt_persons": 337689641.57, + "peinusyr_max": 28, + "rows": 142125, + "sha256": "318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b" + }, + { + "archive": "asecpub26csv.zip", + "household_weighted_persons": 329518151.84999996, + "income_year": 2025, + "marsupwt_persons": 338554893.54, + "peinusyr_max": 29, + "rows": 134729, + "sha256": "fe819d0d2fc4470c282e76c247b8aa8b6054811619730307e9f2fe6186707d93" + } + ], + "draw_keys": { + "calls": [], + "requested": "native", + "used": {} + }, + "error": "SourceRuntimeError: PEINUSYR carries value(s) outside the 2024 ASEC domain 0..28: [29.0].", + "immigration_module_sha256": "df6699344c5fddbb5322baeb22f574be846e43c8c7600fd5dea798189c382042", + "script_sha256": "7c28e5000282137583839624f97d731b43637d8f2a49cd8bbfb5e25bcb6e47dc", + "seed": 0, + "source_commit": "42bb166b1484df036809814434c1acf862b8cbb8", + "source_commit_from": "argument", + "source_tree_clean": null, + "stage_seconds": 0.1, + "time_period": 2024, + "weights": "HSUP_WGT * 0.01 by PH_SEQ, each file rescaled to one third of income year 2024's weighted person population (asec_pool.pool_asec_sources, anchor_year=2024)" +} diff --git a/docs/evidence/us-immigration-statuses-767/receipt.stacked_supply.json b/docs/evidence/us-immigration-statuses-767/receipt.stacked_supply.json new file mode 100644 index 0000000000..dcfe67754b --- /dev/null +++ b/docs/evidence/us-immigration-statuses-767/receipt.stacked_supply.json @@ -0,0 +1,230 @@ +{ + "acs": { + "csv_hus.zip": "8281008e53de98f0ef81e7a2ee5a8725991dda1ecfd2713ead73246425e515d0", + "csv_pus.zip": "afdc6d90c6e2f0bab365ed32d95ba4c4d8ac651162f46ac7861295b2dc469894", + "household_mass": 141125706.0, + "person_mass": 334342004.0 + }, + "acs_arm_person_mass": 159947952.85979113, + "acs_arm_weight_scale": 0.4783962258591688, + "afghan_noncitizen_arrivals_at_acs_weights": { + "2021-2022": 71681.0, + "2021-2024": 110913.0 + }, + "arm_household_share": 0.5, + "asec_archives": [ + { + "archive": "asecpub24csv.zip", + "household_mass": 132391925.44999999, + "income_year": 2023, + "person_mass": 320890854.43000007, + "sha256": "cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09" + }, + { + "archive": "asecpub25csv.zip", + "household_mass": 134963892.8, + "income_year": 2024, + "person_mass": 325952369.06, + "sha256": "318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b" + }, + { + "archive": "asecpub26csv.zip", + "household_mass": 137123796.5, + "income_year": 2025, + "person_mass": 329518151.84999996, + "sha256": "fe819d0d2fc4470c282e76c247b8aa8b6054811619730307e9f2fe6186707d93" + } + ], + "asec_arm_person_mass": 162976184.53, + "asec_pool_household_mass": 135028010.2442213, + "asec_pool_person_mass": 325952369.06, + "composition_band_floor": 0.35, + "draws": { + "asylee": { + "acs_arm_supply_bound": 2292044.6361100897, + "acs_candidates_at_acs_weights": 4791101.0, + "acs_residual": 76773.24121281803, + "asec_arm_population": 78226.75878718197, + "asec_arm_share_of_target": 0.5046887663689159, + "asec_stage_population": 155491.90244515426, + "earlier_draws_worst_case_removal": 198844.79859674792, + "expected_share_of_target": null, + "projected_share_of_target": 1.0, + "shortfall_at_bound": 0.0, + "target": 155000.0, + "worst_case_share_of_target": 0.5046887663689159 + }, + "deportation_withheld": { + "acs_arm_supply_bound": 6685225.110438908, + "acs_candidates_at_acs_weights": 13974243.0, + "acs_residual": 0.0, + "asec_arm_population": 0.0, + "asec_arm_share_of_target": null, + "asec_stage_population": 0.0, + "earlier_draws_worst_case_removal": 76773.24121281803, + "expected_share_of_target": null, + "projected_share_of_target": null, + "shortfall_at_bound": 0.0, + "target": 0.0, + "worst_case_share_of_target": null + }, + "paroled_one_year:afghanistan": { + "acs_arm_supply_bound": 34291.919865811076, + "acs_candidates_at_acs_weights": 71681.0, + "acs_residual": 36438.06621330433, + "asec_arm_population": 37127.93378669567, + "asec_arm_share_of_target": 0.5046887663689159, + "asec_stage_population": 74154.29223282929, + "earlier_draws_worst_case_removal": 0, + "expected_share_of_target": 0.9708269261956168, + "projected_share_of_target": 0.9708269261956168, + "shortfall_at_bound": 2146.1463474932534, + "target": 73566.0, + "worst_case_share_of_target": 0.9708269261956168 + }, + "paroled_one_year:nicaragua": { + "acs_arm_supply_bound": 27188.21430802828, + "acs_candidates_at_acs_weights": 56832.0, + "acs_residual": 46098.616514044996, + "asec_arm_population": 46971.383485955004, + "asec_arm_share_of_target": 0.5046887663689159, + "asec_stage_population": 93112.73319928072, + "earlier_draws_worst_case_removal": 0.0, + "expected_share_of_target": 0.7968152766088243, + "projected_share_of_target": 0.7968152766088243, + "shortfall_at_bound": 18910.402206016715, + "target": 93070.0, + "worst_case_share_of_target": 0.7968152766088243 + }, + "paroled_one_year:ukraine": { + "acs_arm_supply_bound": 57045.87955635072, + "acs_candidates_at_acs_weights": 119244.0, + "acs_residual": 101533.13819319058, + "asec_arm_population": 56466.86180680942, + "asec_arm_share_of_target": 0.3573852013089204, + "asec_stage_population": 112933.72361361884, + "earlier_draws_worst_case_removal": 0.0, + "expected_share_of_target": 0.718435071918735, + "projected_share_of_target": 0.718435071918735, + "shortfall_at_bound": 44487.258636839855, + "target": 158000.0, + "worst_case_share_of_target": 0.718435071918735 + }, + "paroled_one_year:venezuela": { + "acs_arm_supply_bound": 152917.91840720573, + "acs_candidates_at_acs_weights": 319647.0, + "acs_residual": 58114.8670419351, + "asec_arm_population": 59215.1329580649, + "asec_arm_share_of_target": 0.5046887663689159, + "asec_stage_population": 117688.41694796596, + "earlier_draws_worst_case_removal": 0.0, + "expected_share_of_target": 1.0, + "projected_share_of_target": 1.0, + "shortfall_at_bound": 0.0, + "target": 117330.0, + "worst_case_share_of_target": 1.0 + }, + "refugee": { + "acs_arm_supply_bound": 363808.8482564773, + "acs_candidates_at_acs_weights": 760476.0, + "acs_residual": 79249.79738097345, + "asec_arm_population": 80750.20261902655, + "asec_arm_share_of_target": 0.5046887663689159, + "asec_stage_population": 160451.65727945004, + "earlier_draws_worst_case_removal": 123713.9927204219, + "expected_share_of_target": null, + "projected_share_of_target": 1.0, + "shortfall_at_bound": 0.0, + "target": 160000.0, + "worst_case_share_of_target": 0.5046887663689159 + }, + "tps:el_salvador": { + "acs_arm_supply_bound": 114363.48816898945, + "acs_candidates_at_acs_weights": 239056.0, + "acs_residual": 84264.82362148818, + "asec_arm_population": 85860.17637851182, + "asec_arm_share_of_target": 0.5046887663689159, + "asec_stage_population": 170410.32409187898, + "earlier_draws_worst_case_removal": 0.0, + "expected_share_of_target": 1.0, + "projected_share_of_target": 1.0, + "shortfall_at_bound": 0.0, + "target": 170125.0, + "worst_case_share_of_target": 1.0 + }, + "tps:honduras": { + "acs_arm_supply_bound": 22196.628087413716, + "acs_candidates_at_acs_weights": 46398.0, + "acs_residual": 25372.31794275228, + "asec_arm_population": 25852.68205724772, + "asec_arm_share_of_target": 0.5046887663689159, + "asec_stage_population": 52053.03129856825, + "earlier_draws_worst_case_removal": 0.0, + "expected_share_of_target": 0.9380050784706966, + "projected_share_of_target": 0.9380050784706966, + "shortfall_at_bound": 3175.6898553385654, + "target": 51225.0, + "worst_case_share_of_target": 0.9380050784706966 + }, + "tps:nepal": { + "acs_arm_supply_bound": 16985.936395355646, + "acs_candidates_at_acs_weights": 35506.0, + "acs_residual": 3546.428432798562, + "asec_arm_population": 3613.571567201438, + "asec_arm_share_of_target": 0.5046887663689159, + "asec_stage_population": 7349.0149911888375, + "earlier_draws_worst_case_removal": 0.0, + "expected_share_of_target": 1.0, + "projected_share_of_target": 1.0, + "shortfall_at_bound": 0.0, + "target": 7160.0, + "worst_case_share_of_target": 1.0 + }, + "tps:nicaragua": { + "acs_arm_supply_bound": 6960.186690025047, + "acs_candidates_at_acs_weights": 14549.0, + "acs_residual": 1441.3556898664547, + "asec_arm_population": 1468.6443101335453, + "asec_arm_share_of_target": 0.5046887663689159, + "asec_stage_population": 3192.9463618784666, + "earlier_draws_worst_case_removal": 0.0, + "expected_share_of_target": 1.0, + "projected_share_of_target": 1.0, + "shortfall_at_bound": 0.0, + "target": 2910.0, + "worst_case_share_of_target": 1.0 + }, + "tps:other_designated": { + "acs_arm_supply_bound": 141412.9675715186, + "acs_candidates_at_acs_weights": 295598.0, + "acs_residual": 10508.027821483449, + "asec_arm_population": 10706.972178516551, + "asec_arm_share_of_target": 0.5046887663689159, + "asec_stage_population": 21484.827706756125, + "earlier_draws_worst_case_removal": 49690.537583766, + "expected_share_of_target": 1.0, + "projected_share_of_target": 1.0, + "shortfall_at_bound": 0.0, + "target": 21215.0, + "worst_case_share_of_target": 1.0 + }, + "tps:venezuela": { + "acs_arm_supply_bound": 338593.5399838922, + "acs_candidates_at_acs_weights": 707768.0, + "acs_residual": 299670.7260153103, + "asec_arm_population": 305344.2739846897, + "asec_arm_share_of_target": 0.5046887663689159, + "asec_stage_population": 605629.9639509566, + "earlier_draws_worst_case_removal": 214137.90563572658, + "expected_share_of_target": 0.9828393132859046, + "projected_share_of_target": 1.0, + "shortfall_at_bound": 0.0, + "target": 605015.0, + "worst_case_share_of_target": 0.7103954585140125 + } + }, + "lowest_worst_case_share_of_target": 0.5046887663689159, + "receipt_file": "receipt.head.json", + "receipt_sha256": "d9bdd7e626acaf9638c8c696f65ce6b9ba377263eff752dc128498c0abe30a2f", + "script_sha256": "0c5718ce093fe583a410c79af40b4e28785014c1aeeb3bf1e0294c3367eda503" +} diff --git a/docs/evidence/us-immigration-statuses-767/scripts/medicaid_reporters.py b/docs/evidence/us-immigration-statuses-767/scripts/medicaid_reporters.py new file mode 100644 index 0000000000..df7e952ba6 --- /dev/null +++ b/docs/evidence/us-immigration-statuses-767/scripts/medicaid_reporters.py @@ -0,0 +1,136 @@ +"""Count the people in the restored Medicaid statuses who report Medicaid. + +Usage: + python -I medicaid_reporters.py LABELS.pkl --archive-dir [--out OUT.json] + +LABELS.pkl is a ``--labels-out`` file from ``stage_receipt_pooled.py``. The +script reads ``NOW_MCAID`` (Medicaid at interview) from the three ASEC person +files that script runs on, after checking each archive against the same +SHA-256, and joins them to the labels on ``(source_year, PERIDNUM)`` as that +script keys them. It then sums the run's person weights, by status, over the +statuses that the reform-coverage probe +``hr1_medicaid_humanitarian_eligibility_restoration`` restores. +``microcosm.build.us_runtime.cps_carried`` maps ``NOW_MCAID`` to +``has_medicaid_health_coverage_at_interview``, the column the Medicaid take-up +stage anchors on. The output holds aggregates only, in millions of people. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import zipfile +from pathlib import Path + +import numpy as np +import pandas as pd + +# The archives and SHA-256 values of stage_receipt_pooled.py. +ARCHIVES = ( + ( + 2023, + "asecpub24csv.zip", + "pppub24.csv", + "cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09", + ), + ( + 2024, + "asecpub25csv.zip", + "pppub25.csv", + "318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b", + ), + ( + 2025, + "asecpub26csv.zip", + "pppub26.csv", + "fe819d0d2fc4470c282e76c247b8aa8b6054811619730307e9f2fe6186707d93", + ), +) +# The statuses the probe adds to the 2027 eligible list. +RESTORED = ( + "REFUGEE", + "ASYLEE", + "DEPORTATION_WITHHELD", + "PAROLED_ONE_YEAR", + "CONDITIONAL_ENTRANT", +) + +parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) +parser.add_argument("labels", type=Path) +parser.add_argument( + "--archive-dir", + type=Path, + action="append", + required=True, + help="directory holding ASEC zips (repeatable; each zip is found by name)", +) +parser.add_argument("--out", type=Path) +args = parser.parse_args() + + +def _archive_path(name: str) -> Path: + found = [d / name for d in args.archive_dir if (d / name).is_file()] + if not found: + raise SystemExit(f"{name} is in none of {args.archive_dir}") + return found[0] + + +labels = pd.read_pickle(args.labels) +key = labels["source_year"].astype(str) + ":" + labels["source_person_id"].astype(str) +if key.duplicated().any(): + raise SystemExit(f"{args.labels}: duplicate person keys") + +parts = [] +archives = [] +for income_year, archive_name, member, pinned in ARCHIVES: + path = _archive_path(archive_name) + digest = hashlib.sha256(path.read_bytes()).hexdigest() + if digest != pinned: + raise SystemExit(f"{path}: SHA-256 {digest} is not the pinned {pinned}") + with zipfile.ZipFile(path) as archive: + name = next(n for n in archive.namelist() if n.lower().endswith(member)) + with archive.open(name) as handle: + frame = pd.read_csv(handle, usecols=["PERIDNUM", "NOW_MCAID"]) + # stage_receipt_pooled.py reads PERIDNUM as an integer, then as text. + frame.index = str(income_year) + ":" + frame["PERIDNUM"].astype(str) + parts.append(frame["NOW_MCAID"]) + archives.append( + {"income_year": income_year, "archive": archive_name, "sha256": digest} + ) +now_mcaid = pd.concat(parts) +if now_mcaid.index.duplicated().any(): + raise SystemExit("the ASEC files repeat a (source_year, PERIDNUM) key") +if set(now_mcaid.index) != set(key): + raise SystemExit("the labels and the ASEC files hold different persons") + +reports = now_mcaid.reindex(key).to_numpy() == 1 +weight = labels["person_weight"].to_numpy(dtype=np.float64) +status = labels["immigration_status_str"].to_numpy() +rows = {} +for name in (*RESTORED, "all restored"): + mask = np.isin(status, RESTORED) if name == "all restored" else status == name + rows[name] = { + "rows": int(mask.sum()), + "persons_millions": round(float(weight[mask].sum()) / 1e6, 6), + "now_mcaid_persons_millions": round( + float(weight[mask & reports].sum()) / 1e6, 6 + ), + } + print( + f"{name:22s} {rows[name]['persons_millions']:7.3f}M people, " + f"{rows[name]['now_mcaid_persons_millions']:6.3f}M report Medicaid at " + f"interview ({rows[name]['rows']} rows)" + ) + +if args.out is not None: + receipt = { + "labels_file": args.labels.name, + "labels_sha256": hashlib.sha256(args.labels.read_bytes()).hexdigest(), + "labels_rows": int(len(labels)), + "script_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "archives": archives, + "column": "NOW_MCAID == 1", + "statuses": rows, + } + args.out.write_text(json.dumps(receipt, indent=1, sort_keys=True) + "\n") diff --git a/docs/evidence/us-immigration-statuses-767/scripts/stacked_supply.py b/docs/evidence/us-immigration-statuses-767/scripts/stacked_supply.py new file mode 100644 index 0000000000..456c18949a --- /dev/null +++ b/docs/evidence/us-immigration-statuses-767/scripts/stacked_supply.py @@ -0,0 +1,342 @@ +"""Project each humanitarian draw's stacked ACS supply against its residual. + +Usage: + python -I stacked_supply.py RECEIPT.json --acs-dir --archive-dir + [--out OUT.json] + +A stacked (multispine) pool joins the ASEC pool and the ACS 2024 1-year, each +arm carrying half the ASEC pool's household mass +(``stacked_spine.DEFAULT_STACKED_HOUSEHOLD_MASS_SHARES``, anchored on the ASEC +channel by ``spine_assembly.assemble_spines``). The immigration stage labels +the ASEC rows at the full pool's person mass (``multispine_pool``'s +``person_weight_scale``); ``reconcile_us_immigration_humanitarian_transfer`` +then asks the ACS rows for each draw's residual: its target less the ASEC +arm's labelled mass. + +This script does not build a stacked pool. It projects one from aggregates: + +- the ASEC arm's labelled mass from RECEIPT.json's + ``humanitarian_draw_achieved`` (a ``stage_receipt_pooled.py`` run on the + default pool), scaled to the stacked weights; +- the ACS arm's supply from the ACS 2024 1-year PUMS (``csv_hus.zip`` and + ``csv_pus.zip`` in ``--acs-dir``, checked against the SHA-256 values in + ``us_runtime/acs_2024_1yr_sources.json``), weighted as ``acs_pums`` weights the + frame (WGTP of each occupied unit, PWGTP for a group-quarters person), with + each draw's candidates chosen by the kernel's own + ``_humanitarian_draw_candidates``. + +Every non-citizen counts as SSN-eligible, and no draw removes another's +candidates, so each supply is an upper bound. Refugee, asylee and +withholding draws take only the indicator-documented code the transfer +assigns, so their bounds are loose; parole and TPS draws take every +non-citizen code, so theirs are tight up to the order of the draws. + +Two lower bounds bracket each draw's share of its target: + +- ``asec_arm_share_of_target``: the ASEC arm's labelled mass alone. The + reconciliation never relabels those rows, so every draw reaches at least + this share. +- ``worst_case_share_of_target``: the ASEC arm plus the ACS supply left after + every earlier draw in manifest order takes as much of the draw's candidates + as it can, up to its own residual (a draw can pass its residual by the + weight of the one tie group that crosses it, which this bound leaves out; + ACS draws key on single persons). The bound holds for parole and TPS + draws, whose SSN rule this script reads exactly. A refugee, asylee or + withholding draw takes only rows the transfer imputes as + ``OTHER_NON_CITIZEN``, which PUMS cannot show, so its worst case is its + ASEC arm share. + +One estimate sits between the worst case and the projected share, for parole +and TPS draws only: + +- ``expected_share_of_target``: the ASEC arm plus the ACS mass the draw + expects to take when each earlier parole or TPS draw takes its residual + from the candidates no earlier draw took. The reconciliation selects by + person-keyed hash draws, which order the free candidates as a uniform + random order would, so each free candidate of a draw that takes mass + ``take`` from free mass ``available`` stays free with probability about + ``1 - take / available``. Refugee, asylee and withholding draws take only + ``OTHER_NON_CITIZEN`` rows, which PUMS cannot show, so the estimate leaves + their takes out. Those takes can only lower a later draw's share, up to + the weight of the one tie group that crosses its residual, and they lower + it by more than that only once its free supply falls below its residual, + as TPS Venezuela's does. + +The output holds aggregates only. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import zipfile +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.build.us_runtime import immigration + +# The archives and SHA-256 values of stage_receipt_pooled.py. +ARCHIVES = ( + ( + 2023, + "asecpub24csv.zip", + "pppub24.csv", + "hhpub24.csv", + "cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09", + ), + ( + 2024, + "asecpub25csv.zip", + "pppub25.csv", + "hhpub25.csv", + "318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b", + ), + ( + 2025, + "asecpub26csv.zip", + "pppub26.csv", + "hhpub26.csv", + "fe819d0d2fc4470c282e76c247b8aa8b6054811619730307e9f2fe6186707d93", + ), +) +ANCHOR_INCOME_YEAR = 2024 +TIME_PERIOD = 2024 +ARM_SHARE = 0.5 + +parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) +parser.add_argument("receipt", type=Path) +parser.add_argument("--acs-dir", type=Path, required=True) +parser.add_argument( + "--archive-dir", + type=Path, + action="append", + required=True, + help="directory holding ASEC zips (repeatable; each zip is found by name)", +) +parser.add_argument("--out", type=Path) +args = parser.parse_args() + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _archive_path(name: str) -> Path: + found = [d / name for d in args.archive_dir if (d / name).is_file()] + if not found: + raise SystemExit(f"{name} is in none of {args.archive_dir}") + return found[0] + + +def _read_pums(path: Path, token: str, usecols: list[str]) -> pd.DataFrame: + parts = [] + with zipfile.ZipFile(path) as archive: + for name in sorted(archive.namelist()): + low = name.lower() + if low.endswith(".csv") and token in low: + with archive.open(name) as handle: + parts.append( + pd.read_csv(handle, usecols=usecols, dtype={"SERIALNO": str}) + ) + return pd.concat(parts, ignore_index=True) + + +# The ASEC pool's household and person masses, rescaled as the pool rescales. +asec_files = [] +for income_year, archive_name, member, household_member, pinned in ARCHIVES: + path = _archive_path(archive_name) + digest = _sha256(path) + if digest != pinned: + raise SystemExit(f"{path}: SHA-256 {digest} is not the pinned {pinned}") + with zipfile.ZipFile(path) as archive: + names = archive.namelist() + with archive.open(next(n for n in names if n.lower().endswith(member))) as f: + persons = pd.read_csv(f, usecols=["PH_SEQ"]) + with archive.open( + next(n for n in names if n.lower().endswith(household_member)) + ) as f: + households = pd.read_csv(f, usecols=["H_SEQ", "HSUP_WGT"]) + weight = households.drop_duplicates("H_SEQ").set_index("H_SEQ")["HSUP_WGT"] * 0.01 + person_mass = float(persons["PH_SEQ"].map(weight).sum()) + household_mass = float(weight.loc[persons["PH_SEQ"].unique()].sum()) + asec_files.append( + { + "income_year": income_year, + "archive": archive_name, + "sha256": digest, + "person_mass": person_mass, + "household_mass": household_mass, + } + ) +anchor_person_mass = next( + f["person_mass"] for f in asec_files if f["income_year"] == ANCHOR_INCOME_YEAR +) +pool_person_mass = 0.0 +pool_household_mass = 0.0 +for entry in asec_files: + scale = anchor_person_mass / len(ARCHIVES) / entry["person_mass"] + pool_person_mass += entry["person_mass"] * scale + pool_household_mass += entry["household_mass"] * scale + +# The ACS frame's weights: WGTP per occupied unit, PWGTP for a GQ person. +hus_path = args.acs_dir / "csv_hus.zip" +pus_path = args.acs_dir / "csv_pus.zip" +# The archives must be the ones the stacked pool pins. +acs_pins = json.loads( + (Path(immigration.__file__).parent / "acs_2024_1yr_sources.json").read_text() +) +acs_digests = {} +for artifact in acs_pins["artifacts"]: + path = args.acs_dir / artifact["filename"] + digest = _sha256(path) + if digest != artifact["sha256"]: + raise SystemExit(f"{path}: SHA-256 {digest} is not the pinned one") + acs_digests[artifact["filename"]] = digest +hus = _read_pums(hus_path, "hus", ["SERIALNO", "WGTP"]) +pus = _read_pums(pus_path, "pus", ["SERIALNO", "CIT", "POBP", "YOEP", "AGEP", "PWGTP"]) +unit_weight = pd.Series(hus["WGTP"].to_numpy(dtype=np.float64), index=hus["SERIALNO"]) +person_unit_weight = pus["SERIALNO"].map(unit_weight).to_numpy(dtype=np.float64) +gq = person_unit_weight == 0 +if pus.loc[gq, "SERIALNO"].duplicated().any(): + raise SystemExit("an ACS WGTP=0 unit holds more than one person") +acs_weight = np.where(gq, pus["PWGTP"].to_numpy(dtype=np.float64), person_unit_weight) +occupied = pus["SERIALNO"].drop_duplicates() +acs_household_mass = float(unit_weight.loc[occupied].clip(lower=0).sum()) + float( + acs_weight[gq].sum() +) +acs_person_mass = float(acs_weight.sum()) + +kappa = ARM_SHARE * pool_household_mass / acs_household_mass +asec_arm_person_mass = ARM_SHARE * pool_person_mass +acs_arm_person_mass = kappa * acs_person_mass +full_person_mass = asec_arm_person_mass + acs_arm_person_mass +# The stage labels ASEC rows at the full person mass; stacked ASEC rows carry +# asec_arm_person_mass of it. Candidate masses in the stage scale by rho. +rho = full_person_mass / pool_person_mass +to_stacked = asec_arm_person_mass / full_person_mass + +person = pd.DataFrame( + { + "CIT": pus["CIT"].to_numpy(dtype=np.float64), + "POBP": pus["POBP"].to_numpy(dtype=np.float64), + "YOEP": pus["YOEP"].to_numpy(dtype=np.float64), + "age": pus["AGEP"].to_numpy(dtype=np.float64), + } +) +profile = immigration._source_aware_immigration_profile(person, time_period=TIME_PERIOD) +ssn_codes = np.where(profile.is_citizen, 1, 3) + +receipt = json.loads(args.receipt.read_text()) +achieved = receipt["humanitarian_draw_achieved"] +draws = {} +# Each earlier draw's candidates and residual, in manifest order. +earlier: list[tuple[np.ndarray, float]] = [] +# Each ACS person's probability that no earlier parole or TPS draw took them. +free = np.ones(len(acs_weight), dtype=np.float64) +for draw in immigration.us_immigration_controls().humanitarian: + target = float(draw.target) + stage = float(achieved[draw.label]["population"]) + stage_exhausted = stage < target * (1 - 1e-9) + stage_mass = min(target, rho * stage) if stage_exhausted else min(target, stage) + immutable = stage_mass * to_stacked + residual = max(0.0, target - immutable) + candidates = immigration._humanitarian_draw_candidates( + draw, ssn_codes=ssn_codes, profile=profile + ) + acs_candidates = float(acs_weight[candidates].sum()) + supply = kappa * acs_candidates + # An earlier draw takes at most its residual, and at most its overlap with + # this draw's candidates. + removed = sum( + min(prior_residual, kappa * float(acs_weight[candidates & prior].sum())) + for prior, prior_residual in earlier + ) + earlier.append((candidates, residual)) + worst_supply = max(0.0, supply - removed) + exact_ssn_rule = draw.category in {"paroled_one_year", "tps"} + worst_reach = immutable + (min(worst_supply, residual) if exact_ssn_rule else 0.0) + expected_reach = None + if exact_ssn_rule: + available = kappa * float((acs_weight * free)[candidates].sum()) + take = min(residual, available) + if available > 0: + free[candidates] *= 1.0 - take / available + expected_reach = immutable + take + draws[draw.label] = { + "target": target, + "asec_stage_population": stage, + "asec_arm_population": immutable, + "acs_residual": residual, + "acs_candidates_at_acs_weights": acs_candidates, + "acs_arm_supply_bound": supply, + "earlier_draws_worst_case_removal": removed, + "shortfall_at_bound": max(0.0, residual - supply), + "projected_share_of_target": ( + (immutable + min(supply, residual)) / target if target > 0 else None + ), + "asec_arm_share_of_target": immutable / target if target > 0 else None, + "worst_case_share_of_target": worst_reach / target if target > 0 else None, + "expected_share_of_target": ( + expected_reach / target + if expected_reach is not None and target > 0 + else None + ), + } + print( + f"{draw.label:32s} target {target:9,.0f} ASEC arm {immutable:9,.0f} " + f"residual {residual:9,.0f} ACS bound {supply:9,.0f} " + f"short {max(0.0, residual - supply):9,.0f} " + f"worst-case share {worst_reach / target if target > 0 else float('nan'):.3f} " + "expected share " + + ( + f"{expected_reach / target:.3f}" + if expected_reach is not None and target > 0 + else "n/a" + ) + ) +positive = [entry for entry in draws.values() if entry["target"] > 0] +lowest_worst_case = min(entry["worst_case_share_of_target"] for entry in positive) +print(f"lowest worst-case share of target: {lowest_worst_case:.4f}") + +# OAW before this change took exact-year Afghan arrivals from 2021 on. +afghan = ( + (~profile.is_citizen) + & (profile.birth_country == 200) + & profile.exact_arrival + & (profile.arrival_year >= 2021) +) +oaw_years = { + "2021-2022": float(acs_weight[afghan & (profile.arrival_year <= 2022)].sum()), + "2021-2024": float(acs_weight[afghan].sum()), +} +print( + f"kappa {kappa:.4f}; ACS Afghan non-citizen arrivals at ACS weights: " + + ", ".join(f"{k} {v:,.0f}" for k, v in oaw_years.items()) +) + +if args.out is not None: + out = { + "receipt_file": args.receipt.name, + "receipt_sha256": _sha256(args.receipt), + "script_sha256": _sha256(Path(__file__)), + "acs": { + **acs_digests, + "household_mass": acs_household_mass, + "person_mass": acs_person_mass, + }, + "asec_archives": asec_files, + "asec_pool_household_mass": pool_household_mass, + "asec_pool_person_mass": pool_person_mass, + "arm_household_share": ARM_SHARE, + "acs_arm_weight_scale": kappa, + "asec_arm_person_mass": asec_arm_person_mass, + "acs_arm_person_mass": acs_arm_person_mass, + "afghan_noncitizen_arrivals_at_acs_weights": oaw_years, + "composition_band_floor": immigration._HUMANITARIAN_TARGET_RELATIVE_BAND[0], + "lowest_worst_case_share_of_target": lowest_worst_case, + "draws": draws, + } + args.out.write_text(json.dumps(out, indent=1, sort_keys=True) + "\n") diff --git a/docs/evidence/us-immigration-statuses-767/scripts/stage_receipt_pooled.py b/docs/evidence/us-immigration-statuses-767/scripts/stage_receipt_pooled.py new file mode 100644 index 0000000000..78cc02be21 --- /dev/null +++ b/docs/evidence/us-immigration-statuses-767/scripts/stage_receipt_pooled.py @@ -0,0 +1,487 @@ +"""Pooled default-pool stage receipt for the US immigration stage (#767, #779). + +Runs the US immigration source stage alone on the three Census ASEC person +files of main's default pool (income years 2023, 2024 and 2025: the 2024, 2025 +and 2026 ASEC, ``asec_sources.ASEC_DEFAULT_POOL_INCOME_YEARS``), then runs the +composition gate on its output and writes a JSON receipt of aggregates. + +- Each archive must match the SHA-256 that + ``us_runtime/education_assistance_source.py`` pins for it. +- Weights (``--weighting household``, the default) follow + ``asec_pool.pool_asec_sources`` with ``anchor_year=2024``, as + ``build_pooled_asec_unit_frame`` calls it for target year 2024: each person + carries the household's ``HSUP_WGT`` times 0.01, and each file is rescaled to + one third of income year 2024's weighted person population. + ``--weighting person`` uses ``MARSUPWT / 100 / 3`` instead. +- Each person carries the pool's full lineage: ``source_year`` (income year), + ``source_household_id`` (``PH_SEQ``) and ``source_person_id`` (``PERIDNUM``). + The tree's kernel picks which of them key its draws. The script wraps + ``immigration._stable_person_draws`` and records, for each draw call, the + key that reproduces its draws: ``full`` (all three columns), ``legacy`` + (``source_year`` and ``source_person_id``) or ``person_id``. +- ``--draw-keys full`` or ``--draw-keys legacy`` hands that key to every draw + call that brings none of its own, so runs of two trees can share their + draws and differ only in method. It needs a tree whose + ``_stable_person_draws`` takes ``keys``. +- Seed 0 and period 2024, Route A's ``--target-year``. + +A refusal is recorded as the receipt's ``error`` instead of raised. Main's tree +has no humanitarian controls or Pew universe, so its receipt carries the +composition and its own composition gate's verdict only. + +The archives are Census's public-use ASEC CSV releases, at the URLs +``education_assistance_source.py`` names: + + https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asecpub24csv.zip + https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asecpub25csv.zip + https://www2.census.gov/programs-surveys/cps/datasets/2026/march/asecpub26csv.zip + +Run it with any interpreter that has the workspace's dependencies, pointing at +the tree to measure (a checkout, or ``git archive packages`` extracted): + + .venv/bin/python -I docs/evidence/us-immigration-statuses-767/scripts/stage_receipt_pooled.py \\ + --archive-dir --out receipt.json + +``--labels-out`` also writes the row-level labels (a pickle) for +cross-tabulating two trees' runs. Those rows come from the public files but are +record-level, so they stay out of the repository. +""" + +from __future__ import annotations + +import argparse +import hashlib +import inspect +import json +import subprocess +import sys +import time +import zipfile +from pathlib import Path + +PERIOD = 2024 +ANCHOR_INCOME_YEAR = 2024 +ARCHIVES = ( + ( + 2023, + "asecpub24csv.zip", + "pppub24.csv", + "hhpub24.csv", + "cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09", + ), + ( + 2024, + "asecpub25csv.zip", + "pppub25.csv", + "hhpub25.csv", + "318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b", + ), + ( + 2025, + "asecpub26csv.zip", + "pppub26.csv", + "hhpub26.csv", + "fe819d0d2fc4470c282e76c247b8aa8b6054811619730307e9f2fe6186707d93", + ), +) + +parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) +parser.add_argument("src_root", type=Path, help="tree whose packages/*/src to run") +parser.add_argument( + "--archive-dir", + type=Path, + action="append", + required=True, + help="directory holding ASEC zips (repeatable; each zip is found by name)", +) +parser.add_argument("--out", type=Path, required=True) +parser.add_argument("--weighting", choices=("household", "person"), default="household") +parser.add_argument( + "--source-commit", + help="commit the tree came from, when src_root is not a git checkout", +) +parser.add_argument("--labels-out", type=Path) +parser.add_argument( + "--draw-keys", + choices=("native", "full", "legacy"), + default="native", + help="key the tree's draws as its kernel does (native) or force one key", +) +args = parser.parse_args() + +ROOT = args.src_root.resolve() +OUT = args.out.resolve() +for _src in sorted((ROOT / "packages").glob("*/src")): + sys.path.insert(0, str(_src)) + +import numpy as np +import pandas as pd + +import microcosm.build.us_runtime.immigration as immigration +from microcosm.build.source_runtime import SourceRuntimeConfig, run_source_stage +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights + +assert Path(immigration.__file__).resolve().is_relative_to(ROOT), immigration.__file__ + +KEY_COLUMNS = { + "full": ("source_year", "source_household_id", "source_person_id"), + "legacy": ("source_year", "source_person_id"), + "person_id": ("person_id",), +} +KEY_LABELS = { + "full": "source_year:source_household_id(PH_SEQ):source_person_id(PERIDNUM)", + "legacy": "source_year:source_person_id(PERIDNUM)", + "person_id": "person_id", +} +kernel_draws = immigration._stable_person_draws +if ( + args.draw_keys != "native" + and "keys" not in inspect.signature(kernel_draws).parameters +): + raise SystemExit( + f"--draw-keys {args.draw_keys} needs a tree whose _stable_person_draws " + "takes keys" + ) +draw_calls: list[dict[str, object]] = [] + + +def _joined_keys(person: pd.DataFrame, columns: tuple[str, ...]) -> pd.Series: + keys = person[columns[0]].astype(str) + for column in columns[1:]: + keys = keys + ":" + person[column].astype(str) + return keys.reset_index(drop=True) + + +def _hashed_draws(keys: pd.Series, *, seed: int, salt: str) -> np.ndarray: + """The kernel's draw for each key: blake2b of ``seed:salt:key`` over 2**64.""" + + return np.fromiter( + ( + int.from_bytes( + hashlib.blake2b( + f"{seed}:{salt}:{key}".encode(), digest_size=8 + ).digest(), + byteorder="big", + signed=False, + ) + / float(2**64) + for key in keys + ), + dtype=np.float64, + count=len(keys), + ) + + +def _recording_draws(person: pd.DataFrame, **kwargs: object) -> np.ndarray: + """Run the kernel's draw and record which key reproduces it.""" + + caller_keys = kwargs.get("keys") + forced = args.draw_keys != "native" and caller_keys is None + if forced: + kwargs["keys"] = _joined_keys(person, KEY_COLUMNS[args.draw_keys]) + draws = np.asarray(kernel_draws(person, **kwargs)) + salt = str(kwargs["salt"]) + if len(person) == 0: + draw_calls.append({"salt": salt, "rows": 0, "key": None, "forced": forced}) + return draws + rows = np.unique( + np.r_[ + np.arange(min(1000, len(person))), + np.arange(max(0, len(person) - 1000), len(person)), + ] + ) + + def reproduces(keys: pd.Series) -> bool: + expected = _hashed_draws(keys.iloc[rows], seed=int(kwargs["seed"]), salt=salt) + return bool(np.array_equal(expected, draws[rows])) + + named = [ + name + for name, columns in KEY_COLUMNS.items() + if set(columns).issubset(person.columns) + and reproduces(_joined_keys(person, columns)) + ] + if caller_keys is not None: + if not reproduces(pd.Series(caller_keys).astype(str).reset_index(drop=True)): + raise SystemExit(f"draw call {salt!r} does not use its caller's keys") + key = "caller" + (f"={named[0]}" if len(named) == 1 else "") + elif len(named) == 1: + key = named[0] + else: + raise SystemExit(f"draw call {salt!r}: keys {named} reproduce its draws") + draw_calls.append( + {"salt": salt, "rows": int(len(person)), "key": key, "forced": forced} + ) + return draws + + +immigration._stable_person_draws = _recording_draws + + +def _archive_path(name: str) -> Path: + found = [d / name for d in args.archive_dir if (d / name).is_file()] + if not found: + raise SystemExit(f"{name} is in none of {args.archive_dir}") + return found[0] + + +def _git(*command: str) -> str | None: + result = subprocess.run( + ["git", "-C", str(ROOT), *command], capture_output=True, text=True + ) + return result.stdout.strip() if result.returncode == 0 else None + + +git_top = _git("rev-parse", "--show-toplevel") +if git_top is not None and Path(git_top).resolve() == ROOT: + source_commit = _git("rev-parse", "HEAD") + source_commit_from = "git" + source_tree_clean = _git("status", "--porcelain", "--", "packages") == "" + if args.source_commit and args.source_commit != source_commit: + raise SystemExit( + f"--source-commit {args.source_commit} is not the checkout's " + f"HEAD {source_commit}" + ) +else: + source_commit = args.source_commit + source_commit_from = "argument" if source_commit else None + source_tree_clean = None + +columns = list( + dict.fromkeys( + [ + *immigration.US_IMMIGRATION_REQUIRED_SOURCE_COLUMNS, + "MARSUPWT", + "PERIDNUM", + "PH_SEQ", + ] + ) +) +parts = [] +archives = [] +raw_population = {} +for income_year, archive_name, member, household_member, pinned in ARCHIVES: + path = _archive_path(archive_name) + digest = hashlib.sha256(path.read_bytes()).hexdigest() + assert digest == pinned, (path, digest) + with zipfile.ZipFile(path) as archive: + names = archive.namelist() + name = next(n for n in names if n.lower().endswith(member)) + with archive.open(name) as handle: + frame = pd.read_csv(handle, usecols=columns) + name = next(n for n in names if n.lower().endswith(household_member)) + with archive.open(name) as handle: + household = pd.read_csv(handle, usecols=["H_SEQ", "HSUP_WGT"]) + household_weight = household.drop_duplicates("H_SEQ").set_index("H_SEQ")["HSUP_WGT"] + frame["household_weight"] = frame["PH_SEQ"].map(household_weight) * 0.01 + assert not frame["household_weight"].isna().any(), income_year + frame["source_year"] = income_year + frame["source_household_id"] = frame["PH_SEQ"].to_numpy() + frame["source_person_id"] = frame["PERIDNUM"].astype(str) + raw_population[income_year] = float(frame["household_weight"].sum()) + parts.append(frame) + archives.append( + { + "income_year": income_year, + "archive": archive_name, + "sha256": digest, + "rows": int(len(frame)), + "peinusyr_max": int(frame["PEINUSYR"].max()), + "household_weighted_persons": raw_population[income_year], + "marsupwt_persons": float(frame["MARSUPWT"].sum() / 100.0), + } + ) +target_population = raw_population[ANCHOR_INCOME_YEAR] +for frame in parts: + year = int(frame["source_year"].iloc[0]) + if args.weighting == "household": + scale = target_population / len(ARCHIVES) / raw_population[year] + frame["person_weight"] = frame["household_weight"] * scale + else: + frame["person_weight"] = frame["MARSUPWT"] / 100.0 / len(ARCHIVES) + frame.drop(columns=["household_weight"], inplace=True) +person = pd.concat(parts, ignore_index=True) +person["person_id"] = np.arange(1, len(person) + 1, dtype=np.int64) +person["person_household_id"] = 0 + +started = time.monotonic() +error = None +try: + output = run_source_stage( + immigration.us_immigration_stage_spec(), + tables={"person": person}, + operation_handlers={ + "derive_immigration_status": ( + immigration.derive_us_immigration_status_from_manifest + ) + }, + config=SourceRuntimeConfig(seed=0, target_year=PERIOD), + ) +except Exception as exc: # the receipt records a refusal instead of hiding it + error = f"{type(exc).__name__}: {exc}" +stage_seconds = time.monotonic() - started + +receipt = { + "source_commit": source_commit, + "source_commit_from": source_commit_from, + "source_tree_clean": source_tree_clean, + "immigration_module_sha256": hashlib.sha256( + Path(immigration.__file__).read_bytes() + ).hexdigest(), + "script_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "archives": archives, + "seed": 0, + "time_period": PERIOD, + "weights": ( + "HSUP_WGT * 0.01 by PH_SEQ, each file rescaled to one third of income " + "year 2024's weighted person population (asec_pool.pool_asec_sources, " + "anchor_year=2024)" + if args.weighting == "household" + else "MARSUPWT / 100 / 3 per file" + ), + "draw_keys": { + "requested": args.draw_keys, + "used": { + name: KEY_LABELS.get(name, name) + for name in sorted({str(call["key"]) for call in draw_calls}) + }, + "calls": draw_calls, + }, + "stage_seconds": round(stage_seconds, 1), + "error": error, +} +if error is None: + weights = output["person_weight"].to_numpy(dtype=np.float64) + status = output["immigration_status_str"].astype(str).to_numpy() + ssn = output["ssn_card_type"].astype(str).to_numpy() + age = output["A_AGE"].to_numpy() + code = output["PEINUSYR"].to_numpy() + year = output["source_year"].to_numpy() + composition = {} + for value in sorted(set(status)): + mask = status == value + composition[value] = { + "weighted": float(weights[mask].sum()), + "rows": int(mask.sum()), + "by_income_year": { + str(y): float(weights[mask & (year == y)].sum()) + for y in (2023, 2024, 2025) + }, + "code_29_weighted": float(weights[mask & (code == 29)].sum()), + "under_19_weighted": float(weights[mask & (age < 19)].sum()), + } + receipt.update( + { + "rows": int(len(output)), + "composition": composition, + "none_ssn_iff_undocumented": bool( + ((ssn == "NONE") == (status == "UNDOCUMENTED")).all() + ), + } + ) + # Every tree runs its own composition gate. This PR's tree also defines the + # humanitarian controls and the Pew universe; main's has neither. + gate_person = output.drop(columns=["person_weight"]).reset_index(drop=True).copy() + ids = np.arange(1, len(gate_person) + 1, dtype=np.int64) + for entity in US_SCHEMA.group_entities: + gate_person[US_SCHEMA.membership_column(entity)] = ids + frame = Frame( + { + "person": gate_person, + **{ + entity: pd.DataFrame({US_SCHEMA.id_column(entity): ids}) + for entity in US_SCHEMA.group_entities + }, + }, + US_SCHEMA, + {"household": Weights(weights, WeightKind.DESIGN)}, + ) + gate_parameters = inspect.signature( + immigration.us_immigration_composition_gate + ).parameters + gate_kwargs: dict[str, object] = {} + if "time_period" in gate_parameters: + gate_kwargs["time_period"] = PERIOD + if not hasattr(immigration, "us_immigration_controls"): + gate = immigration.us_immigration_composition_gate(frame, **gate_kwargs) + receipt.update( + {"gate_passed": bool(gate.passed), "gate_failures": list(gate.failures)} + ) + else: + labor_force = np.isin(output["A_LFSR"].to_numpy(), [1, 2, 3, 4]) & (age >= 16) + controls = immigration.us_immigration_controls() + targets: dict[str, float] = {} + for draw in controls.humanitarian: + targets[draw.status] = targets.get(draw.status, 0.0) + float(draw.target) + pew = np.isin(status, immigration._PEW_UNAUTHORIZED_STATUS_VALUES) | ( + (status == "CUBAN_HAITIAN_ENTRANT") & (ssn == "NON_CITIZEN_VALID_EAD") + ) + gate = immigration.us_immigration_composition_gate( + frame, controls=controls, **gate_kwargs + ) + receipt.update( + { + "humanitarian_targets_by_status": targets, + "ratio_to_target": { + k: composition.get(k, {}).get("weighted", 0.0) / v if v else None + for k, v in targets.items() + }, + "pew_universe": float(weights[pew].sum()), + "pew_universe_anchor": float(controls.undocumented.population_anchor), + "pew_labor_force": float(weights[pew & labor_force].sum()), + "pew_labor_force_control": float(controls.undocumented.workers), + "gate_passed": bool(gate.passed), + "gate_failures": list(gate.failures), + "humanitarian_draw_achieved": gate.details.get( + "humanitarian_draw_achieved" + ), + "humanitarian_target_relative_band": list( + immigration._HUMANITARIAN_TARGET_RELATIVE_BAND + ), + } + ) +OUT.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n") +if error is None and args.labels_out is not None: + pd.DataFrame( + { + "source_year": output["source_year"].to_numpy(), + "source_person_id": output["source_person_id"].astype(str).to_numpy(), + "person_weight": output["person_weight"].to_numpy(dtype=np.float64), + "immigration_status_str": output["immigration_status_str"].astype(str), + "ssn_card_type": output["ssn_card_type"].astype(str), + } + ).to_pickle(args.labels_out) + +print("weighting:", args.weighting) +print("source commit:", source_commit, f"({source_commit_from})") +print( + "draw keys:", + args.draw_keys, + "->", + receipt["draw_keys"]["used"], + f"over {len(draw_calls)} draw calls", +) +print("archives:", [(a["income_year"], a["rows"], a["peinusyr_max"]) for a in archives]) +if error is not None: + print("REFUSED:", error) + raise SystemExit(0) +print(f"stage {stage_seconds:.0f}s on {len(output)} rows") +ratios = receipt.get("ratio_to_target", {}) +for value, row in composition.items(): + ratio = ratios.get(value) + ratio_text = f" ratio {ratio:.2f}" if ratio is not None else "" + years = " ".join(f"{y}:{v / 1e6:.3f}" for y, v in row["by_income_year"].items()) + print( + f" {value:26s} {row['weighted'] / 1e6:8.3f}M{ratio_text} [{years}]" + f" code29 {row['code_29_weighted'] / 1e6:.3f}M" + f" u19 {row['under_19_weighted'] / 1e6:.3f}M" + ) +print("NONE ssn iff UNDOCUMENTED:", receipt["none_ssn_iff_undocumented"]) +if "pew_universe" in receipt: + print( + f"Pew universe {receipt['pew_universe'] / 1e6:.3f}M vs anchor " + f"{receipt['pew_universe_anchor'] / 1e6:.1f}M; labor force " + f"{receipt['pew_labor_force'] / 1e6:.3f}M vs control " + f"{receipt['pew_labor_force_control'] / 1e6:.1f}M" + ) +print("gate passed:", receipt["gate_passed"], "failures:", receipt["gate_failures"]) diff --git a/docs/evidence/us-immigration-statuses-767/scripts/xtab_labels.py b/docs/evidence/us-immigration-statuses-767/scripts/xtab_labels.py new file mode 100644 index 0000000000..47b5617881 --- /dev/null +++ b/docs/evidence/us-immigration-statuses-767/scripts/xtab_labels.py @@ -0,0 +1,86 @@ +"""Cross-tabulate two stage runs' immigration labels (millions of persons). + +Usage: python -I xtab_labels.py MAIN_LABELS.pkl HEAD_LABELS.pkl [OUT.json] + +Rows align on (source_year, source_person_id); the script refuses unequal +row sets or person weights. Prints the README's main-by-head table. +""" + +import json +import sys + +import numpy as np +import pandas as pd + +ORDER = [ + "CITIZEN", + "LEGAL_PERMANENT_RESIDENT", + "UNDOCUMENTED", + "DACA", + "CUBAN_HAITIAN_ENTRANT", + "PAROLED_ONE_YEAR", + "REFUGEE", + "ASYLEE", + "TPS", +] + + +def load(path): + frame = pd.read_pickle(path) + key = frame["source_year"].astype(str) + ":" + frame["source_person_id"].astype(str) + if key.duplicated().any(): + raise SystemExit(f"{path}: duplicate person keys") + return frame.set_index(key) + + +main_path, head_path = sys.argv[1:3] +main, head = load(main_path), load(head_path) +if set(main.index) != set(head.index): + raise SystemExit("the two runs hold different persons") +head = head.loc[main.index] +if not np.allclose(main["person_weight"], head["person_weight"], rtol=0, atol=1e-9): + raise SystemExit("the two runs weight persons differently") +weight = main["person_weight"].to_numpy() +table = ( + pd.DataFrame( + { + "main": main["immigration_status_str"].to_numpy(), + "head": head["immigration_status_str"].to_numpy(), + "w": weight, + } + ) + .groupby(["main", "head"])["w"] + .sum() + .div(1e6) + .unstack(fill_value=0.0) +) +statuses = [s for s in ORDER if s in set(table.index) | set(table.columns)] +statuses += sorted((set(table.index) | set(table.columns)) - set(statuses)) +table = table.reindex(index=statuses, columns=statuses, fill_value=0.0) +pd.set_option("display.width", 200) +print(table.round(3).to_string()) +totals = { + "main": {s: round(float(table.loc[s].sum()), 3) for s in statuses}, + "head": {s: round(float(table[s].sum()), 3) for s in statuses}, +} +print(json.dumps(totals, indent=1)) +none_main = main["ssn_card_type"].eq("NONE").to_numpy() +none_head = head["ssn_card_type"].eq("NONE").to_numpy() +und_main = main["immigration_status_str"].eq("UNDOCUMENTED").to_numpy() +und_head = head["immigration_status_str"].eq("UNDOCUMENTED").to_numpy() +print( + "NONE iff UNDOCUMENTED:", + bool((none_main == und_main).all()), + bool((none_head == und_head).all()), +) +if len(sys.argv) > 3: + out = { + "main_labels": main_path, + "head_labels": head_path, + "millions": { + r: {c: round(float(table.loc[r, c]), 6) for c in statuses} for r in statuses + }, + "totals": totals, + } + with open(sys.argv[3], "w") as stream: + json.dump(out, stream, indent=1, sort_keys=True) diff --git a/docs/spec-engine.md b/docs/spec-engine.md index 33da8b2e36..4574077397 100644 --- a/docs/spec-engine.md +++ b/docs/spec-engine.md @@ -174,7 +174,7 @@ run_provenance_identity: spec_binding: # {country, schema_id, schema_version, # canonicalizer_version, spec_sha256} authority_versions: # semantic contract version per named - # authority (stacked authority v10 is + # authority (stacked authority v11 is # the live example) — field, bump # rules, and precedence below code_inventory_digest # builder_code_identity + kernel set diff --git a/docs/us-multispine-operator-ordering.md b/docs/us-multispine-operator-ordering.md index 19a1f4de37..de7a0b5be4 100644 --- a/docs/us-multispine-operator-ordering.md +++ b/docs/us-multispine-operator-ordering.md @@ -247,13 +247,14 @@ by_origin_battery manifest uses schema version 2 and binds its support contract and receipt. The canonical stacked authority is version 12, the outer stacked checkpoint materializer uses version 13, and the stacked pool stage checkpoint - materializer uses version 7. + materializer uses version 8 (the source-aware humanitarian status surface). The outer base identity binds primary-QRF version 6, the ACS universe and QBI reconciliation contracts, the tail schema and support contract, and late-producer registry schema version 17, including execution-receipt - contract 4, transition authority 2, and resource-semantics receipt 2. The - primary execution config is version 6 (it binds the recipient-share bound, - #1259), its portable worker identity is + contract 4, transition authority 2, resource-semantics receipt 2, and the + signed static and derivation-mode semantics of every virtual DAG resource. + The primary execution config is version 6 (it binds the recipient-share + bound, #1259), its portable worker identity is version 1, and its checkpoint input sidecar is version 2. The companion pool manifest uses schema version 10. Older outer authority or materializer payloads are stale; primary-QRF @@ -312,7 +313,7 @@ by_origin_battery final publication reject a missing, stale, or reissued authority; NON-CANONICAL test receipts cannot ship. - Outer materializer v10 also embeds one signed resource-semantics row for + Outer materializer v13 also embeds one signed resource-semantics row for every DAG producer. Static configs are exact; donor tables are bound by the declared canonical scalar-content codec; primary and transfer banks name their outer/stage identity derivations; and source receipts name their @@ -594,6 +595,49 @@ authorization tuple (`gate`, `plan_sha256`, `prompt_sha256`, `checklist_sha256`, and `evidence_sha256`) checked as data; this boundary does not claim public-key or cryptographic signature verification. +Attested legacy scoring checks a schema-9 pool against reconstructions that +follow today's late-producer registry, as main does, except for the two +producer contracts microcosm #767 changed. The frozen resource +`us/legacy_schema16_late_producer_schedule.json`, which loads only if its +recomputed payload SHA-256 equals +`02e618cc656eb39990ed99dca2b30a52794e01e2b06a3c2df87ca4a7d85ab086`, supplies two +things: the schema-16 execution receipt contract (registry schema 16, receipt +contract v3, transition authority v1, and the top binding from before the +constrained immigration reconciliation), and the input inventories of the two +producers #767 changed, the immigration source stage +`with_us_immigration_inputs` and its transfer group +`transfer:person/source_operator_immigration`. The legacy contracts derive +those two producers from their schema-16 inventories (57 and 93 inputs, where +today's contracts have 56 and 100), and legacy scoring refuses unless each +hashes to the digest the schema-16 registry derived. Every other legacy +contract is today's. The pool's late-producer schedule, its execution rows and +its stacked authority (the v11 authority, rebuilt from today's anchors and that +schedule) must match these reconstructions. They keep main's own changes since +schema 16: microcosm #1239 and #1258 moved the primary-PUF S-corporation outputs +and the itemization transfer batches, so the legacy schedule and authority no +longer hash to the sealed payload +`02e618cc656eb39990ed99dca2b30a52794e01e2b06a3c2df87ca4a7d85ab086` and authority +`e660a8ce42b69a39d29c5f0ec37264bc69d61b03f27adc386336ec8889531bb2`. Late-transfer +execution rows also bind today's `acs_transfer_execution_contract_identity` on +this path, as on main, so #767's schema-3 transfer contract reaches it there; +main's own post-transfer changes since `7c4a5ac0` already moved that identity +for all nineteen transfer groups. Post-clone source execution rows likewise +bind today's source-stage binding, which hashes the whole packaged +`us/source_stages.json` and carries each bound stage's resolved spec. #767 +rewrote the `immigration_status` stage, so its change reaches the fifteen +source rows that bind a stage spec (direct housing assistance binds none). +Main has changed `source_stages.json` in fourteen commits since `7c4a5ac0`, so +on main too this binding refuses a pool whose rows bound an older file. +Restoring all of these to sealed history is separate work. The pool's operator order must be the +literal eleven-step order ending at `by_origin_battery`, from before +`us_immigration_composition_gate` existed, and the loader replays its +immigration transfer targets as ordinary transfer receipts, without the paired +reconciliation and QRF pattern evidence current pools must carry. None of these +relaxations applies without `legacy_worker_authentication`, and the loader +refuses a current-contract artifact whose version numbers were merely lowered +(at the operator order, and at the legacy schedule if the artifact borrows the +historical order). + Current manifests, diagnostics, `release_manifest.json`, and scoring receipts surface `worker_execution_authentication`: manifest, execution-config, and worker schema versions, `semantic_identity_sha256`, and audit aliases. @@ -943,9 +987,9 @@ Registry schema version 17 and execution-receipt contract version 4 bind the canonical input declarations, outputs, edges, waves, exact kind-specific virtual-resource bindings, content-hashed execution-row schema, and immutable transition authority version 2. The schedule SHA-256 is -`f1fb754e6ade602be394837bc0d7d091f2fba4b4b83fb00f59a0258fe56eb5da`; +`46cea046d2fb8223d32772bd6a7807c52791640943949154982dcc52d93fa9eb`; the full payload SHA-256 is -`76a79c9d05cfde1ec1c812de530b5080e6bc212f9c9e693eba628711d6f553e1`. +`3f1daff0b5840d70d818e256515b2cf5d0a28bf4ad9f932deece3a56964047e0`. Reversing registry iteration produces those same bytes. The virtual-resource payload ledger is independently versioned: ACS-universe @@ -971,7 +1015,7 @@ and valid. Neither receipt authorizes an upstream null. | PUF raw predictor sources | Every filing-status, count, and income component is observed in its declared source universe. Raw WAGP/SEMP authority is present and agrees with mapped leaves; a cross-grain source collision is rejected. A null on any eligible member fails before coercion. | Structure supplies status/count; ACS-native or ASEC-carried earnings supply earnings; early transfer supplies interest, dividends, and gains. | No. ACS under-15 WAGP/SEMP blanks are an exact source-universe state, not transfer starvation; all other source nulls fail. | | PUF tax-unit features | Every clone-1 recipient has a finite feature vector. Post-aggregation NaN, `+inf`, and `-inf` are counted by named predictor and rejected before fitting; none is coerced or snapped to zero. | Universe-aware person sums plus tax-unit structural inputs. | No. Eligible member values must be complete; the only special case is an all-child unit whose numeric-zero predictor is explicitly owned and counted by the named universe-zero rule. | | Primary QRF banks and chain | Donor/recipient banks are immutable; target order and RNG prefix are contiguous; all targets complete; live recipient identity, source-universe receipt, and feature digest match before finalization. | The processed full PUF donor and strict recipient checkpoint initialized above. | No. Mutation or missing receipt invalidates the bank; it cannot resume under legacy semantics. | -| Outer pool checkpoint identity and resume | Primary-QRF schema v6, primary execution config v5, portable worker identity v1, tail-manifest schema v2, late-registry schema v17/receipt contract v4, outer stacked materializer v13/authority v12, stacked pool-stage materializer v7, pool manifest schema v10, and the ACS-universe, QBI-mutation, tail-support, late-DAG, and signed virtual-resource-semantics identities must match exactly before any cached stage is discovered. The retiring legacy envelope remains manifest schema v4/materializer v3. | Fresh input pins, live stack receipt, scale controls, code identity, and all semantic contract identities. | No. An older stacked materializer or authority payload is stale; a self-consistent old receipt cannot reopen a checkpoint. Primary-QRF v6 remains current. | +| Outer pool checkpoint identity and resume | Primary-QRF schema v6, primary execution config v5, portable worker identity v1, tail-manifest schema v2, late-registry schema v17/receipt contract v4, outer stacked materializer v13/authority v12, stacked pool-stage materializer v8, pool manifest schema v10, and the ACS-universe, QBI-mutation, tail-support, late-DAG, and signed virtual-resource-semantics identities must match exactly before any cached stage is discovered. The retiring legacy envelope remains manifest schema v4/materializer v3. | Fresh input pins, live stack receipt, scale controls, code identity, and all semantic contract identities. | No. An older stacked materializer or authority payload is stale; a self-consistent old receipt cannot reopen a checkpoint. Primary-QRF v6 remains current. | | Clone-2 capital-gains tail | Each filing status requires as many eligible recipient households as selected q99.5 donors. Eligibility requires unique single-tax-unit PUF-detail lineage and half-weight capacity for the global maximum assigned donor weight. An adequate status assigns every selected donor once; a thin status skips as a whole with a named, counted `insufficient_support` receipt. | Completed clone-1 QRF output and full PUF tail donors. At 1%, `SINGLE` and `HEAD_OF_HOUSEHOLD` attach, `JOINT` and `SEPARATE` skip, and zero-requirement `SURVIVING_SPOUSE` is `not_applicable`. | No widening or partial attachment is permitted. All 22 AGI bands provide nearest-first fallback only inside a status. Universe-aware PUF recipients remain eligible, including explicitly receipted empty-universe tax units. | | Late producer DAG | Before any callback, all declared inputs are filled on their required scopes or carry an input-specific counted absence receipt; numeric inputs are finite. The exact derived order, readiness rows, once-only source finalizer, and bounded transfer receipts must validate. | ACS earnings-universe materialization, primary PUF/tail, 16 source producers, and 19 bounded transfer groups execute in six derived waves. | No. The refusing producer names the unfilled input and its declared producing stage. A cycle fails at import with its path. | | Late transfer completion | Every declared PUF-clone or ASEC source-producer cell is nonnull; all complementary recipients are filled; the allowed count for both unmodeled and residual rows is zero. | Forty-five PUF and 29 source targets, with two overlaps, supply the 72-target late surface. | No. A missing producer or recipient value is terminal at this boundary. | diff --git a/docs/us-release-build-rule.md b/docs/us-release-build-rule.md index 111d41d8d9..29e26b7c8f 100644 --- a/docs/us-release-build-rule.md +++ b/docs/us-release-build-rule.md @@ -254,7 +254,7 @@ known violations at run time: Tests pin the watched lists against the installed engine's formula sources, exercise each guard on a real engine, and compare the shipped baseline plans -(18 smoke keys, 75 validation keys and 1 demographics key) against whole-file +(23 smoke keys, 75 validation keys and 1 demographics key) against whole-file values on a small written H5 in three batches. On 2026-09-24, the sweep below completed on a written fixture containing eight diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py index e0be9d2e11..6e4da63b85 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py @@ -26,9 +26,9 @@ ) from .schemas import load_schema_registry -EXPECTED_AUTHORED_FIELD_COUNT = 32609 -EXPECTED_RESOLVED_BINDING_FIELD_COUNT = 9_785 -EXPECTED_CONFIGURATION_FIELD_COUNT = 42394 +EXPECTED_AUTHORED_FIELD_COUNT = 32_749 +EXPECTED_RESOLVED_BINDING_FIELD_COUNT = 10_032 +EXPECTED_CONFIGURATION_FIELD_COUNT = 42_781 class FieldUsageError(AssertionError): @@ -369,8 +369,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "bdd9e79157590fade42d28cc2b5d5c8b58e13903af75375a0e3235c94d2a8339", ), "country_manifest": ( - 104, - "8a186065f5b8ffc59bc3f62fe927975e6f36bc1e913aa61761652c9a8aa67988", + 107, + "2c2099d3d5ef3d664f74a2e732870391022a852cf49d5cf96810c9a984d2b20b", ), "generated_authorities": ( 8_614, @@ -417,12 +417,12 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "e1dd7dc5123ab0f39d08ea4939d98dd09a6fdb8e7449a7ca3125fb1ddbd5b4e9", ), "imputation_producer_graph": ( - 24511, - "5869ddc4afa6eb0c09f290b709e75467decc6929e7f79c98341878928e6169fc", + 24577, + "cd6f3f0bb0cca334bec3c133060352b1b99a4e7be118924de8741a9da748325c", ), "imputation_transfer_execution": ( - 110, - "4e1cdee017270e10a123df1e37be3d7c3a88056684c09eace90bd102c7d9ec83", + 128, + "072e12dcb7b0ec4af2d954c3280cca7ebad3496826b20de3bbc713ec1e9344e9", ), "imputation_waiver_records": ( 70, @@ -449,12 +449,12 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "6a781915fd491d2c4b16d2b7d482f69cf362c904130093c59f9629f7a319269b", ), "resolved_seed_protocol": ( - 827, - "525ed8621baa2c6e0c0970a6cec85b3b250f8aadfe93750a08deb0d9f240f15e", + 1035, + "0da36c3d74127193e82c128071fb687028d4e6cf096f045c8d96d3c52391bc41", ), "resolved_seed_site_bindings": ( - 277, - "368fc19a8f07c4abf38ebcf2fbc4414d33c403c0f1f9881c2d2e3f5d6160feb6", + 316, + "199ec98045871070860164b522892327ab840b48c0c6a53972ebc0ff0d7d091f", ), "resolved_vintage_authorities": ( 67, @@ -481,8 +481,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "d6782c5de5bbed1bdc6bf653c4a6d4aadcad4ccc72d35e1092e130fcb04680a3", ), "source_stages": ( - 1_926, - "8bd6122267288e2df25b2880a43191865707a240892bec484cb830ee72404964", + 1939, + "0e43f91c8617181bdceeef7570dd5a7e871f77e1acd3961f9b7203f825f20882", ), "spine_assembly_household_mass_shares": ( 2, @@ -501,16 +501,16 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "cf0000464013118955571dac2691d4cc1b900c97c6570349b489678cdf937649", ), "spine_pipeline_contract": ( - 88, - "ae011455154bae0df3913ca9a056058a909d913439b292cb46c11f63c7d0d9a3", + 89, + "8d281f8f1e2f9914684150b0424ff816d0ec09ed0d9053ea5a64e877a86cc35b", ), "spine_sampling": ( 17, "29c6c1b3243e178783e7ab139993ba3a9b42d62edd4e9e4e1f3b28688daf2c6d", ), "spine_seed_site_bindings": ( - 277, - "dfa7ae701f62f7b1f06954798d29cfba3fc53a0be21e1751646d1f5bafd6071f", + 316, + "bb848f38e068858ac23e964a4ffbbf5542253da50515c72fe7753d8c56912486", ), "spine_support_roles": ( 29, diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 3206916b84..185ec9006a 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -296,6 +296,19 @@ "pregnancy_assignment", "wic_claim_assignment", "snap_discretionary_exemption_assignment", + "immigration_humanitarian_paroled_one_year_afghanistan_assignment", + "immigration_humanitarian_paroled_one_year_ukraine_assignment", + "immigration_humanitarian_paroled_one_year_nicaragua_assignment", + "immigration_humanitarian_paroled_one_year_venezuela_assignment", + "immigration_humanitarian_refugee_assignment", + "immigration_humanitarian_asylee_assignment", + "immigration_humanitarian_deportation_withheld_assignment", + "immigration_humanitarian_tps_venezuela_assignment", + "immigration_humanitarian_tps_el_salvador_assignment", + "immigration_humanitarian_tps_honduras_assignment", + "immigration_humanitarian_tps_nicaragua_assignment", + "immigration_humanitarian_tps_nepal_assignment", + "immigration_humanitarian_tps_other_designated_assignment", "immigration_ead_workers_assignment", "immigration_ead_students_assignment", "ssi_take_up_assignment", @@ -348,15 +361,15 @@ EXPECTED_HASHES = { "acs_group_predictors": "a927bb7ecf3e84f54c93583ab79318654514ac546aefafba67da5285615fbd60", "acs_person_predictors": "878c788a6f037d7aca12b3586ea034eff04f3034ffa11935a736493042551f25", - "authority": "bf681aedc859fd6398ef72da68f584069b7d25b3085c549bf99f82e8324c4eef", + "authority": "7cdedd5bf056f1407910eddcd40ad0660bd7ef2d6defe224612c02b3476cbf15", "early_families": "4aa9f736fd76e83955477ad1667e58f48f264783f05bdc7f0102cd32d61323bd", - "full_checkpoint": "ecf64f2bfbb59ffaaa717d8a09d38e56a38d6d43c1aa0a209d57492b038fb19a", + "full_checkpoint": "6aefc9bd351e4ca81998012de0eceacab9c802334d1ae2582f29cde4a6cd5036", "gap_fill_schedule": "1c31f9868f7884347cc19cf1ff65da43f950b9114941a715bab168246db414a7", - "graph_nodes": "1b208fff4f6d878c522147fee617177544bda8963c940f98a4bfc37507cb955f", + "graph_nodes": "2606b852c3ecdc7ae0f7ff5b30c24a5ce6834b218db0898b44b18b906daa9558", "geography_assignment": "44e7f5f6cd7ceeef85d532c3d46306d03de4f4ee811376e6d29920742746b65c", "late_families": "7857518a5327a816acae80d55a04f43e7445a5e47be822d3ccb747bcb375e199", - "late_resource_semantics": "4464b646c45dff39a500e07bf60a8505d45725712fc27dcfc6de1e839fd6b0c3", - "late_schedule": "f1fb754e6ade602be394837bc0d7d091f2fba4b4b83fb00f59a0258fe56eb5da", + "late_resource_semantics": "5653131d24487b1779e7c0a60deaed10d3a2a0b5a876c65dda40a6df0ef4e436", + "late_schedule": "46cea046d2fb8223d32772bd6a7807c52791640943949154982dcc52d93fa9eb", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "6655e37f15befe053f8de2583c216f97098e4c2bde526f315299c2858c97f055", # #1258: puf_support's source bytes carry the S-corp leaf from the @@ -380,9 +393,12 @@ # microcosm#1259's recipient-share bound in puf_support moves both again. # microcosm#1261 moves both again: the PUF donor carries partnership and # S-corp line-32 shares as separate leaves. - "seed_map": "fa48af060da522bb1edcb80580098f258763757139292602759cbccbfff76b37", - "seed_protocol": "5840ac6cefd5ce3d2172aca9e3bfc08d2344e97a8e9b42a723a4edcaa9f8ff53", - "source_manifest": "dc3879886d2a69f20fb982a675d7fb322c0e4017afef6471be54b0ee3cbd2af6", + # microcosm#767 moves the seed map, the protocol and the source manifest: + # the attested immigration kernel gains the humanitarian status draws and + # the immigration stage gains cited humanitarian stock targets. + "seed_map": "4556389f3383a2d2600857cf7117b9306f8f45a9a6231ce2a8fe62bb6a5e388c", + "seed_protocol": "0f4550f80efd0273589b533a4a453e74b06aa71b9ed1e08fdc016ef3d9d27fef", + "source_manifest": "e9a70a60c3db9250066ee607d5e1fb07cd9d6a54424ab9eb9405f5e65b5a59b8", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", "tail": "565f0ceeadab69d3e96a886a00dd0305f7425fa3acde24d12ba48db2b37d5fc0", } @@ -450,14 +466,14 @@ "primary_targets": 66, "producer_authored_outputs": 92, "producer_compiled_outputs": 230, - "producer_inputs": 2_747, + "producer_inputs": 2_753, "producer_nodes": 38, "producer_virtual_resources": 75, "release_rungs": 5, "resolved_references": 337, - "seed_owner_bindings": 112, + "seed_owner_bindings": 125, "seed_owner_rows": 54, - "seed_sites": 53, + "seed_sites": 66, "seed_streams": 14, "source_operators": 16, "source_stages": 38, @@ -1033,7 +1049,7 @@ def add( "producer_inputs_exact", clauses={ "producer input rows differ": inputs_exact, - "input row count differs": input_count == 2747, + "input row count differs": input_count == 2753, }, homes=("/imputation/producer_graph/nodes/*/inputs",), consumers=( @@ -1041,7 +1057,7 @@ def add( "compiler_ir.node_slices", ), observed={"rows": input_count}, - expected={"rows": 2747, "relation": "source rows preserved exactly"}, + expected={"rows": 2753, "relation": "source rows preserved exactly"}, ) outputs_exact = set(expected_outputs) == set(compiled_by_id) and all( _json_equal( @@ -1799,7 +1815,7 @@ def add( "seed_site_definitions_exact", clauses={ "seed site definitions differ": site_definitions_exact, - "seed site count differs": len(protocol_sites) == 53, + "seed site count differs": len(protocol_sites) == 66, }, homes=("/bundle/seed_protocol",), consumers=("compiler_ir.seed_stream_map.sites", "compiler_ir.node_slices"), @@ -1807,7 +1823,7 @@ def add( "sites": len(protocol_sites), "sha256": sha256_json([site.to_wire() for site in protocol.sites]), }, - expected={"sites": 53, "relation": "all site fields preserved exactly"}, + expected={"sites": 66, "relation": "all site fields preserved exactly"}, ) binding_by_site = {binding.site: binding for binding in spec.seed_site_bindings} binding_exact = set(binding_by_site) == set(compiled_sites) and all( @@ -1822,7 +1838,7 @@ def add( "owner binding count differs": sum( len(site.owners) for site in compiled_sites.values() ) - == 112, + == 125, "one or more sites have no owner": all( site.owners for site in compiled_sites.values() ), @@ -1832,7 +1848,7 @@ def add( observed={ "bindings": sum(len(site.owners) for site in compiled_sites.values()) }, - expected={"bindings": 112, "coverage": "all 53 sites"}, + expected={"bindings": 125, "coverage": "all 66 sites"}, ) expected_owner_sites: dict[tuple[str, str], list[str]] = {} for site in protocol.sites: @@ -1876,7 +1892,7 @@ def add( "seed groups overlap": groups_disjoint, "seed groups do not cover the protocol exactly": set(grouped_ids) == set(protocol_sites), - "seed group cardinality differs": len(grouped_ids) == 53, + "seed group cardinality differs": len(grouped_ids) == 66, }, homes=("/bundle/seed_protocol", "/spine/seed_site_bindings"), consumers=("compiler_ir.seed_stream_map",), @@ -1886,7 +1902,7 @@ def add( }, expected={ "groups": len(EXPECTED_SEED_GROUPS), - "sites": 53, + "sites": 66, "partition": "disjoint and exhaustive", }, ) diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/imputation.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/imputation.schema.json index cd9031b680..f76e296e62 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/imputation.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/imputation.schema.json @@ -339,6 +339,22 @@ "immigration_status_model_target": { "type": "string" }, + "immigration_required_predictors": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "uniqueItems": true + }, + "immigration_baseline_excluded_statuses": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "uniqueItems": true + }, "immigration_status_targets": { "type": "array", "items": { @@ -397,6 +413,62 @@ ] } }, + "required": [ + "activation", + "contract" + ] + }, + "humanitarian_immigration": { + "type": "object", + "additionalProperties": false, + "properties": { + "activation": { + "type": "object", + "additionalProperties": false, + "properties": { + "all_targets": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "all_targets" + ] + }, + "contract": { + "type": "object", + "additionalProperties": false, + "properties": { + "candidate_shortfall": { + "type": "string" + }, + "mutable_rows": { + "type": "string" + }, + "selection": { + "type": "string" + }, + "target_basis": { + "type": "string" + }, + "targets": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "candidate_shortfall", + "mutable_rows", + "selection", + "target_basis", + "targets" + ] + } + }, "required": [ "activation", "contract" @@ -634,6 +706,7 @@ }, "required": [ "adult_care", + "humanitarian_immigration", "passive_partnership_s_corp", "schedule_d_capital_gain_distributions", "self_employment_components" @@ -886,6 +959,8 @@ "donor_combined_components", "group_optional_names", "housing", + "immigration_baseline_excluded_statuses", + "immigration_required_predictors", "immigration_status_model_target", "immigration_status_targets", "post_transfer_features", diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json index c2035a2b5d..ccec9a9496 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json @@ -1739,9 +1739,270 @@ "source", "target" ] + }, + "humanitarian_status_stocks": { + "type": "object", + "additionalProperties": false, + "properties": { + "asylee": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + }, + "deportation_withheld": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + }, + "paroled_one_year": { + "type": "object", + "additionalProperties": false, + "properties": { + "afghanistan": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + }, + "nicaragua": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + }, + "ukraine": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + }, + "venezuela": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + } + }, + "required": [ + "afghanistan", + "nicaragua", + "ukraine", + "venezuela" + ] + }, + "refugee": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + }, + "tps": { + "type": "object", + "additionalProperties": false, + "properties": { + "el_salvador": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + }, + "honduras": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + }, + "nepal": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + }, + "nicaragua": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + }, + "other_designated": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + }, + "venezuela": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + } + }, + "required": [ + "el_salvador", + "honduras", + "nepal", + "nicaragua", + "other_designated", + "venezuela" + ] + } + }, + "required": [ + "asylee", + "deportation_withheld", + "paroled_one_year", + "refugee", + "tps" + ] } }, "required": [ + "humanitarian_status_stocks", "kind", "seed_from_build_config", "time_period_from_build_config", diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/seeds.py b/packages/microcosm-build/src/microcosm/build/spec_engine/seeds.py index cae6e3b72f..c302f5d9dc 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/seeds.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/seeds.py @@ -745,6 +745,31 @@ def _stable_site( ), candidate_universe="all_person_rows_before_abawd_eligibility_mask", ), + *( + _stable_site( + f"immigration_humanitarian_{label.replace(':', '_')}_assignment", + salt=f"immigration:{label}", + key_grammar=("source_year:source_person_id_if_present", "else_person_id"), + candidate_universe=( + "all_person_rows_then_category_origin_window_pool_mask" + ), + ) + for label in ( + "paroled_one_year:afghanistan", + "paroled_one_year:ukraine", + "paroled_one_year:nicaragua", + "paroled_one_year:venezuela", + "refugee", + "asylee", + "deportation_withheld", + "tps:venezuela", + "tps:el_salvador", + "tps:honduras", + "tps:nicaragua", + "tps:nepal", + "tps:other_designated", + ) + ), _stable_site( "immigration_ead_workers_assignment", salt="immigration:ead_workers", diff --git a/packages/microcosm-build/src/microcosm/build/us/country_package.json b/packages/microcosm-build/src/microcosm/build/us/country_package.json index 5e14125c81..33f01820c2 100644 --- a/packages/microcosm-build/src/microcosm/build/us/country_package.json +++ b/packages/microcosm-build/src/microcosm/build/us/country_package.json @@ -92,6 +92,11 @@ "kind": "legacy_json", "schema_id": "legacy_json" }, + { + "path": "legacy_schema16_late_producer_schedule.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, { "path": "obbba_reforms.json", "kind": "legacy_json", diff --git a/packages/microcosm-build/src/microcosm/build/us/legacy_schema16_late_producer_schedule.json b/packages/microcosm-build/src/microcosm/build/us/legacy_schema16_late_producer_schedule.json new file mode 100644 index 0000000000..8b1ee6c78c --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us/legacy_schema16_late_producer_schedule.json @@ -0,0 +1 @@ +{"acs_earnings_universe_input_inventory":{"operator":"acs_pums_earnings_universe","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"@acs_pums_earnings_universe_execution_config","entity":"person","value_kind":"non_null"}]],"label":"execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"mapped_earnings:employment_income_before_lsr","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"mapped_earnings:self_employment_income_before_lsr","optional":true,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"non_null"}]],"label":"person_tax_unit_link","optional":false,"required_scope":null},{"alternatives":[[{"column":"SEMP","entity":"person","value_kind":"column_present"}]],"label":"raw_source:SEMP","optional":false,"required_scope":null},{"alternatives":[[{"column":"WAGP","entity":"person","value_kind":"column_present"}]],"label":"raw_source:WAGP","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"non_null"}],[{"column":"person_source_id","entity":"person","value_kind":"non_null"}]],"label":"stable_person_lineage","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"support_clone_index","optional":false,"required_scope":null}]},"edges":[["acs_pums_earnings_universe","primary_puf_qrf"],["primary_puf_qrf","source:impute_us_housing_assistance_to_puf_support"],["primary_puf_qrf","source:with_us_adult_care_inputs"],["primary_puf_qrf","source:with_us_child_support_inputs"],["primary_puf_qrf","source:with_us_childcare_inputs"],["primary_puf_qrf","source:with_us_disability_benefits"],["primary_puf_qrf","source:with_us_education_inputs"],["primary_puf_qrf","source:with_us_energy_subsidy_input"],["primary_puf_qrf","source:with_us_immigration_inputs"],["primary_puf_qrf","source:with_us_medicare_take_up_input"],["primary_puf_qrf","source:with_us_pregnancy_inputs"],["primary_puf_qrf","source:with_us_prior_year_income_inputs"],["primary_puf_qrf","source:with_us_retirement_contribution_inputs"],["primary_puf_qrf","source:with_us_retirement_distribution_inputs"],["primary_puf_qrf","source:with_us_weeks_unemployed"],["primary_puf_qrf","source:with_us_wic_claim_input"],["primary_puf_qrf","source:with_us_workers_compensation"],["primary_puf_qrf","transfer:person/adult_care"],["primary_puf_qrf","transfer:person/model_required_boolean"],["primary_puf_qrf","transfer:person/puf_tax_itemization__batch_1"],["primary_puf_qrf","transfer:person/puf_tax_itemization__batch_2"],["primary_puf_qrf","transfer:person/puf_tax_itemization__batch_3"],["primary_puf_qrf","transfer:person/puf_tax_itemization__batch_4"],["primary_puf_qrf","transfer:person/puf_tax_itemization__batch_5"],["primary_puf_qrf","transfer:person/source_operator_child_support"],["primary_puf_qrf","transfer:person/source_operator_disability_benefits"],["primary_puf_qrf","transfer:person/source_operator_education_inputs"],["primary_puf_qrf","transfer:person/source_operator_immigration"],["primary_puf_qrf","transfer:person/source_operator_medicare_take_up"],["primary_puf_qrf","transfer:person/source_operator_retirement_contributions"],["primary_puf_qrf","transfer:person/source_operator_retirement_distributions"],["primary_puf_qrf","transfer:person/source_operator_weeks_unemployed"],["primary_puf_qrf","transfer:person/source_operator_wic_claim"],["primary_puf_qrf","transfer:person/source_operator_workers_compensation"],["primary_puf_qrf","transfer:spm_unit/source_operator_energy_subsidy"],["primary_puf_qrf","transfer:tax_unit/puf_tax_itemization"],["source:impute_us_housing_assistance_to_puf_support","source_finalizer"],["source:with_us_adult_care_inputs","source_finalizer"],["source:with_us_adult_care_inputs","transfer:person/adult_care"],["source:with_us_child_support_inputs","source_finalizer"],["source:with_us_child_support_inputs","transfer:person/source_operator_child_support"],["source:with_us_childcare_inputs","source:with_us_adult_care_inputs"],["source:with_us_childcare_inputs","source_finalizer"],["source:with_us_disability_benefits","source_finalizer"],["source:with_us_disability_benefits","transfer:person/source_operator_disability_benefits"],["source:with_us_education_inputs","source_finalizer"],["source:with_us_education_inputs","transfer:person/source_operator_education_inputs"],["source:with_us_energy_subsidy_input","source_finalizer"],["source:with_us_energy_subsidy_input","transfer:spm_unit/source_operator_energy_subsidy"],["source:with_us_immigration_inputs","source_finalizer"],["source:with_us_immigration_inputs","transfer:person/source_operator_immigration"],["source:with_us_medicare_take_up_input","source_finalizer"],["source:with_us_medicare_take_up_input","transfer:person/source_operator_medicare_take_up"],["source:with_us_pregnancy_inputs","source:with_us_wic_claim_input"],["source:with_us_pregnancy_inputs","source_finalizer"],["source:with_us_pregnancy_inputs","transfer:person/model_required_boolean"],["source:with_us_prior_year_income_inputs","source_finalizer"],["source:with_us_retirement_contribution_inputs","source_finalizer"],["source:with_us_retirement_contribution_inputs","transfer:person/puf_tax_itemization__batch_2"],["source:with_us_retirement_contribution_inputs","transfer:person/puf_tax_itemization__batch_3"],["source:with_us_retirement_contribution_inputs","transfer:person/source_operator_retirement_contributions"],["source:with_us_retirement_distribution_inputs","source_finalizer"],["source:with_us_retirement_distribution_inputs","transfer:person/source_operator_retirement_distributions"],["source:with_us_weeks_unemployed","source_finalizer"],["source:with_us_weeks_unemployed","transfer:person/source_operator_weeks_unemployed"],["source:with_us_wic_claim_input","source_finalizer"],["source:with_us_wic_claim_input","transfer:person/source_operator_wic_claim"],["source:with_us_workers_compensation","source_finalizer"],["source:with_us_workers_compensation","transfer:person/source_operator_workers_compensation"],["transfer:person/puf_tax_itemization__batch_2","source:with_us_education_inputs"],["transfer:person/puf_tax_itemization__batch_5","source:with_us_adult_care_inputs"]],"execution_receipt_contract":{"row_binding":"declared_globally_reconciled_input_and_scope_exact_output_source_and_primary_callback_resource_receipt_and_previous_execution_sha256","top_binding":"entry_and_output_frame_sha256_execution_chain_source_completion_and_nineteen_transfer_groups","transition_authority":{"authority_id":"us_stacked_late_producer_transition","independent_digest_required":true,"metadata_key":"us_late_producer_transition_authority","version":1},"version":3,"virtual_resource_binding":"exact_kind_specific_semantic_payload_and_sha256"},"external_stages":["post_clone_input_surface"],"order":["acs_pums_earnings_universe","primary_puf_qrf","source:impute_us_housing_assistance_to_puf_support","source:with_us_child_support_inputs","source:with_us_childcare_inputs","source:with_us_disability_benefits","source:with_us_energy_subsidy_input","source:with_us_immigration_inputs","source:with_us_medicare_take_up_input","source:with_us_pregnancy_inputs","source:with_us_prior_year_income_inputs","source:with_us_retirement_contribution_inputs","source:with_us_retirement_distribution_inputs","source:with_us_weeks_unemployed","source:with_us_workers_compensation","transfer:person/puf_tax_itemization__batch_1","transfer:person/puf_tax_itemization__batch_4","transfer:person/puf_tax_itemization__batch_5","transfer:tax_unit/puf_tax_itemization","source:with_us_adult_care_inputs","source:with_us_wic_claim_input","transfer:person/model_required_boolean","transfer:person/puf_tax_itemization__batch_2","transfer:person/puf_tax_itemization__batch_3","transfer:person/source_operator_child_support","transfer:person/source_operator_disability_benefits","transfer:person/source_operator_immigration","transfer:person/source_operator_medicare_take_up","transfer:person/source_operator_retirement_contributions","transfer:person/source_operator_retirement_distributions","transfer:person/source_operator_weeks_unemployed","transfer:person/source_operator_workers_compensation","transfer:spm_unit/source_operator_energy_subsidy","source:with_us_education_inputs","transfer:person/adult_care","transfer:person/source_operator_wic_claim","source_finalizer","transfer:person/source_operator_education_inputs"],"overlap_ownership":{"artifact_kind":"microcosm_us_late_overlap_ownership","doctrine":{"clone_2_policy":"inherit_or_mirror_clone_1_final_owner_bytes","non_owner_write_policy":"masked_or_verified_byte_exact_noop","owner_cardinality":"exactly_one_per_target_origin_clone_role","retirement_legacy_order":"primary_then_postclone_source_owner_then_late_transfer","tail_preservation_guard":"unchanged_fail_closed"},"ownership":[{"clone_index":0,"entity":"person","final_owner":"transfer:person/puf_tax_itemization__batch_2","finalization":"late_transfer_owner_last","origin":"asec","producer_actions":[{"action":"scope_masked_noop","owns_final":false,"producer":"primary_puf_qrf"},{"action":"consume_only_byte_exact_noop","owns_final":false,"producer":"source:with_us_education_inputs"},{"action":"final_write","owns_final":true,"producer":"transfer:person/puf_tax_itemization__batch_2"}],"target":"qualified_tuition_expenses"},{"clone_index":1,"entity":"person","final_owner":"primary_puf_qrf","finalization":"primary_write","origin":"asec","producer_actions":[{"action":"primary_write","owns_final":true,"producer":"primary_puf_qrf"},{"action":"consume_only_byte_exact_noop","owns_final":false,"producer":"source:with_us_education_inputs"},{"action":"producer_masked_byte_exact_noop","owns_final":false,"producer":"transfer:person/puf_tax_itemization__batch_2"}],"target":"qualified_tuition_expenses"},{"clone_index":2,"entity":"person","final_owner":"primary_puf_qrf","finalization":"byte_exact_clone_1_inheritance","origin":"asec","producer_actions":[{"action":"byte_exact_clone_1_inheritance","owns_final":true,"producer":"primary_puf_qrf"},{"action":"consume_only_byte_exact_noop","owns_final":false,"producer":"source:with_us_education_inputs"},{"action":"producer_masked_byte_exact_noop","owns_final":false,"producer":"transfer:person/puf_tax_itemization__batch_2"}],"target":"qualified_tuition_expenses"},{"clone_index":0,"entity":"person","final_owner":"transfer:person/puf_tax_itemization__batch_2","finalization":"late_transfer_owner_last","origin":"acs","producer_actions":[{"action":"scope_masked_noop","owns_final":false,"producer":"primary_puf_qrf"},{"action":"origin_projection_masked_noop","owns_final":false,"producer":"source:with_us_education_inputs"},{"action":"final_write","owns_final":true,"producer":"transfer:person/puf_tax_itemization__batch_2"}],"target":"qualified_tuition_expenses"},{"clone_index":1,"entity":"person","final_owner":"primary_puf_qrf","finalization":"primary_write","origin":"acs","producer_actions":[{"action":"primary_write","owns_final":true,"producer":"primary_puf_qrf"},{"action":"origin_projection_masked_noop","owns_final":false,"producer":"source:with_us_education_inputs"},{"action":"producer_masked_byte_exact_noop","owns_final":false,"producer":"transfer:person/puf_tax_itemization__batch_2"}],"target":"qualified_tuition_expenses"},{"clone_index":2,"entity":"person","final_owner":"primary_puf_qrf","finalization":"byte_exact_clone_1_inheritance","origin":"acs","producer_actions":[{"action":"byte_exact_clone_1_inheritance","owns_final":true,"producer":"primary_puf_qrf"},{"action":"origin_projection_masked_noop","owns_final":false,"producer":"source:with_us_education_inputs"},{"action":"producer_masked_byte_exact_noop","owns_final":false,"producer":"transfer:person/puf_tax_itemization__batch_2"}],"target":"qualified_tuition_expenses"},{"clone_index":0,"entity":"person","final_owner":"source:with_us_retirement_contribution_inputs","finalization":"source_direct_split","origin":"asec","producer_actions":[{"action":"scope_masked_noop","owns_final":false,"producer":"primary_puf_qrf"},{"action":"final_write","owns_final":true,"producer":"source:with_us_retirement_contribution_inputs"},{"action":"producer_masked_byte_exact_noop","owns_final":false,"producer":"transfer:person/puf_tax_itemization__batch_2"}],"target":"traditional_ira_contributions_desired"},{"clone_index":1,"entity":"person","final_owner":"source:with_us_retirement_contribution_inputs","finalization":"source_owner_last_overwrite","origin":"asec","producer_actions":[{"action":"interim_write_overwritten_by_owner_last","owns_final":false,"producer":"primary_puf_qrf"},{"action":"final_write","owns_final":true,"producer":"source:with_us_retirement_contribution_inputs"},{"action":"producer_masked_byte_exact_noop","owns_final":false,"producer":"transfer:person/puf_tax_itemization__batch_2"}],"target":"traditional_ira_contributions_desired"},{"clone_index":2,"entity":"person","final_owner":"source:with_us_retirement_contribution_inputs","finalization":"byte_exact_clone_1_mirror","origin":"asec","producer_actions":[{"action":"interim_clone_1_inheritance_overwritten_by_owner_last","owns_final":false,"producer":"primary_puf_qrf"},{"action":"byte_exact_clone_1_mirror","owns_final":true,"producer":"source:with_us_retirement_contribution_inputs"},{"action":"producer_masked_byte_exact_noop","owns_final":false,"producer":"transfer:person/puf_tax_itemization__batch_2"}],"target":"traditional_ira_contributions_desired"},{"clone_index":0,"entity":"person","final_owner":"transfer:person/puf_tax_itemization__batch_2","finalization":"late_transfer_owner_last","origin":"acs","producer_actions":[{"action":"scope_masked_noop","owns_final":false,"producer":"primary_puf_qrf"},{"action":"origin_projection_masked_noop","owns_final":false,"producer":"source:with_us_retirement_contribution_inputs"},{"action":"final_write","owns_final":true,"producer":"transfer:person/puf_tax_itemization__batch_2"}],"target":"traditional_ira_contributions_desired"},{"clone_index":1,"entity":"person","final_owner":"primary_puf_qrf","finalization":"primary_write","origin":"acs","producer_actions":[{"action":"primary_write","owns_final":true,"producer":"primary_puf_qrf"},{"action":"origin_projection_masked_noop","owns_final":false,"producer":"source:with_us_retirement_contribution_inputs"},{"action":"producer_masked_byte_exact_noop","owns_final":false,"producer":"transfer:person/puf_tax_itemization__batch_2"}],"target":"traditional_ira_contributions_desired"},{"clone_index":2,"entity":"person","final_owner":"primary_puf_qrf","finalization":"byte_exact_clone_1_inheritance","origin":"acs","producer_actions":[{"action":"byte_exact_clone_1_inheritance","owns_final":true,"producer":"primary_puf_qrf"},{"action":"origin_projection_masked_noop","owns_final":false,"producer":"source:with_us_retirement_contribution_inputs"},{"action":"producer_masked_byte_exact_noop","owns_final":false,"producer":"transfer:person/puf_tax_itemization__batch_2"}],"target":"traditional_ira_contributions_desired"},{"clone_index":0,"entity":"person","final_owner":"source:with_us_retirement_contribution_inputs","finalization":"source_direct_split","origin":"asec","producer_actions":[{"action":"scope_masked_noop","owns_final":false,"producer":"primary_puf_qrf"},{"action":"final_write","owns_final":true,"producer":"source:with_us_retirement_contribution_inputs"},{"action":"producer_masked_byte_exact_noop","owns_final":false,"producer":"transfer:person/puf_tax_itemization__batch_3"}],"target":"self_employed_pension_contributions_desired"},{"clone_index":1,"entity":"person","final_owner":"source:with_us_retirement_contribution_inputs","finalization":"source_owner_last_overwrite","origin":"asec","producer_actions":[{"action":"interim_write_overwritten_by_owner_last","owns_final":false,"producer":"primary_puf_qrf"},{"action":"final_write","owns_final":true,"producer":"source:with_us_retirement_contribution_inputs"},{"action":"producer_masked_byte_exact_noop","owns_final":false,"producer":"transfer:person/puf_tax_itemization__batch_3"}],"target":"self_employed_pension_contributions_desired"},{"clone_index":2,"entity":"person","final_owner":"source:with_us_retirement_contribution_inputs","finalization":"byte_exact_clone_1_mirror","origin":"asec","producer_actions":[{"action":"interim_clone_1_inheritance_overwritten_by_owner_last","owns_final":false,"producer":"primary_puf_qrf"},{"action":"byte_exact_clone_1_mirror","owns_final":true,"producer":"source:with_us_retirement_contribution_inputs"},{"action":"producer_masked_byte_exact_noop","owns_final":false,"producer":"transfer:person/puf_tax_itemization__batch_3"}],"target":"self_employed_pension_contributions_desired"},{"clone_index":0,"entity":"person","final_owner":"transfer:person/puf_tax_itemization__batch_3","finalization":"late_transfer_owner_last","origin":"acs","producer_actions":[{"action":"scope_masked_noop","owns_final":false,"producer":"primary_puf_qrf"},{"action":"origin_projection_masked_noop","owns_final":false,"producer":"source:with_us_retirement_contribution_inputs"},{"action":"final_write","owns_final":true,"producer":"transfer:person/puf_tax_itemization__batch_3"}],"target":"self_employed_pension_contributions_desired"},{"clone_index":1,"entity":"person","final_owner":"primary_puf_qrf","finalization":"primary_write","origin":"acs","producer_actions":[{"action":"primary_write","owns_final":true,"producer":"primary_puf_qrf"},{"action":"origin_projection_masked_noop","owns_final":false,"producer":"source:with_us_retirement_contribution_inputs"},{"action":"producer_masked_byte_exact_noop","owns_final":false,"producer":"transfer:person/puf_tax_itemization__batch_3"}],"target":"self_employed_pension_contributions_desired"},{"clone_index":2,"entity":"person","final_owner":"primary_puf_qrf","finalization":"byte_exact_clone_1_inheritance","origin":"acs","producer_actions":[{"action":"byte_exact_clone_1_inheritance","owns_final":true,"producer":"primary_puf_qrf"},{"action":"origin_projection_masked_noop","owns_final":false,"producer":"source:with_us_retirement_contribution_inputs"},{"action":"producer_masked_byte_exact_noop","owns_final":false,"producer":"transfer:person/puf_tax_itemization__batch_3"}],"target":"self_employed_pension_contributions_desired"}],"schema_version":1,"sha256":"5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356","targets":[{"entity":"person","source_producer":"source:with_us_education_inputs","source_touch":"consume_only_byte_exact_noop","target":"qualified_tuition_expenses","transfer_producer":"transfer:person/puf_tax_itemization__batch_2"},{"entity":"person","source_producer":"source:with_us_retirement_contribution_inputs","source_touch":"persisted_owner_last_write","target":"traditional_ira_contributions_desired","transfer_producer":"transfer:person/puf_tax_itemization__batch_2"},{"entity":"person","source_producer":"source:with_us_retirement_contribution_inputs","source_touch":"persisted_owner_last_write","target":"self_employed_pension_contributions_desired","transfer_producer":"transfer:person/puf_tax_itemization__batch_3"}]},"payload_sha256":"02e618cc656eb39990ed99dca2b30a52794e01e2b06a3c2df87ca4a7d85ab086","primary_puf_input_inventory":{"operator":"primary_puf_qrf","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"dividend_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"dividend_income","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"dividend_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"filing_status_input","entity":"tax_unit","value_kind":"non_null"}]],"label":"filing_status","optional":false,"required_scope":null},{"alternatives":[[{"column":"long_term_capital_gains","entity":"person","value_kind":"finite_numeric"}],[{"column":"long_term_capital_gains_before_response","entity":"person","value_kind":"finite_numeric"}],[{"column":"long_term_capital_gains","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"long_term_capital_gains","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"non_null"}]],"label":"person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"alimony_expense","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:alimony_expense","optional":true,"required_scope":null},{"alternatives":[[{"column":"alimony_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:alimony_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"business_is_sstb","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:business_is_sstb","optional":true,"required_scope":null},{"alternatives":[[{"column":"casualty_loss","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:casualty_loss","optional":true,"required_scope":null},{"alternatives":[[{"column":"charitable_cash_donations","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:charitable_cash_donations","optional":true,"required_scope":null},{"alternatives":[[{"column":"charitable_non_cash_donations","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:charitable_non_cash_donations","optional":true,"required_scope":null},{"alternatives":[[{"column":"educator_expense","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:educator_expense","optional":true,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:employment_income_before_lsr","optional":true,"required_scope":null},{"alternatives":[[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:estate_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"estate_income_would_be_qualified","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:estate_income_would_be_qualified","optional":true,"required_scope":null},{"alternatives":[[{"column":"farm_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:farm_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"farm_operations_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:farm_operations_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"farm_operations_income_would_be_qualified","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:farm_operations_income_would_be_qualified","optional":true,"required_scope":null},{"alternatives":[[{"column":"farm_rent_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:farm_rent_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"farm_rent_income_would_be_qualified","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:farm_rent_income_would_be_qualified","optional":true,"required_scope":null},{"alternatives":[[{"column":"home_mortgage_interest","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:home_mortgage_interest","optional":true,"required_scope":null},{"alternatives":[[{"column":"investment_income_elected_form_4952","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:investment_income_elected_form_4952","optional":true,"required_scope":null},{"alternatives":[[{"column":"investment_interest_expense","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:investment_interest_expense","optional":true,"required_scope":null},{"alternatives":[[{"column":"long_term_capital_gains_before_response","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:long_term_capital_gains_before_response","optional":true,"required_scope":null},{"alternatives":[[{"column":"long_term_capital_gains_on_collectibles","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:long_term_capital_gains_on_collectibles","optional":true,"required_scope":null},{"alternatives":[[{"column":"miscellaneous_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:miscellaneous_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:non_qualified_dividend_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"non_sch_d_capital_gains","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:non_sch_d_capital_gains","optional":true,"required_scope":null},{"alternatives":[[{"column":"partnership_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:partnership_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"partnership_s_corp_income_would_be_qualified","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:partnership_s_corp_income_would_be_qualified","optional":true,"required_scope":null},{"alternatives":[[{"column":"partnership_self_employment_net_earnings","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:partnership_self_employment_net_earnings","optional":true,"required_scope":null},{"alternatives":[[{"column":"qualified_bdc_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:qualified_bdc_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:qualified_dividend_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"qualified_reit_and_ptp_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:qualified_reit_and_ptp_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"qualified_tuition_expenses","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:qualified_tuition_expenses","optional":true,"required_scope":null},{"alternatives":[[{"column":"real_estate_taxes","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:real_estate_taxes","optional":true,"required_scope":null},{"alternatives":[[{"column":"rental_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:rental_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"rental_income_would_be_qualified","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:rental_income_would_be_qualified","optional":true,"required_scope":null},{"alternatives":[[{"column":"s_corp_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:s_corp_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"salt_refund_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:salt_refund_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employed_pension_contributions_desired","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:self_employed_pension_contributions_desired","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:self_employment_income_before_lsr","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_would_be_qualified","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:self_employment_income_would_be_qualified","optional":true,"required_scope":null},{"alternatives":[[{"column":"short_term_capital_gains","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:short_term_capital_gains","optional":true,"required_scope":null},{"alternatives":[[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:social_security_dependents","optional":true,"required_scope":null},{"alternatives":[[{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:social_security_disability","optional":true,"required_scope":null},{"alternatives":[[{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:social_security_retirement","optional":true,"required_scope":null},{"alternatives":[[{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:social_security_survivors","optional":true,"required_scope":null},{"alternatives":[[{"column":"sstb_self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:sstb_self_employment_income_before_lsr","optional":true,"required_scope":null},{"alternatives":[[{"column":"sstb_self_employment_income_would_be_qualified","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:sstb_self_employment_income_would_be_qualified","optional":true,"required_scope":null},{"alternatives":[[{"column":"sstb_unadjusted_basis_qualified_property","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:sstb_unadjusted_basis_qualified_property","optional":true,"required_scope":null},{"alternatives":[[{"column":"sstb_w2_wages_from_qualified_business","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:sstb_w2_wages_from_qualified_business","optional":true,"required_scope":null},{"alternatives":[[{"column":"student_loan_interest","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:student_loan_interest","optional":true,"required_scope":null},{"alternatives":[[{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:tax_exempt_interest_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:taxable_interest_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:taxable_ira_distributions","optional":true,"required_scope":null},{"alternatives":[[{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:taxable_private_pension_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"traditional_ira_contributions_desired","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:traditional_ira_contributions_desired","optional":true,"required_scope":null},{"alternatives":[[{"column":"unadjusted_basis_qualified_property","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:unadjusted_basis_qualified_property","optional":true,"required_scope":null},{"alternatives":[[{"column":"unreimbursed_business_employee_expenses","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:unreimbursed_business_employee_expenses","optional":true,"required_scope":null},{"alternatives":[[{"column":"w2_wages_from_qualified_business","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:w2_wages_from_qualified_business","optional":true,"required_scope":null},{"alternatives":[[{"column":"@primary_puf_execution_config","entity":"tax_unit","value_kind":"non_null"}]],"label":"primary_puf_execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@primary_qrf_checkpoint","entity":"tax_unit","value_kind":"non_null"}]],"label":"primary_qrf_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"@puf_donor_tax_units","entity":"tax_unit","value_kind":"non_null"}]],"label":"puf_donor","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_full_time_college_student","entity":"person","value_kind":"finite_numeric"}]],"label":"qualified_tuition_allocation_fallback","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"tax_unit","value_kind":"non_null"}]],"label":"resolved_tax_unit_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"self_employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"short_term_capital_gains","entity":"person","value_kind":"finite_numeric"}],[{"column":"short_term_capital_gains","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"short_term_capital_gains","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"non_null"}]],"label":"support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"non_null"}]],"label":"tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"domestic_production_ald","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_output_passthrough:domestic_production_ald","optional":true,"required_scope":null},{"alternatives":[[{"column":"first_home_mortgage_balance","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_output_passthrough:first_home_mortgage_balance","optional":true,"required_scope":null},{"alternatives":[[{"column":"first_home_mortgage_interest","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_output_passthrough:first_home_mortgage_interest","optional":true,"required_scope":null},{"alternatives":[[{"column":"first_home_mortgage_origination_year","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_output_passthrough:first_home_mortgage_origination_year","optional":true,"required_scope":null},{"alternatives":[[{"column":"health_savings_account_ald","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_output_passthrough:health_savings_account_ald","optional":true,"required_scope":null},{"alternatives":[[{"column":"second_home_mortgage_balance","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_output_passthrough:second_home_mortgage_balance","optional":true,"required_scope":null},{"alternatives":[[{"column":"second_home_mortgage_interest","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_output_passthrough:second_home_mortgage_interest","optional":true,"required_scope":null},{"alternatives":[[{"column":"second_home_mortgage_origination_year","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_output_passthrough:second_home_mortgage_origination_year","optional":true,"required_scope":null},{"alternatives":[[{"column":"unrecaptured_section_1250_gain","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_output_passthrough:unrecaptured_section_1250_gain","optional":true,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"non_null"},{"column":"tax_unit_id","entity":"tax_unit","value_kind":"non_null"}]],"label":"tax_unit_person_count","optional":false,"required_scope":null},{"alternatives":[[{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"taxable_interest_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},"producer_count":38,"schedule_sha256":"e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f","schema_version":16,"source_input_inventories":[{"operator":"impute_us_housing_assistance_to_puf_support","requirements":[{"alternatives":[[{"column":"A_AGE","entity":"person","value_kind":"finite_numeric"}],[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"has_esi","entity":"person","value_kind":"finite_numeric"}]],"label":"employer_health_coverage","optional":false,"required_scope":null},{"alternatives":[[{"column":"WSAL_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"filing_status","entity":"tax_unit","value_kind":"non_null"}],[{"column":"filing_status_input","entity":"tax_unit","value_kind":"non_null"}]],"label":"filing_status","optional":false,"required_scope":null},{"alternatives":[[{"column":"receives_housing_assistance","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"housing_assistance_receipt","optional":false,"required_scope":null},{"alternatives":[[{"column":"takes_up_housing_assistance_if_eligible","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"housing_assistance_takeup","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"non_null"}]],"label":"person_spm_unit_link","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_tax_unit_link","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"SEMP_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"self_employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"A_SEX","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_male","entity":"person","value_kind":"finite_numeric"}]],"label":"sex","optional":false,"required_scope":null},{"alternatives":[[{"column":"SS_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"social_security_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"source_wrapper:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@post_clone_source_execution_config","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"source_wrapper:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"source_wrapper:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"source_wrapper:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:source_evidence","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"spm_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"non_null"}]],"label":"spm_support_clone_index","optional":true,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"non_null"}]],"label":"spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"support_clone_index","optional":true,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_role_input","entity":"person","value_kind":"non_null"}]],"label":"tax_unit_role","optional":false,"required_scope":null}]},{"operator":"with_us_adult_care_inputs","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_pre_subsidy_childcare_expenses","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"childcare_expenses","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_spm_unit_link","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_tax_unit_link","optional":false,"required_scope":null},{"alternatives":[[{"column":"PEDISDRS","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:PEDISDRS","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_full_time_college_student","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:is_full_time_college_student","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"spm_unit","value_kind":"non_null"}]],"label":"resolved_spm_unit_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"tax_unit","value_kind":"non_null"}]],"label":"resolved_tax_unit_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"self_employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"source_wrapper:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@post_clone_source_execution_config","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"source_wrapper:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"source_wrapper:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"source_wrapper:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:source_evidence","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"sstb_self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"sstb_earned_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"},{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"support_role","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_role_input","entity":"person","value_kind":"non_null"}]],"label":"tax_unit_role","optional":false,"required_scope":null}]},{"operator":"with_us_child_support_inputs","requirements":[{"alternatives":[[{"column":"A_AGE","entity":"person","value_kind":"finite_numeric"}],[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"has_esi","entity":"person","value_kind":"finite_numeric"}]],"label":"employer_health_coverage","optional":false,"required_scope":null},{"alternatives":[[{"column":"WSAL_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"filing_status","entity":"tax_unit","value_kind":"non_null"}],[{"column":"filing_status_input","entity":"tax_unit","value_kind":"non_null"}]],"label":"filing_status","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_tax_unit_link","optional":false,"required_scope":null},{"alternatives":[[{"column":"CHSP_VAL","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:CHSP_VAL","optional":false,"required_scope":null},{"alternatives":[[{"column":"CSP_VAL","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:CSP_VAL","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"SEMP_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"self_employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"A_SEX","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_male","entity":"person","value_kind":"finite_numeric"}]],"label":"sex","optional":false,"required_scope":null},{"alternatives":[[{"column":"SS_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"social_security_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"source_wrapper:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@post_clone_source_execution_config","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"source_wrapper:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"source_wrapper:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"source_wrapper:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:source_evidence","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"support_clone_index","optional":true,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_role_input","entity":"person","value_kind":"non_null"}]],"label":"tax_unit_role","optional":false,"required_scope":null}]},{"operator":"with_us_childcare_inputs","requirements":[{"alternatives":[[{"column":"A_AGE","entity":"person","value_kind":"finite_numeric"}],[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"has_esi","entity":"person","value_kind":"finite_numeric"}]],"label":"employer_health_coverage","optional":false,"required_scope":null},{"alternatives":[[{"column":"WSAL_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"filing_status","entity":"tax_unit","value_kind":"non_null"}],[{"column":"filing_status_input","entity":"tax_unit","value_kind":"non_null"}]],"label":"filing_status","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_tax_unit_link","optional":false,"required_scope":null},{"alternatives":[[{"column":"SPM_CHILDCAREXPNS","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:SPM_CHILDCAREXPNS","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"SEMP_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"self_employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"A_SEX","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_male","entity":"person","value_kind":"finite_numeric"}]],"label":"sex","optional":false,"required_scope":null},{"alternatives":[[{"column":"SS_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"social_security_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"source_wrapper:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@post_clone_source_execution_config","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"source_wrapper:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"source_wrapper:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"source_wrapper:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:source_evidence","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"non_null"}]],"label":"spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"support_clone_index","optional":true,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_role_input","entity":"person","value_kind":"non_null"}]],"label":"tax_unit_role","optional":false,"required_scope":null}]},{"operator":"with_us_disability_benefits","requirements":[{"alternatives":[[{"column":"A_AGE","entity":"person","value_kind":"finite_numeric"}],[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"has_esi","entity":"person","value_kind":"finite_numeric"}]],"label":"employer_health_coverage","optional":false,"required_scope":null},{"alternatives":[[{"column":"WSAL_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"filing_status","entity":"tax_unit","value_kind":"non_null"}],[{"column":"filing_status_input","entity":"tax_unit","value_kind":"non_null"}]],"label":"filing_status","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_tax_unit_link","optional":false,"required_scope":null},{"alternatives":[[{"column":"DIS_SC1","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:DIS_SC1","optional":false,"required_scope":null},{"alternatives":[[{"column":"DIS_SC2","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:DIS_SC2","optional":false,"required_scope":null},{"alternatives":[[{"column":"DIS_VAL1","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:DIS_VAL1","optional":false,"required_scope":null},{"alternatives":[[{"column":"DIS_VAL2","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:DIS_VAL2","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"SEMP_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"self_employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"A_SEX","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_male","entity":"person","value_kind":"finite_numeric"}]],"label":"sex","optional":false,"required_scope":null},{"alternatives":[[{"column":"SS_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"social_security_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"source_wrapper:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@post_clone_source_execution_config","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"source_wrapper:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"source_wrapper:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"source_wrapper:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:source_evidence","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"support_clone_index","optional":true,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_role_input","entity":"person","value_kind":"non_null"}]],"label":"tax_unit_role","optional":false,"required_scope":null}]},{"operator":"with_us_education_inputs","requirements":[{"alternatives":[[{"column":"ED_VAL","entity":"person","value_kind":"finite_numeric"}]],"label":"education_source","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"non_null"}]],"label":"person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"qualified_tuition_expenses","entity":"person","value_kind":"finite_numeric"}]],"label":"qualified_tuition","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"source_wrapper:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@post_clone_source_execution_config","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"source_wrapper:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"source_wrapper:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"source_wrapper:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:source_evidence","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:tax_unit_id","optional":false,"required_scope":null}]},{"operator":"with_us_energy_subsidy_input","requirements":[{"alternatives":[[{"column":"A_AGE","entity":"person","value_kind":"finite_numeric"}],[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"has_esi","entity":"person","value_kind":"finite_numeric"}]],"label":"employer_health_coverage","optional":false,"required_scope":null},{"alternatives":[[{"column":"WSAL_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"filing_status","entity":"tax_unit","value_kind":"non_null"}],[{"column":"filing_status_input","entity":"tax_unit","value_kind":"non_null"}]],"label":"filing_status","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_tax_unit_link","optional":false,"required_scope":null},{"alternatives":[[{"column":"SPM_ENGVAL","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:SPM_ENGVAL","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"SEMP_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"self_employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"A_SEX","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_male","entity":"person","value_kind":"finite_numeric"}]],"label":"sex","optional":false,"required_scope":null},{"alternatives":[[{"column":"SS_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"social_security_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"source_wrapper:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@post_clone_source_execution_config","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"source_wrapper:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"source_wrapper:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"source_wrapper:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:source_evidence","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"non_null"}]],"label":"spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"support_clone_index","optional":true,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_role_input","entity":"person","value_kind":"non_null"}]],"label":"tax_unit_role","optional":false,"required_scope":null}]},{"operator":"with_us_immigration_inputs","requirements":[{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"non_null"}]],"label":"person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"A_AGE","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:A_AGE","optional":false,"required_scope":null},{"alternatives":[[{"column":"A_HSCOL","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:A_HSCOL","optional":false,"required_scope":null},{"alternatives":[[{"column":"A_MARITL","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:A_MARITL","optional":false,"required_scope":null},{"alternatives":[[{"column":"A_MJOCC","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:A_MJOCC","optional":false,"required_scope":null},{"alternatives":[[{"column":"A_SPOUSE","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:A_SPOUSE","optional":false,"required_scope":null},{"alternatives":[[{"column":"CAID","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:CAID","optional":false,"required_scope":null},{"alternatives":[[{"column":"CHAMPVA","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:CHAMPVA","optional":false,"required_scope":null},{"alternatives":[[{"column":"IHSFLG","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:IHSFLG","optional":false,"required_scope":null},{"alternatives":[[{"column":"MCARE","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:MCARE","optional":false,"required_scope":null},{"alternatives":[[{"column":"MIL","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:MIL","optional":false,"required_scope":null},{"alternatives":[[{"column":"PEAFEVER","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:PEAFEVER","optional":false,"required_scope":null},{"alternatives":[[{"column":"PEINUSYR","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:PEINUSYR","optional":false,"required_scope":null},{"alternatives":[[{"column":"PEIO1COW","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:PEIO1COW","optional":false,"required_scope":null},{"alternatives":[[{"column":"PENATVTY","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:PENATVTY","optional":false,"required_scope":null},{"alternatives":[[{"column":"PEN_SC1","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:PEN_SC1","optional":false,"required_scope":null},{"alternatives":[[{"column":"PEN_SC2","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:PEN_SC2","optional":false,"required_scope":null},{"alternatives":[[{"column":"PRCITSHP","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:PRCITSHP","optional":false,"required_scope":null},{"alternatives":[[{"column":"RESNSS1","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:RESNSS1","optional":false,"required_scope":null},{"alternatives":[[{"column":"RESNSS2","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:RESNSS2","optional":false,"required_scope":null},{"alternatives":[[{"column":"SEMP_VAL","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:SEMP_VAL","optional":false,"required_scope":null},{"alternatives":[[{"column":"SPM_CAPHOUSESUB","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:SPM_CAPHOUSESUB","optional":false,"required_scope":null},{"alternatives":[[{"column":"SSI_YN","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:SSI_YN","optional":false,"required_scope":null},{"alternatives":[[{"column":"SS_YN","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:SS_YN","optional":false,"required_scope":null},{"alternatives":[[{"column":"WSAL_VAL","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:WSAL_VAL","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"source_wrapper:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@post_clone_source_execution_config","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"source_wrapper:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"source_wrapper:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"source_wrapper:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:source_evidence","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"non_null"}],[{"column":"source_person_id","entity":"person","value_kind":"non_null"},{"column":"source_year","entity":"person","value_kind":"non_null"}]],"label":"stable_source_identity","optional":false,"required_scope":null}]},{"operator":"with_us_medicare_take_up_input","requirements":[{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"non_null"}]],"label":"person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"MCARE","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:MCARE","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"source_wrapper:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@post_clone_source_execution_config","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"source_wrapper:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"source_wrapper:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"source_wrapper:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:source_evidence","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:tax_unit_id","optional":false,"required_scope":null}]},{"operator":"with_us_pregnancy_inputs","requirements":[{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"non_null"}]],"label":"person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"pregnancy_structural_source_person_id","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_AGE","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:A_AGE","optional":false,"required_scope":null},{"alternatives":[[{"column":"A_SEX","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:A_SEX","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"source_wrapper:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@post_clone_source_execution_config","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"source_wrapper:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"source_wrapper:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"source_wrapper:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:source_evidence","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"non_null"}],[{"column":"source_household_id","entity":"person","value_kind":"non_null"},{"column":"source_person_id","entity":"person","value_kind":"non_null"},{"column":"source_year","entity":"person","value_kind":"non_null"}]],"label":"stable_source_identity","optional":false,"required_scope":null}]},{"operator":"with_us_prior_year_income_inputs","requirements":[{"alternatives":[[{"column":"A_AGE","entity":"person","value_kind":"finite_numeric"}],[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"has_esi","entity":"person","value_kind":"finite_numeric"}]],"label":"employer_health_coverage","optional":false,"required_scope":null},{"alternatives":[[{"column":"WSAL_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_last_year","entity":"person","value_kind":"finite_numeric"}]],"label":"employment_income_last_year","optional":false,"required_scope":null},{"alternatives":[[{"column":"filing_status","entity":"tax_unit","value_kind":"non_null"}],[{"column":"filing_status_input","entity":"tax_unit","value_kind":"non_null"}]],"label":"filing_status","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_tax_unit_link","optional":false,"required_scope":null},{"alternatives":[[{"column":"I_ERNVAL","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:I_ERNVAL","optional":false,"required_scope":null},{"alternatives":[[{"column":"I_SEVAL","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:I_SEVAL","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:PERIDNUM","optional":false,"required_scope":null},{"alternatives":[[{"column":"SEMP_VAL","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:SEMP_VAL","optional":false,"required_scope":null},{"alternatives":[[{"column":"WSAL_VAL","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:WSAL_VAL","optional":false,"required_scope":null},{"alternatives":[[{"column":"source_year","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:source_year","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"SEMP_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"self_employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_last_year","entity":"person","value_kind":"finite_numeric"}]],"label":"self_employment_income_last_year","optional":false,"required_scope":null},{"alternatives":[[{"column":"A_SEX","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_male","entity":"person","value_kind":"finite_numeric"}]],"label":"sex","optional":false,"required_scope":null},{"alternatives":[[{"column":"SS_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"social_security_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"source_wrapper:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@post_clone_source_execution_config","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"source_wrapper:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"source_wrapper:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"source_wrapper:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:source_evidence","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"support_clone_index","optional":true,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_role_input","entity":"person","value_kind":"non_null"}]],"label":"tax_unit_role","optional":false,"required_scope":null}]},{"operator":"with_us_retirement_contribution_inputs","requirements":[{"alternatives":[[{"column":"A_AGE","entity":"person","value_kind":"finite_numeric"}],[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"has_esi","entity":"person","value_kind":"finite_numeric"}]],"label":"employer_health_coverage","optional":false,"required_scope":null},{"alternatives":[[{"column":"WSAL_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"filing_status","entity":"tax_unit","value_kind":"non_null"}],[{"column":"filing_status_input","entity":"tax_unit","value_kind":"non_null"}]],"label":"filing_status","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_tax_unit_link","optional":false,"required_scope":null},{"alternatives":[[{"column":"RETCB_VAL","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:RETCB_VAL","optional":false,"required_scope":null},{"alternatives":[[{"column":"SEMP_VAL","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:SEMP_VAL","optional":false,"required_scope":null},{"alternatives":[[{"column":"WSAL_VAL","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:WSAL_VAL","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"SEMP_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"self_employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"A_SEX","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_male","entity":"person","value_kind":"finite_numeric"}]],"label":"sex","optional":false,"required_scope":null},{"alternatives":[[{"column":"SS_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"social_security_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"source_wrapper:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@post_clone_source_execution_config","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"source_wrapper:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"source_wrapper:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"source_wrapper:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:source_evidence","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"support_clone_index","optional":true,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_role_input","entity":"person","value_kind":"non_null"}]],"label":"tax_unit_role","optional":false,"required_scope":null}]},{"operator":"with_us_retirement_distribution_inputs","requirements":[{"alternatives":[[{"column":"A_AGE","entity":"person","value_kind":"finite_numeric"}],[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"has_esi","entity":"person","value_kind":"finite_numeric"}]],"label":"employer_health_coverage","optional":false,"required_scope":null},{"alternatives":[[{"column":"WSAL_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"filing_status","entity":"tax_unit","value_kind":"non_null"}],[{"column":"filing_status_input","entity":"tax_unit","value_kind":"non_null"}]],"label":"filing_status","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_tax_unit_link","optional":false,"required_scope":null},{"alternatives":[[{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"}]],"label":"puf_taxable_ira_distribution","optional":false,"required_scope":null},{"alternatives":[[{"column":"DST_SC1","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:DST_SC1","optional":false,"required_scope":null},{"alternatives":[[{"column":"DST_SC1_YNG","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:DST_SC1_YNG","optional":false,"required_scope":null},{"alternatives":[[{"column":"DST_SC2","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:DST_SC2","optional":false,"required_scope":null},{"alternatives":[[{"column":"DST_SC2_YNG","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:DST_SC2_YNG","optional":false,"required_scope":null},{"alternatives":[[{"column":"DST_VAL1","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:DST_VAL1","optional":false,"required_scope":null},{"alternatives":[[{"column":"DST_VAL1_YNG","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:DST_VAL1_YNG","optional":false,"required_scope":null},{"alternatives":[[{"column":"DST_VAL2","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:DST_VAL2","optional":false,"required_scope":null},{"alternatives":[[{"column":"DST_VAL2_YNG","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:DST_VAL2_YNG","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"SEMP_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"self_employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"A_SEX","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_male","entity":"person","value_kind":"finite_numeric"}]],"label":"sex","optional":false,"required_scope":null},{"alternatives":[[{"column":"SS_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"social_security_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"source_wrapper:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@post_clone_source_execution_config","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"source_wrapper:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"source_wrapper:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"source_wrapper:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:source_evidence","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"support_clone_index","optional":true,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_role_input","entity":"person","value_kind":"non_null"}]],"label":"tax_unit_role","optional":false,"required_scope":null}]},{"operator":"with_us_weeks_unemployed","requirements":[{"alternatives":[[{"column":"A_AGE","entity":"person","value_kind":"finite_numeric"}],[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_tax_unit_dependent","entity":"person","value_kind":"finite_numeric"},{"column":"is_tax_unit_head","entity":"person","value_kind":"finite_numeric"},{"column":"is_tax_unit_spouse","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_unit_role_input","entity":"person","value_kind":"non_null"}]],"label":"explicit_tax_unit_roles","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"non_null"},{"column":"filing_status","entity":"tax_unit","value_kind":"non_null"},{"column":"tax_unit_id","entity":"tax_unit","value_kind":"non_null"}],[{"column":"person_tax_unit_id","entity":"person","value_kind":"non_null"},{"column":"filing_status_input","entity":"tax_unit","value_kind":"non_null"},{"column":"tax_unit_id","entity":"tax_unit","value_kind":"non_null"}],[{"column":"tax_unit_is_joint","entity":"person","value_kind":"finite_numeric"}]],"label":"joint_filing_status","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"A_SEX","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_male","entity":"person","value_kind":"finite_numeric"}]],"label":"sex","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"finite_numeric"}]],"label":"source_identity","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"source_wrapper:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@post_clone_source_execution_config","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"source_wrapper:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"source_wrapper:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"source_wrapper:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:source_evidence","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"source_year","entity":"person","value_kind":"finite_numeric"}]],"label":"source_year","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"UC_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"unemployment_compensation","entity":"person","value_kind":"finite_numeric"}]],"label":"unemployment_compensation_predictor","optional":true,"required_scope":null},{"alternatives":[[{"column":"LKWEEKS","entity":"person","value_kind":"finite_numeric"}]],"label":"weeks_source","optional":false,"required_scope":null}]},{"operator":"with_us_wic_claim_input","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_pregnant","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:is_pregnant","optional":false,"required_scope":null},{"alternatives":[[{"column":"own_children_in_household","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:own_children_in_household","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"source_wrapper:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@post_clone_source_execution_config","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"source_wrapper:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"source_wrapper:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"source_wrapper:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:source_evidence","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"non_null"}],[{"column":"person_support_source_id","entity":"person","value_kind":"non_null"}],[{"column":"source_household_id","entity":"person","value_kind":"non_null"},{"column":"source_person_id","entity":"person","value_kind":"non_null"},{"column":"source_year","entity":"person","value_kind":"non_null"}]],"label":"stable_source_identity","optional":false,"required_scope":null}]},{"operator":"with_us_workers_compensation","requirements":[{"alternatives":[[{"column":"A_AGE","entity":"person","value_kind":"finite_numeric"}],[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"has_esi","entity":"person","value_kind":"finite_numeric"}]],"label":"employer_health_coverage","optional":false,"required_scope":null},{"alternatives":[[{"column":"WSAL_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"filing_status","entity":"tax_unit","value_kind":"non_null"}],[{"column":"filing_status_input","entity":"tax_unit","value_kind":"non_null"}]],"label":"filing_status","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_tax_unit_link","optional":false,"required_scope":null},{"alternatives":[[{"column":"WC_VAL","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:WC_VAL","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"SEMP_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"self_employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"A_SEX","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_male","entity":"person","value_kind":"finite_numeric"}]],"label":"sex","optional":false,"required_scope":null},{"alternatives":[[{"column":"SS_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"social_security_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"source_wrapper:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@post_clone_source_execution_config","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"source_wrapper:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"source_wrapper:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"source_wrapper:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:source_evidence","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"support_clone_index","optional":true,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_role_input","entity":"person","value_kind":"non_null"}]],"label":"tax_unit_role","optional":false,"required_scope":null}]}],"source_producer_count":16,"status":"derived_and_import_validated","transfer_group_count":19,"transfer_groups":[{"entity":"person","family":"adult_care","name":"transfer:person/adult_care","targets":["is_incapable_of_self_care","pre_subsidy_care_expenses"]},{"entity":"person","family":"model_required_boolean","name":"transfer:person/model_required_boolean","targets":["is_pregnant"]},{"entity":"person","family":"puf_tax_itemization__batch_1","name":"transfer:person/puf_tax_itemization__batch_1","targets":["tax_exempt_interest_income","long_term_capital_gains_on_collectibles","non_sch_d_capital_gains","alimony_expense","salt_refund_income","charitable_cash_donations","charitable_non_cash_donations","home_mortgage_interest"]},{"entity":"person","family":"puf_tax_itemization__batch_2","name":"transfer:person/puf_tax_itemization__batch_2","targets":["investment_interest_expense","investment_income_elected_form_4952","student_loan_interest","educator_expense","qualified_tuition_expenses","casualty_loss","unreimbursed_business_employee_expenses","traditional_ira_contributions_desired"]},{"entity":"person","family":"puf_tax_itemization__batch_3","name":"transfer:person/puf_tax_itemization__batch_3","targets":["self_employed_pension_contributions_desired","estate_income","farm_income","farm_rent_income","partnership_income","partnership_self_employment_net_earnings","estate_income_would_be_qualified","farm_operations_income_would_be_qualified"]},{"entity":"person","family":"puf_tax_itemization__batch_4","name":"transfer:person/puf_tax_itemization__batch_4","targets":["farm_rent_income_would_be_qualified","partnership_s_corp_income_would_be_qualified","rental_income_would_be_qualified","self_employment_income_would_be_qualified","sstb_self_employment_income_would_be_qualified","business_is_sstb","qualified_bdc_income","qualified_reit_and_ptp_income"]},{"entity":"person","family":"puf_tax_itemization__batch_5","name":"transfer:person/puf_tax_itemization__batch_5","targets":["sstb_self_employment_income_before_lsr","sstb_unadjusted_basis_qualified_property","sstb_w2_wages_from_qualified_business","unadjusted_basis_qualified_property","w2_wages_from_qualified_business"]},{"entity":"person","family":"source_operator_child_support","name":"transfer:person/source_operator_child_support","targets":["child_support_expense","child_support_received"]},{"entity":"person","family":"source_operator_disability_benefits","name":"transfer:person/source_operator_disability_benefits","targets":["disability_benefits"]},{"entity":"person","family":"source_operator_education_inputs","name":"transfer:person/source_operator_education_inputs","targets":["attends_eligible_educational_institution_for_american_opportunity_credit","educational_assistance","has_american_opportunity_credit_1098_t_or_exception","has_american_opportunity_credit_institution_ein","is_enrolled_at_least_half_time_for_american_opportunity_credit","is_pursuing_credential_for_american_opportunity_credit"]},{"entity":"person","family":"source_operator_immigration","name":"transfer:person/source_operator_immigration","targets":["ssn_card_type","immigration_status_str"]},{"entity":"person","family":"source_operator_medicare_take_up","name":"transfer:person/source_operator_medicare_take_up","targets":["takes_up_medicare_if_eligible"]},{"entity":"person","family":"source_operator_retirement_contributions","name":"transfer:person/source_operator_retirement_contributions","targets":["roth_401k_contributions_desired","roth_ira_contributions_desired","traditional_401k_contributions_desired"]},{"entity":"person","family":"source_operator_retirement_distributions","name":"transfer:person/source_operator_retirement_distributions","targets":["keogh_distributions","tax_exempt_ira_distributions","taxable_401k_distributions","taxable_403b_distributions","taxable_sep_distributions"]},{"entity":"person","family":"source_operator_weeks_unemployed","name":"transfer:person/source_operator_weeks_unemployed","targets":["weeks_unemployed"]},{"entity":"person","family":"source_operator_wic_claim","name":"transfer:person/source_operator_wic_claim","targets":["takes_up_wic_if_eligible"]},{"entity":"person","family":"source_operator_workers_compensation","name":"transfer:person/source_operator_workers_compensation","targets":["workers_compensation"]},{"entity":"tax_unit","family":"puf_tax_itemization","name":"transfer:tax_unit/puf_tax_itemization","targets":["domestic_production_ald","unrecaptured_section_1250_gain","first_home_mortgage_balance","first_home_mortgage_interest","first_home_mortgage_origination_year","health_savings_account_ald"]},{"entity":"spm_unit","family":"source_operator_energy_subsidy","name":"transfer:spm_unit/source_operator_energy_subsidy","targets":["spm_unit_energy_subsidy"]}],"transfer_input_inventories":[{"operator":"transfer:person/adult_care","requirements":[{"alternatives":[[{"column":"tax_unit_role_input","entity":"person","value_kind":"non_null"}]],"label":"adult_care_tax_unit_role","optional":false,"required_scope":null},{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:person/model_required_boolean","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:person/puf_tax_itemization__batch_1","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:person/puf_tax_itemization__batch_2","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:person/puf_tax_itemization__batch_3","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:person/puf_tax_itemization__batch_4","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:person/puf_tax_itemization__batch_5","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:person/source_operator_child_support","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:person/source_operator_disability_benefits","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:person/source_operator_education_inputs","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:person/source_operator_immigration","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:person/source_operator_medicare_take_up","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:person/source_operator_retirement_contributions","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:person/source_operator_retirement_distributions","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:person/source_operator_weeks_unemployed","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:person/source_operator_wic_claim","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:person/source_operator_workers_compensation","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:spm_unit/source_operator_energy_subsidy","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"spm_unit","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"spm_unit","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"spm_unit","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:tax_unit/puf_tax_itemization","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"tax_unit","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"tax_unit","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"tax_unit","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]}],"transfer_target_count":70,"waves":[["acs_pums_earnings_universe"],["primary_puf_qrf"],["source:impute_us_housing_assistance_to_puf_support","source:with_us_child_support_inputs","source:with_us_childcare_inputs","source:with_us_disability_benefits","source:with_us_energy_subsidy_input","source:with_us_immigration_inputs","source:with_us_medicare_take_up_input","source:with_us_pregnancy_inputs","source:with_us_prior_year_income_inputs","source:with_us_retirement_contribution_inputs","source:with_us_retirement_distribution_inputs","source:with_us_weeks_unemployed","source:with_us_workers_compensation","transfer:person/puf_tax_itemization__batch_1","transfer:person/puf_tax_itemization__batch_4","transfer:person/puf_tax_itemization__batch_5","transfer:tax_unit/puf_tax_itemization"],["source:with_us_adult_care_inputs","source:with_us_wic_claim_input","transfer:person/model_required_boolean","transfer:person/puf_tax_itemization__batch_2","transfer:person/puf_tax_itemization__batch_3","transfer:person/source_operator_child_support","transfer:person/source_operator_disability_benefits","transfer:person/source_operator_immigration","transfer:person/source_operator_medicare_take_up","transfer:person/source_operator_retirement_contributions","transfer:person/source_operator_retirement_distributions","transfer:person/source_operator_weeks_unemployed","transfer:person/source_operator_workers_compensation","transfer:spm_unit/source_operator_energy_subsidy"],["source:with_us_education_inputs","transfer:person/adult_care","transfer:person/source_operator_wic_claim"],["source_finalizer","transfer:person/source_operator_education_inputs"]]} diff --git a/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json index e1b0f72ad5..99b4070aef 100644 --- a/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json @@ -1493,6 +1493,104 @@ "parameter_changes": {}, "period": 2024, "reason": "PolicyEngine-US multiplies HUD HAP by the restored SPM-unit take-up leaf after eligibility. Microcosm keeps that leaf exactly equal to source-backed housing-assistance receipt, so neutralizing it must remove the assistance paid to measured/imputed recipients. A 6,000-household production-ingredient smoke scored $202.795 million baseline-minus-neutralized; the $100 million floor is below that observed subset effect but far above numerical noise. A default-only or absent carry makes the source reconciliation or this uniquely isolating probe fail." + }, + { + "binding_inputs": [ + "immigration_status_str" + ], + "budget_measure": "medicaid_enrolled", + "effect_direction": "reform_minus_baseline", + "expected_sign": "positive", + "id": "hr1_medicaid_humanitarian_eligibility_restoration", + "issue": "PolicyEngine/microcosm#767", + "measure_unit": "persons", + "min_abs_effect": 10000.0, + "name": "H.R.1 SS71109 Medicaid humanitarian-status restoration", + "parameter_changes": { + "gov.hhs.medicaid.eligibility.eligible_immigration_statuses": { + "2026-10-01.2100-12-31": [ + "CITIZEN", + "LEGAL_PERMANENT_RESIDENT", + "REFUGEE", + "ASYLEE", + "DEPORTATION_WITHHELD", + "CUBAN_HAITIAN_ENTRANT", + "CONDITIONAL_ENTRANT", + "PAROLED_ONE_YEAR" + ] + } + }, + "period": 2027, + "reason": "H.R.1 SS71109 narrows the Medicaid qualified-immigrant list to citizens, LPRs, and Cuban/Haitian entrants from 2026-10-01. The probe restores the pre-narrowing list at 2027 law, adding REFUGEE, ASYLEE, DEPORTATION_WITHHELD, PAROLED_ONE_YEAR and CONDITIONAL_ENTRANT; the source stage draws no conditional entrants, so the reform binds only through the first four values of immigration_status_str. The probe counts weighted people in medicaid_enrolled rather than summing medicaid dollars, because PolicyEngine-US prices each enrollee through medicaid_slcsp_state_denominator, an aggregate over the whole file that the household-batched post-export scorer refuses (PolicyEngine/microcosm#956). On the default-pool stage receipt (docs/evidence/us-immigration-statuses-767), 0.391 million of the 0.714 million weighted people in the restored statuses report Medicaid at interview, and the Medicaid take-up stage sets takes_up_medicaid_if_eligible for every reporter. The reform only widens the status list, so every reporter whom the narrowed list alone kept out enrolls. The 10,000-person floor is 2.6% of the reporters: a structural zero fails it, and the effect clears it unless Medicaid's other rules keep out more than 97% of the reporters. A ~0 count means the humanitarian statuses regressed to zero records (microcosm #767's silent-zero failure) or the engine channel broke." + }, + { + "binding_inputs": [ + "immigration_status_str" + ], + "budget_measure": "aca_ptc", + "effect_direction": "reform_minus_baseline", + "expected_sign": "positive", + "id": "hr1_aca_below_fpl_exception_restoration", + "issue": "PolicyEngine/microcosm#767", + "min_abs_effect": 5000000.0, + "name": "H.R.1 SS71302 ACA below-FPL immigrant exception restoration", + "parameter_changes": { + "gov.aca.below_fpl_immigration_exception_in_effect": { + "2026-01-01.2026-12-31": true + } + }, + "period": 2026, + "reason": "H.R.1 SS71302 repeals the 26 USC 36B(c)(1)(B) below-FPL lawfully-present exception from 2026. Restoring it for 2026 binds, at 2026 law, only through tax units containing a non-citizen who is ACA-lawfully-present yet Medicaid-ineligible by status - on this file exactly the TPS population the source stage draws to the CRS RS20844 per-country stocks (0.860 million weighted persons on the default-pool stage receipt, docs/evidence/us-immigration-statuses-767; the other humanitarian classes remain Medicaid-qualified at 2026 annual parameters). Below-poverty TPS units with marketplace take-up must regain premium tax credits above the floor; ~$0 means the TPS records vanished or the exception channel broke." + }, + { + "binding_inputs": [ + "immigration_status_str" + ], + "budget_measure": "aca_ptc", + "effect_direction": "reform_minus_baseline", + "expected_sign": "positive", + "id": "hr1_aca_lawful_presence_restoration", + "issue": "PolicyEngine/microcosm#767", + "min_abs_effect": 10000000.0, + "name": "H.R.1 SS71301 ACA lawful-presence list restoration", + "parameter_changes": { + "gov.aca.ineligible_immigration_statuses": { + "2027-01-01.2100-12-31": [ + "DACA", + "UNDOCUMENTED" + ] + } + }, + "period": 2027, + "reason": "H.R.1 SS71301 adds TPS/REFUGEE/ASYLEE/DEPORTATION_WITHHELD/PAROLED_ONE_YEAR to the ACA ineligible-status list from 2027-01-01. The probe restores the pre-H.R.1 list (DACA and UNDOCUMENTED only) at 2027, re-qualifying every humanitarian status the source stage draws (1.574 million weighted persons across parole, refugee, asylee and TPS on the default-pool stage receipt). Units among them with marketplace take-up must regain premium tax credits above the floor; ~$0 means the humanitarian statuses regressed or the lawful-presence channel broke." + }, + { + "binding_inputs": [ + "immigration_status_str" + ], + "budget_measure": "snap", + "effect_direction": "reform_minus_baseline", + "expected_sign": "positive", + "id": "hr1_snap_humanitarian_eligibility_restoration", + "issue": "PolicyEngine/microcosm#767", + "min_abs_effect": 25000000.0, + "name": "H.R.1 SS10108 SNAP humanitarian-status restoration", + "parameter_changes": { + "gov.usda.snap.eligibility.eligible_immigration_statuses": { + "2025-07-01.2100-12-31": [ + "CITIZEN", + "LEGAL_PERMANENT_RESIDENT", + "REFUGEE", + "ASYLEE", + "DEPORTATION_WITHHELD", + "CUBAN_HAITIAN_ENTRANT", + "CONDITIONAL_ENTRANT", + "PAROLED_ONE_YEAR" + ] + } + }, + "period": 2026, + "reason": "H.R.1 SS10108 narrows SNAP alien eligibility to citizens, LPRs, Cuban/Haitian entrants, and COFA citizens from 2025-07-01 (modeled 2025-07). Restoring the pre-H.R.1 list at 2026 re-includes the REFUGEE/ASYLEE/DEPORTATION_WITHHELD/PAROLED_ONE_YEAR members the source stage draws to cited DHS/OHSS stocks; is_snap_excluded_member stops excluding them, so SNAP allotments for their households must rise above the floor. ~$0 means the humanitarian statuses regressed to zero records or the SNAP immigration channel broke." } ], "schema_version": 1, diff --git a/packages/microcosm-build/src/microcosm/build/us/source_stages.json b/packages/microcosm-build/src/microcosm/build/us/source_stages.json index c37306b4ad..5fa92a3a94 100644 --- a/packages/microcosm-build/src/microcosm/build/us/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/us/source_stages.json @@ -2143,20 +2143,20 @@ { "stage": "immigration_status", "survey": "CPS ASEC + published unauthorized-population estimates", - "source": "https://www.pewresearch.org/short-reads/2024/07/22/what-we-know-about-unauthorized-immigrants-living-in-the-us/", + "source": "https://www.pewresearch.org/race-and-ethnicity/2025/08/21/u-s-unauthorized-immigrant-population-reached-a-record-14-million-in-2023/", "grain": "person", "artifacts": [ { "kind": "public_microdata", "format": "census_asec", "vintage": "build_year", - "locator": "Census CPS ASEC person file citizenship (PRCITSHP), entry year (PEINUSYR), nativity (PENATVTY), and program-participation fields" + "locator": "Census CPS ASEC person file citizenship (PRCITSHP), entry year (PEINUSYR), nativity (PENATVTY), labor-force status (A_LFSR), and program-participation fields" }, { "kind": "administrative_table", "format": "published_estimate", "vintage": "latest_available", - "locator": "Pew Research Center unauthorized-immigrant population and worker estimates; Higher Ed Immigration Portal undocumented-student estimates" + "locator": "Pew Research Center unauthorized-immigrant population and worker estimates; Higher Ed Immigration Portal undocumented-student estimates; OHSS refugee and asylee annual flow reports; DHS OAW/U4U parole reports and CBP CHNV releases; CRS RS20844 TPS designations table; EOIR adjudication statistics via CRS IN12501" } ], "operations": [ @@ -2170,16 +2170,74 @@ "seed_from_build_config": true, "time_period_from_build_config": true, "undocumented_workers": { - "target": 8300000, - "source": "https://www.pewresearch.org/short-reads/2024/07/22/what-we-know-about-unauthorized-immigrants-living-in-the-us/" + "target": 9700000, + "source": "https://www.pewresearch.org/race-and-ethnicity/2025/08/21/u-s-unauthorized-immigrant-population-reached-a-record-14-million-in-2023/" }, "undocumented_students": { "target": 408000, "source": "https://www.higheredimmigrationportal.org/research/undocumented-students-in-higher-education-updated-march-2021/" }, "undocumented_population_anchor": { - "value": 11000000, - "source": "https://www.pewresearch.org/short-reads/2024/07/22/what-we-know-about-unauthorized-immigrants-living-in-the-us/" + "value": 14000000, + "source": "https://www.pewresearch.org/race-and-ethnicity/2025/08/21/u-s-unauthorized-immigrant-population-reached-a-record-14-million-in-2023/" + }, + "humanitarian_status_stocks": { + "paroled_one_year": { + "afghanistan": { + "target": 73566, + "source": "https://www.dhs.gov/sites/default/files/2023-06/PLCY%20-%20AFG%202503%20Update%202_0.pdf" + }, + "ukraine": { + "target": 158000, + "source": "https://www.dhs.gov/sites/default/files/2024-12/2024_1104_dmo_plcy_uniting_for_ukraine_process_overview_and_assessment.pdf" + }, + "nicaragua": { + "target": 93070, + "source": "https://www.cbp.gov/newsroom/national-media-release/cbp-releases-december-2024-monthly-update" + }, + "venezuela": { + "target": 117330, + "source": "https://www.cbp.gov/newsroom/national-media-release/cbp-releases-december-2024-monthly-update" + } + }, + "refugee": { + "target": 160000, + "source": "https://ohss.dhs.gov/topics/immigration/refugees/annual-flow-report/fy-24-refugees-flow-report" + }, + "asylee": { + "target": 155000, + "source": "https://ohss.dhs.gov/topics/immigration/asylees/annual-flow-report/fy24-asylees-flow-report" + }, + "deportation_withheld": { + "target": 0, + "source": "https://crsreports.congress.gov/product/pdf/IN/IN12501" + }, + "tps": { + "venezuela": { + "target": 605015, + "source": "https://www.everycrsreport.com/reports/RS20844.html" + }, + "el_salvador": { + "target": 170125, + "source": "https://www.everycrsreport.com/reports/RS20844.html" + }, + "honduras": { + "target": 51225, + "source": "https://www.everycrsreport.com/reports/RS20844.html" + }, + "nicaragua": { + "target": 2910, + "source": "https://www.everycrsreport.com/reports/RS20844.html" + }, + "nepal": { + "target": 7160, + "source": "https://www.everycrsreport.com/reports/RS20844.html" + }, + "other_designated": { + "target": 21215, + "source": "https://www.everycrsreport.com/reports/RS20844.html" + } + } } } ], @@ -2187,7 +2245,7 @@ "ssn_card_type", "immigration_status_str" ], - "notes": "Citizenship is measured (PRCITSHP), not imputed. Non-citizens with any ASEC-UA legal-status indicator (Van Hook et al., SSRN 4662801) become OTHER_NON_CITIZEN. The CPS carries no work-authorization variable, so among residual non-citizens, workers and students spill to NON_CITIZEN_VALID_EAD in deterministic seeded order until the remaining undocumented worker count matches Pew's 8.3M and the undocumented student count matches the Higher Ed Immigration Portal's ~408k \u2014 the only forced margins. The total undocumented population is emergent (\u224813M on the 2024 ASEC, within the range of published 2023-24 estimates) and is gated against Pew's 11.0M 2022 anchor with a coarse plausibility band rather than forced; reconciling the level is the calibration lane's job. Immigration-status tags carry only statutory tests the data supports: DACA (arrival cohort among EAD holders) and CUBAN_HAITIAN_ENTRANT (nativity plus post-1980 arrival); other documented non-citizens stay LEGAL_PERMANENT_RESIDENT, the modal true status \u2014 blanket REFUGEE/TPS labels are deliberately not fabricated because they would mislabel millions and over-grant refugee-class benefit exemptions." + "notes": "Citizenship is measured (PRCITSHP), not imputed. Non-citizens with any ASEC-UA legal-status indicator (Van Hook et al., SSRN 4662801) become OTHER_NON_CITIZEN. Humanitarian/temporary-protection statuses (microcosm #767) are drawn between the indicators and the EAD spill, sequentially and mutually exclusively in decreasing order of target precision \u2014 PAROLED_ONE_YEAR per program origin (OAW Afghans 73,566 as of 2022-03-31; U4U Ukrainians over 158,000 as of 2023-09-30; CHNV Nicaraguans 93,070 and Venezuelans 117,330 paroled through 2024-12), REFUGEE (FY2023+FY2024 admissions, 60,050+100,060 OHSS \u2014 refugees must apply for LPR adjustment after one year, so the not-yet-adjusted stock is about the trailing two years), ASYLEE (FY2022-FY2024 grants 35,080+51,410+68,060=154,550 OHSS, rounded to 155,000; 88% of 2015-22 affirmative adult grantees had adjusted by end-2024, so about three trailing years), then TPS per country (CRS RS20844 Table 1 as of 2025-03-31, with required-arrival cutoffs binding at PEINUSYR granularity: El Salvador <=2001, Honduras/Nicaragua <=1999, Nepal <=2015). Candidates are restricted by program origin countries (Census Appendix I PENATVTY codes) and arrival windows; Cuba/Haiti-born are excluded everywhere (the CUBAN_HAITIAN_ENTRANT class is the better statutory label and keeps H.R.1 eligibility \u2014 Haiti TPS 330,735 rides that label), the DACA statutory cohort is excluded (dual TPS/DACA holders stay DACA), and Ukraine/Afghanistan TPS registrants (101,150/8,105) are carried by the parole draw since they are overwhelmingly the same U4U/OAW parolees. Parole targets are gross admissions: cohort members who later won asylum keep the parole label, which is channel-equivalent under H.R.1 (both classes lose Medicaid/SNAP/PTC on identical dates). DEPORTATION_WITHHELD carries an explicit zero target: EOIR grants about 2.2k withholdings a year (CRS IN12501: under 1% of 666,177 FY2024 decisions) and publishes no stock, so the accumulated population is below the CPS's resolution for a defensible draw. CONDITIONAL_ENTRANT is deliberately not emitted (the INA 203(a)(7) class closed in 1980). REFUGEE/ASYLEE draw from the indicator-documented pool only; parole/TPS may draw residual-pool people, who then receive NON_CITIZEN_VALID_EAD SSN cards (both programs confer employment authorization). Pew's 2025 report supersedes its earlier series and estimates a coherent 2023 pair of 14.0M unauthorized immigrants and 9.7M in the labor force. Its modeled universe is broader than the engine's UNDOCUMENTED enum: it retains DACA recipients, parolees, TPS holders, similar temporary protections, and the residual-EAD subset of the engine's broad CUBAN_HAITIAN_ENTRANT class, while excluding refugees, people already granted asylum, and assumed-documented Cuban/Haitian entrants. The worker EAD spill binds that modeled universe to 9.7M using actual ASEC labor-force status (A_LFSR codes 1-4 at ages 16+) restored from the same pinned official ASEC archives by exact PERIDNUM join, never prior-year WSAL_VAL/SEMP_VAL earnings. The student spill binds the corresponding broad residual universe to the Higher Ed Immigration Portal's ~408k students. The total modeled Pew-defined unauthorized population is emergent and gated against the 14.0M 2023 anchor with a coarse band; per-category humanitarian masses are gated against their cited targets with a coarse band (the ASEC undercovers 2022-24 arrivals \u2014 the Ukraine parole pool saturates below its admin count); reconciling levels is the calibration lane's job. DACA (arrival cohort among EAD holders) and CUBAN_HAITIAN_ENTRANT (nativity plus post-1980 arrival) keep their statutory tests; other documented non-citizens stay LEGAL_PERMANENT_RESIDENT, the modal true status." }, { "stage": "hours_worked", diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/imputation.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/imputation.yaml index bc359ab3bd..fecc5fe9d6 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/imputation.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/imputation.yaml @@ -342,7 +342,11 @@ waiver_records: - dependent_child_status - pregnancy_status transfer_execution: - schema_version: 2 + schema_version: 3 + immigration_required_predictors: + - __acs_transfer_is_us_citizen + - __acs_transfer_birth_country_code + - __acs_transfer_arrival_year group_optional_names: __acs_transfer_employment_income: __acs_transfer_group_employment_income_sum __acs_transfer_interest_dividend_rental_income: __acs_transfer_group_investment_income_sum @@ -405,6 +409,14 @@ transfer_execution: - ssn_card_type - immigration_status_str immigration_status_model_target: __acs_transfer_immigration_status_pair + immigration_baseline_excluded_statuses: + - ASYLEE + - CUBAN_HAITIAN_ENTRANT + - DACA + - DEPORTATION_WITHHELD + - PAROLED_ONE_YEAR + - REFUGEE + - TPS discrete_numeric_targets: - first_home_mortgage_origination_year - second_home_mortgage_origination_year @@ -450,6 +462,19 @@ transfer_execution: tax_unit_role: tax_unit_role_input tax_unit_link: person_tax_unit_id mutable_rows: newly_imputed_expense_cells_only + humanitarian_immigration: + activation: + all_targets: + - ssn_card_type + - immigration_status_str + contract: + targets: + - ssn_card_type + - immigration_status_str + mutable_rows: newly_imputed_paired_cells_only + target_basis: manifest_residual_after_immutable_asec_mass + selection: stable_person_hash_in_manifest_draw_order + candidate_shortfall: take_every_candidate_error_below_composition_band_floor schedule_d_capital_gain_distributions: activation: derive_schedule_d: true @@ -9530,6 +9555,15 @@ producer_graph: producing_stage: post_clone_input_surface required_scope: asec_source tolerated_absence_receipts: [] + - alternatives: + - - column: A_LFSR + entity: person + value_kind: finite_numeric + column: '@effective:raw_person:A_LFSR' + entity: person + producing_stage: post_clone_input_surface + required_scope: asec_source + tolerated_absence_receipts: [] - alternatives: - - column: A_MARITL entity: person @@ -9683,15 +9717,6 @@ producer_graph: producing_stage: post_clone_input_surface required_scope: asec_source tolerated_absence_receipts: [] - - alternatives: - - - column: SEMP_VAL - entity: person - value_kind: finite_numeric - column: '@effective:raw_person:SEMP_VAL' - entity: person - producing_stage: post_clone_input_surface - required_scope: asec_source - tolerated_absence_receipts: [] - alternatives: - - column: SPM_CAPHOUSESUB entity: person @@ -9719,15 +9744,6 @@ producer_graph: producing_stage: post_clone_input_surface required_scope: asec_source tolerated_absence_receipts: [] - - alternatives: - - - column: WSAL_VAL - entity: person - value_kind: finite_numeric - column: '@effective:raw_person:WSAL_VAL' - entity: person - producing_stage: post_clone_input_surface - required_scope: asec_source - tolerated_absence_receipts: [] - alternatives: - - column: '@resolved_weight' entity: person @@ -9822,6 +9838,15 @@ producer_graph: - - column: person_id entity: person value_kind: non_null + - - column: source_household_id + entity: person + value_kind: non_null + - column: source_person_id + entity: person + value_kind: non_null + - column: source_year + entity: person + value_kind: non_null - - column: source_person_id entity: person value_kind: non_null @@ -24986,6 +25011,84 @@ producer_graph: producing_stage: post_clone_input_surface required_scope: whole_pool tolerated_absence_receipts: [] + - alternatives: + - - column: YOEP + entity: person + value_kind: column_present + column: '@effective:immigration_acs_arrival' + entity: person + producing_stage: post_clone_input_surface + required_scope: acs_source + tolerated_absence_receipts: [] + - alternatives: + - - column: CIT + entity: person + value_kind: finite_numeric + column: '@effective:immigration_acs_citizenship' + entity: person + producing_stage: post_clone_input_surface + required_scope: acs_source + tolerated_absence_receipts: [] + - alternatives: + - - column: POBP + entity: person + value_kind: finite_numeric + column: '@effective:immigration_acs_origin' + entity: person + producing_stage: post_clone_input_surface + required_scope: acs_source + tolerated_absence_receipts: [] + - alternatives: + - - column: PEINUSYR + entity: person + value_kind: finite_numeric + column: '@effective:immigration_asec_arrival' + entity: person + producing_stage: post_clone_input_surface + required_scope: asec_source + tolerated_absence_receipts: [] + - alternatives: + - - column: PRCITSHP + entity: person + value_kind: finite_numeric + column: '@effective:immigration_asec_citizenship' + entity: person + producing_stage: post_clone_input_surface + required_scope: asec_source + tolerated_absence_receipts: [] + - alternatives: + - - column: PENATVTY + entity: person + value_kind: finite_numeric + column: '@effective:immigration_asec_origin' + entity: person + producing_stage: post_clone_input_surface + required_scope: asec_source + tolerated_absence_receipts: [] + - alternatives: + - - column: person_id + entity: person + value_kind: non_null + - - column: source_household_id + entity: person + value_kind: non_null + - column: source_person_id + entity: person + value_kind: non_null + - column: source_year + entity: person + value_kind: non_null + - - column: source_person_id + entity: person + value_kind: non_null + - column: source_year + entity: person + value_kind: non_null + column: '@effective:immigration_stable_person_lineage' + entity: person + producing_stage: post_clone_input_surface + required_scope: whole_pool + tolerated_absence_receipts: [] - alternatives: - - column: is_female entity: person @@ -25305,7 +25408,7 @@ producer_graph: - alternatives: - - column: person_id entity: person - value_kind: finite_numeric + value_kind: non_null column: person_id entity: person producing_stage: primary_puf_qrf @@ -33412,7 +33515,7 @@ producer_graph: version: 4 row_binding: declared_globally_reconciled_input_and_scope_exact_output_source_and_primary_callback_resource_receipt_and_previous_execution_sha256 virtual_resource_binding: exact_kind_specific_semantic_payload_and_sha256 - top_binding: entry_and_output_frame_sha256_execution_chain_source_completion_and_nineteen_transfer_groups + top_binding: entry_and_output_frame_sha256_execution_chain_source_completion_nineteen_transfer_groups_and_constrained_immigration_reconciliation transition_authority: authority_id: us_stacked_late_producer_transition metadata_key: us_late_producer_transition_authority diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml index 74e61be790..5e2e9fca07 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml @@ -99,7 +99,7 @@ sources: stage_asset: id: source_stages path: microcosm.build.us/source_stages.json - sha256: 959214c437bfa59d43e32279b82f29a0987649cd7579ff87567c70f084eae66c + sha256: 67587983a455bfffde688b024e7d4bd03a9d1d1f3b79291881558e60bb20311a stage_manifest: version: 1 country: us @@ -2238,19 +2238,21 @@ stages: flag persist. - stage: immigration_status survey: CPS ASEC + published unauthorized-population estimates - source: https://www.pewresearch.org/short-reads/2024/07/22/what-we-know-about-unauthorized-immigrants-living-in-the-us/ + source: https://www.pewresearch.org/race-and-ethnicity/2025/08/21/u-s-unauthorized-immigrant-population-reached-a-record-14-million-in-2023/ grain: person artifacts: - kind: public_microdata format: census_asec vintage: build_year locator: Census CPS ASEC person file citizenship (PRCITSHP), entry year (PEINUSYR), nativity (PENATVTY), - and program-participation fields + labor-force status (A_LFSR), and program-participation fields - kind: administrative_table format: published_estimate vintage: latest_available locator: Pew Research Center unauthorized-immigrant population and worker estimates; Higher Ed Immigration - Portal undocumented-student estimates + Portal undocumented-student estimates; OHSS refugee and asylee annual flow reports; DHS OAW/U4U + parole reports and CBP CHNV releases; CRS RS20844 TPS designations table; EOIR adjudication statistics + via CRS IN12501 operations: - kind: read_table table: person @@ -2259,29 +2261,96 @@ stages: seed_from_build_config: true time_period_from_build_config: true undocumented_workers: - target: 8300000 - source: https://www.pewresearch.org/short-reads/2024/07/22/what-we-know-about-unauthorized-immigrants-living-in-the-us/ + target: 9700000 + source: https://www.pewresearch.org/race-and-ethnicity/2025/08/21/u-s-unauthorized-immigrant-population-reached-a-record-14-million-in-2023/ undocumented_students: target: 408000 source: https://www.higheredimmigrationportal.org/research/undocumented-students-in-higher-education-updated-march-2021/ undocumented_population_anchor: - value: 11000000 - source: https://www.pewresearch.org/short-reads/2024/07/22/what-we-know-about-unauthorized-immigrants-living-in-the-us/ + value: 14000000 + source: https://www.pewresearch.org/race-and-ethnicity/2025/08/21/u-s-unauthorized-immigrant-population-reached-a-record-14-million-in-2023/ + humanitarian_status_stocks: + paroled_one_year: + afghanistan: + target: 73566 + source: https://www.dhs.gov/sites/default/files/2023-06/PLCY%20-%20AFG%202503%20Update%202_0.pdf + ukraine: + target: 158000 + source: https://www.dhs.gov/sites/default/files/2024-12/2024_1104_dmo_plcy_uniting_for_ukraine_process_overview_and_assessment.pdf + nicaragua: + target: 93070 + source: https://www.cbp.gov/newsroom/national-media-release/cbp-releases-december-2024-monthly-update + venezuela: + target: 117330 + source: https://www.cbp.gov/newsroom/national-media-release/cbp-releases-december-2024-monthly-update + refugee: + target: 160000 + source: https://ohss.dhs.gov/topics/immigration/refugees/annual-flow-report/fy-24-refugees-flow-report + asylee: + target: 155000 + source: https://ohss.dhs.gov/topics/immigration/asylees/annual-flow-report/fy24-asylees-flow-report + deportation_withheld: + target: 0 + source: https://crsreports.congress.gov/product/pdf/IN/IN12501 + tps: + venezuela: + target: 605015 + source: https://www.everycrsreport.com/reports/RS20844.html + el_salvador: + target: 170125 + source: https://www.everycrsreport.com/reports/RS20844.html + honduras: + target: 51225 + source: https://www.everycrsreport.com/reports/RS20844.html + nicaragua: + target: 2910 + source: https://www.everycrsreport.com/reports/RS20844.html + nepal: + target: 7160 + source: https://www.everycrsreport.com/reports/RS20844.html + other_designated: + target: 21215 + source: https://www.everycrsreport.com/reports/RS20844.html outputs: - ssn_card_type - immigration_status_str notes: 'Citizenship is measured (PRCITSHP), not imputed. Non-citizens with any ASEC-UA legal-status - indicator (Van Hook et al., SSRN 4662801) become OTHER_NON_CITIZEN. The CPS carries no work-authorization - variable, so among residual non-citizens, workers and students spill to NON_CITIZEN_VALID_EAD in deterministic - seeded order until the remaining undocumented worker count matches Pew''s 8.3M and the undocumented - student count matches the Higher Ed Immigration Portal''s ~408k — the only forced margins. The total - undocumented population is emergent (≈13M on the 2024 ASEC, within the range of published 2023-24 - estimates) and is gated against Pew''s 11.0M 2022 anchor with a coarse plausibility band rather than - forced; reconciling the level is the calibration lane''s job. Immigration-status tags carry only statutory - tests the data supports: DACA (arrival cohort among EAD holders) and CUBAN_HAITIAN_ENTRANT (nativity - plus post-1980 arrival); other documented non-citizens stay LEGAL_PERMANENT_RESIDENT, the modal true - status — blanket REFUGEE/TPS labels are deliberately not fabricated because they would mislabel millions - and over-grant refugee-class benefit exemptions.' + indicator (Van Hook et al., SSRN 4662801) become OTHER_NON_CITIZEN. Humanitarian/temporary-protection + statuses (microcosm #767) are drawn between the indicators and the EAD spill, sequentially and mutually + exclusively in decreasing order of target precision — PAROLED_ONE_YEAR per program origin (OAW Afghans + 73,566 as of 2022-03-31; U4U Ukrainians over 158,000 as of 2023-09-30; CHNV Nicaraguans 93,070 and + Venezuelans 117,330 paroled through 2024-12), REFUGEE (FY2023+FY2024 admissions, 60,050+100,060 OHSS + — refugees must apply for LPR adjustment after one year, so the not-yet-adjusted stock is about the + trailing two years), ASYLEE (FY2022-FY2024 grants 35,080+51,410+68,060=154,550 OHSS, rounded to 155,000; + 88% of 2015-22 affirmative adult grantees had adjusted by end-2024, so about three trailing years), + then TPS per country (CRS RS20844 Table 1 as of 2025-03-31, with required-arrival cutoffs binding + at PEINUSYR granularity: El Salvador <=2001, Honduras/Nicaragua <=1999, Nepal <=2015). Candidates + are restricted by program origin countries (Census Appendix I PENATVTY codes) and arrival windows; + Cuba/Haiti-born are excluded everywhere (the CUBAN_HAITIAN_ENTRANT class is the better statutory label + and keeps H.R.1 eligibility — Haiti TPS 330,735 rides that label), the DACA statutory cohort is excluded + (dual TPS/DACA holders stay DACA), and Ukraine/Afghanistan TPS registrants (101,150/8,105) are carried + by the parole draw since they are overwhelmingly the same U4U/OAW parolees. Parole targets are gross + admissions: cohort members who later won asylum keep the parole label, which is channel-equivalent + under H.R.1 (both classes lose Medicaid/SNAP/PTC on identical dates). DEPORTATION_WITHHELD carries + an explicit zero target: EOIR grants about 2.2k withholdings a year (CRS IN12501: under 1% of 666,177 + FY2024 decisions) and publishes no stock, so the accumulated population is below the CPS''s resolution + for a defensible draw. CONDITIONAL_ENTRANT is deliberately not emitted (the INA 203(a)(7) class closed + in 1980). REFUGEE/ASYLEE draw from the indicator-documented pool only; parole/TPS may draw residual-pool + people, who then receive NON_CITIZEN_VALID_EAD SSN cards (both programs confer employment authorization). + Pew''s 2025 report supersedes its earlier series and estimates a coherent 2023 pair of 14.0M unauthorized + immigrants and 9.7M in the labor force. Its modeled universe is broader than the engine''s UNDOCUMENTED + enum: it retains DACA recipients, parolees, TPS holders, similar temporary protections, and the residual-EAD + subset of the engine''s broad CUBAN_HAITIAN_ENTRANT class, while excluding refugees, people already + granted asylum, and assumed-documented Cuban/Haitian entrants. The worker EAD spill binds that modeled + universe to 9.7M using actual ASEC labor-force status (A_LFSR codes 1-4 at ages 16+) restored from + the same pinned official ASEC archives by exact PERIDNUM join, never prior-year WSAL_VAL/SEMP_VAL + earnings. The student spill binds the corresponding broad residual universe to the Higher Ed Immigration + Portal''s ~408k students. The total modeled Pew-defined unauthorized population is emergent and gated + against the 14.0M 2023 anchor with a coarse band; per-category humanitarian masses are gated against + their cited targets with a coarse band (the ASEC undercovers 2022-24 arrivals — the Ukraine parole + pool saturates below its admin count); reconciling levels is the calibration lane''s job. DACA (arrival + cohort among EAD holders) and CUBAN_HAITIAN_ENTRANT (nativity plus post-1980 arrival) keep their statutory + tests; other documented non-citizens stay LEGAL_PERMANENT_RESIDENT, the modal true status.' - stage: hours_worked survey: Census CPS ASEC source: https://www.census.gov/programs-surveys/cps.html diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/spine.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/spine.yaml index 1ffcd46431..0fc8b8d348 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/spine.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/spine.yaml @@ -17,6 +17,7 @@ pipeline_contract: - materialize_multispine_agreement_outputs - stacked_completeness_gate - by_origin_battery + - us_immigration_composition_gate pre_clone_source_operator_order: - derive_us_cps_carried_inputs - with_us_hours_worked_inputs @@ -330,6 +331,58 @@ seed_site_bindings: owners: - kind: source_stage id: snap_abawd_discretionary_exemption +- site: immigration_humanitarian_paroled_one_year_afghanistan_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_paroled_one_year_ukraine_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_paroled_one_year_nicaragua_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_paroled_one_year_venezuela_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_refugee_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_asylee_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_deportation_withheld_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_tps_venezuela_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_tps_el_salvador_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_tps_honduras_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_tps_nicaragua_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_tps_nepal_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_tps_other_designated_assignment + owners: + - kind: source_stage + id: immigration_status - site: immigration_ead_workers_assignment owners: - kind: source_stage diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py index f3cf089d57..63837663a1 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py @@ -210,8 +210,12 @@ from microcosm.build.us_runtime.education_assistance_source import ( ASEC_EDUCATION_ASSISTANCE_ARCHIVES, ASEC_EDUCATION_ASSISTANCE_INCOME_YEARS, + ASEC_EDUCATION_ASSISTANCE_SOURCE_COLUMNS, + ASEC_LABOR_FORCE_STATUS_COLUMN, + ASEC_LABOR_FORCE_STATUS_VALID_CODES, fetch_asec_education_assistance_source, fill_asec_education_assistance_source, + fill_asec_labor_force_status_source, load_asec_education_assistance_sources, ) from microcosm.build.us_runtime.education_inputs import ( @@ -390,16 +394,23 @@ with_us_housing_inputs, ) from microcosm.build.us_runtime.immigration import ( + HUMANITARIAN_STATUS_CATEGORIES, IMMIGRATION_STATUS_VALUES, SSN_CARD_TYPE_VALUES, US_IMMIGRATION_NONCONSTANT_PERSON_COLUMNS, US_IMMIGRATION_OUTPUT_COLUMNS, US_IMMIGRATION_REQUIRED_SOURCE_COLUMNS, US_IMMIGRATION_STAGE_NAME, + HumanitarianDraw, + ImmigrationControls, UndocumentedControls, derive_us_immigration_status_from_manifest, + reconcile_us_immigration_humanitarian_transfer, us_immigration_composition_gate, us_immigration_composition_summary, + us_immigration_controls, + us_immigration_evidence_features, + us_immigration_humanitarian_draw_mask, us_immigration_stage_spec, with_us_immigration_inputs, ) @@ -1212,12 +1223,15 @@ "US_JCT_TAX_EXPENDITURE_TARGET_SPECS", "US_JCT_TAX_EXPENDITURE_TARGET_REFERENCES", "SOI_VARIABLE_MAP", + "HUMANITARIAN_STATUS_CATEGORIES", "IMMIGRATION_STATUS_VALUES", "SSN_CARD_TYPE_VALUES", "US_IMMIGRATION_NONCONSTANT_PERSON_COLUMNS", "US_IMMIGRATION_OUTPUT_COLUMNS", "US_IMMIGRATION_REQUIRED_SOURCE_COLUMNS", "US_IMMIGRATION_STAGE_NAME", + "HumanitarianDraw", + "ImmigrationControls", "UndocumentedControls", "US_HOURS_WORKED_NONCONSTANT_PERSON_COLUMNS", "US_HOURS_WORKED_OUTPUT_COLUMNS", @@ -1550,8 +1564,12 @@ "US_EDUCATION_INPUTS_OWNED_OUTPUT_COLUMNS", "ASEC_EDUCATION_ASSISTANCE_ARCHIVES", "ASEC_EDUCATION_ASSISTANCE_INCOME_YEARS", + "ASEC_EDUCATION_ASSISTANCE_SOURCE_COLUMNS", + "ASEC_LABOR_FORCE_STATUS_COLUMN", + "ASEC_LABOR_FORCE_STATUS_VALID_CODES", "fetch_asec_education_assistance_source", "fill_asec_education_assistance_source", + "fill_asec_labor_force_status_source", "load_asec_education_assistance_sources", "ASEC_PUBLIC_ASSISTANCE_TYPE_AUDIT_PINS", "ASEC_PUBLIC_ASSISTANCE_TYPE_INCOME_YEARS", @@ -1866,8 +1884,12 @@ "us_prior_year_income_summary", "with_us_prior_year_income_inputs", "derive_us_immigration_status_from_manifest", + "reconcile_us_immigration_humanitarian_transfer", "us_immigration_composition_gate", "us_immigration_composition_summary", + "us_immigration_controls", + "us_immigration_evidence_features", + "us_immigration_humanitarian_draw_mask", "us_immigration_stage_spec", "with_us_immigration_inputs", "US_TAKE_UP_SHARE_BAND", @@ -2284,14 +2306,16 @@ def to_manifest(self) -> dict[str, object]: US_IMMIGRATION_STAGE_NAME: DonorSpec( survey="CPS ASEC + published unauthorized-population estimates", source=( - "https://www.pewresearch.org/short-reads/2024/07/22/" - "what-we-know-about-unauthorized-immigrants-living-in-the-us/" + "https://www.pewresearch.org/race-and-ethnicity/2025/08/21/" + "u-s-unauthorized-immigrant-population-reached-a-record-" + "14-million-in-2023/" ), notes=( "SSN card type and immigration status from ASEC citizenship, " - "entry-year, nativity, and program-participation fields via the " - "ASEC-UA residual method (SSRN 4662801), targeted to published " - "undocumented population/worker/student control totals." + "entry-year, nativity, labor-force, and program-participation " + "fields via the ASEC-UA residual method (SSRN 4662801), with " + "Pew's broad 2023 unauthorized population and labor-force " + "definitions applied consistently." ), ), US_HOURS_WORKED_STAGE_NAME: DonorSpec( diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_local_immigration.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_local_immigration.py index 8d586b6427..9efe7844af 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_local_immigration.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_local_immigration.py @@ -24,18 +24,32 @@ fields cover any time last year. Both gaps can only leave the ACS residual (likely undocumented) pool the same size or larger than the ASEC method would find for the same people. +- **Labor-force status is measured.** The kernel's worker test is CPS + ``A_LFSR`` (employed or unemployed) at age 16 or older, the universe of + Pew's unauthorized labor force. ACS ``ESR`` carries the same current + status (:data:`ACS_TO_CPS_INDICATOR_MAPPING`). The donor's delivered + workers are measured the same way when the donor carries ``A_LFSR``; the + support base drops that column after its immigration stage, so such a + donor falls back to the earnings pair, and the receipt names the test it + used (microcosm#1554). - **The ACS rows target the donor residual.** The Pew worker and Higher Ed - student controls are national counts. The donor spine was labelled against - the full controls on its own mass and then pooled at rescaled household - weights, so it already delivers part of each count, and not necessarily - the share its person weight suggests. The stage measures that part on the - frame it sees (donor rows labelled ``NONE`` who work or attend college, on - their pooled weights) and targets the ACS rows at - ``max(0, control - donor delivered)``. Donor plus ACS then pools to the - control whenever the ACS rows have the capacity (microcosm#1052 review, - following the #1060 residual targets). The receipt records, per status, - the control, the donor delivered count, the residual, the ACS capacity and - assigned count, and the pooled total. + student controls are national counts, and so is every cited humanitarian + stock (microcosm#767). The donor spine was labelled against the full + controls on its own mass and then pooled at rescaled household weights, so + it already delivers part of each count, and not necessarily the share its + person weight suggests. The stage measures that part on the frame it sees + and targets the ACS rows at ``max(0, control - donor delivered)``. The + worker and student controls bind Pew's broad unauthorized universe + (residual ``NONE`` rows, DACA, the temporary protections Pew retains and + EAD Cuban/Haitian entrants), so their delivered counts are the donor rows + in that universe who work or attend college; a humanitarian draw's + delivered count is the donor rows already carrying its status inside its + origin, arrival and SSN cohort. Donor plus ACS then pools to each control + whenever the ACS rows have the capacity (microcosm#1052 review, following + the #1060 residual targets). The receipt records, per control and per + humanitarian draw, the national count, the donor delivered count, the + residual, the ACS assigned count and the pooled total, plus the ACS + capacity for the worker and student controls. - **Draws** are the kernel's seeded blake2b draws, keyed on ``acs_2024_1yr:SERIALNO:SPORDER``. ``SPORDER`` alone (the ACS ``source_person_id``) repeats in every household. @@ -84,6 +98,7 @@ import pandas as pd from microcosm.build.gates import GateResult +from microcosm.build.source_runtime import SourceRuntimeError from microcosm.build.us_runtime.acs_pums import ( ACS_2024_1YR_SPINE, ACS_SOURCE_COLUMN_RENAMES, @@ -92,16 +107,23 @@ from microcosm.build.us_runtime.base_pool import spine_column from microcosm.build.us_runtime.immigration import ( _ARRIVAL_YEAR_MIDPOINTS, + _CPS_LABOR_FORCE_STATUS_CODES, _NON_CITIZEN_SHARE_BAND, + _PEW_INCLUDED_HUMANITARIAN_STATUS_VALUES, _SSN_CODE_TO_NAME, IMMIGRATION_STATUS_VALUES, SSN_CARD_TYPE_VALUES, US_IMMIGRATION_OUTPUT_COLUMNS, + HumanitarianDraw, + ImmigrationControls, _asec_arrival_year, _assign_ssn_card_codes, _derive_immigration_status, - _packaged_controls, + _humanitarian_emitted_mask, + _pew_unauthorized_rows, + _source_aware_immigration_profile, us_immigration_composition_gate, + us_immigration_controls, ) from microcosm.frame import Frame from microcosm.frame.units import US_SCHEMA @@ -130,16 +152,24 @@ #: arrival-based status-duration proxy (years since measured arrival), which #: can clear the bar early for people who gained qualified status later. YEARS_SINCE_US_ENTRY_COLUMN = "years_since_us_entry" -#: How the ACS worker and student targets are set (microcosm#1052 review): -#: the national control minus the donor rows' pooled delivered count, floored -#: at zero. A staging receipt without it predates the review and is refused. -ACS_LOCAL_IMMIGRATION_CONTROLS_METHOD = "national_control_minus_donor_pooled_delivered" +#: How the ACS targets are set: each national control minus the donor rows' +#: pooled delivered count, floored at zero. The microcosm#1052 review set the +#: rule for the worker and student controls; v2 (microcosm#767) measures their +#: delivered counts in Pew's broad unauthorized universe and gives every +#: humanitarian stock draw its own residual. A staging receipt with another +#: method predates one of the two and is refused. +ACS_LOCAL_IMMIGRATION_CONTROLS_METHOD = ( + "national_control_minus_donor_pooled_delivered_v2" +) #: Statuses the engine's five-year bar applies to: policyengine-us 2.2.1 #: ``gov.hhs.medicaid.eligibility.eligible_immigration_statuses`` (the #: 2018-01-01 list) less ``CITIZEN`` and less #: ``bar_exempt_immigration_statuses`` (8 U.S.C. 1613(b)). In 2.2.1 Medicaid #: and Washington TANF read the clock against this bar, and Washington RCA -#: reads it as a closing window; the federal SNAP status test does not. Of these, the stage assigns only ``LEGAL_PERMANENT_RESIDENT``. +#: reads it as a closing window; the federal SNAP status test does not. Of +#: these, the stage assigns ``LEGAL_PERMANENT_RESIDENT`` and, through the +#: humanitarian parole draw, ``PAROLED_ONE_YEAR`` (microcosm#767); the kernel +#: never emits ``CONDITIONAL_ENTRANT``. FIVE_YEAR_BAR_STATUSES: tuple[str, ...] = ( "LEGAL_PERMANENT_RESIDENT", "CONDITIONAL_ENTRANT", @@ -195,15 +225,27 @@ #: Donor person columns the entry clock reads: the ASEC citizenship and #: entry-year codes. _ASEC_ENTRY_SOURCE_COLUMNS: tuple[str, ...] = ("PRCITSHP", "PEINUSYR") -#: The two counts the kernel forces: residual-pool (``NONE``) non-citizens -#: who work, and who attend college. +#: The two counts the kernel forces: non-citizens in Pew's broad +#: unauthorized universe (residual ``NONE``, DACA, the temporary protections +#: Pew retains and EAD Cuban/Haitian entrants) who work, and who attend +#: college. _CONTROLLED_STATUSES: tuple[str, ...] = ( "undocumented_workers", "undocumented_students", ) -#: Donor columns for the kernel's worker test (positive wages or -#: self-employment), most faithful first: the raw ASEC fields the donor's own -#: immigration stage read, then harmonized inputs with the same sign. +#: The kernel's worker test: ASEC ``A_LFSR`` in the labor force at 16 or +#: older. The ASEC-by-PUF support base attaches ``A_LFSR`` only while its own +#: immigration stage runs and drops it afterwards +#: (``tools/build_us_puf_support_base.py``), so a donor built there carries +#: none and falls back to :data:`_DONOR_WORKER_COLUMNS`. +_DONOR_LABOR_FORCE_COLUMN = "A_LFSR" +_LABOR_FORCE_MIN_AGE = 16 +#: Earnings-proxy donor columns for the worker test (positive wages or +#: self-employment), used when the donor carries no ``A_LFSR``: the raw ASEC +#: fields first, then harmonized inputs with the same sign. Prior-year +#: earnings and current labor-force status differ at the margins (a worker +#: laid off this year, a new entrant without last year's earnings), so the +#: worker residual is then approximate; the receipt names the test used. _DONOR_WORKER_COLUMNS: tuple[tuple[str, str], ...] = ( ("WSAL_VAL", "SEMP_VAL"), ("employment_income_before_lsr", "self_employment_income_before_lsr"), @@ -228,6 +270,13 @@ _ACTIVE_DUTY = (1, 2) #: ACS COW government codes -> CPS PEIO1COW government codes. _COW_TO_PEIO1COW: Mapping[int, int] = MappingProxyType({5: 1, 4: 2, 3: 3}) +#: ACS ESR -> CPS A_LFSR. ESR 1/2 (civilian employed, at work / with a job +#: but not at work) and 3 (unemployed) keep their codes; 4/5 (Armed Forces) +#: are CPS code 0, outside the civilian universe; 6 (not in the labor force) +#: is CPS 7. A blank ESR (under 16) is CPS 0. +_ESR_TO_A_LFSR: Mapping[int, int] = MappingProxyType( + {1: 1, 2: 2, 3: 3, 4: 0, 5: 0, 6: 7} +) _CPS_ARMED_FORCES_OCCUPATION = 11 _CPS_COLLEGE = 2 _CPS_YES, _CPS_NO = 1, 2 @@ -248,6 +297,11 @@ "RELSHIPP spouse pairing)" ), "A_HSCOL": "2 (college) when SCHG is 15 or 16", + "A_LFSR": ( + "ESR 1/2/3 (employed at work, with a job not at work, unemployed) " + "keep their codes; ESR 4/5 (Armed Forces) and a blank ESR (under " + "16) are 0; ESR 6 (not in the labor force) is 7" + ), "WSAL_VAL": "WAGP (sign only; blank = under 15, no wages)", "SEMP_VAL": "SEMP (sign only; blank = under 15)", "MCARE": "1 when HINS3 == 1 (Medicare at interview)", @@ -281,10 +335,13 @@ ) _DRAW_KEY_FORMAT = f"{ACS_2024_1YR_SPINE}:SERIALNO:SPORDER" _CONTROLS_DECISION = ( - "each national control minus the donor rows' pooled delivered count of " - "that status (donor rows labelled NONE who work or attend college, on the " - "frame's pooled weights), floored at 0; donor plus ACS pools to the " - "control whenever the ACS rows have the capacity" + "each national control minus the donor rows' pooled delivered count, " + "floored at 0: for the worker and student controls, donor rows in Pew's " + "broad unauthorized universe who work or attend college; for each " + "humanitarian stock draw, donor rows carrying its status inside its " + "origin, arrival and SSN cohort; all on the frame's pooled weights. " + "Donor plus ACS pools to each control whenever the ACS rows have the " + "capacity" ) _ENTRY_CLOCK = { "label": "arrival-based status-duration proxy", @@ -326,12 +383,13 @@ @dataclass(frozen=True) class _ResidualTargets: - """The two counts the kernel forces, as ACS residual targets. + """The worker and student counts the kernel forces, as ACS residuals. :class:`~microcosm.build.us_runtime.immigration.UndocumentedControls` holds positive national counts; a residual can be zero (the donor already delivers the control), which the kernel reads as "spill every ACS - candidate". The kernel reads only these two attributes. + candidate". The kernel's SSN assignment reads only these two attributes + of ``ImmigrationControls.undocumented``. """ workers: float @@ -356,6 +414,29 @@ def _cps_yes_no(values: pd.Series) -> np.ndarray: return np.where(_is_any(values, _YES), _CPS_YES, _CPS_NO).astype(np.int64) +def _cps_labor_force_status(esr: pd.Series) -> np.ndarray: + """CPS ``A_LFSR`` from ACS ``ESR``; a blank (under 16) is code 0. + + Raises: + ValueError: If any non-blank ``ESR`` (an infinity included) is outside + the ACS codebook. + """ + + codes = _numeric(esr) + outside = ~np.isnan(codes) & ~np.isin( + codes, np.asarray(list(_ESR_TO_A_LFSR), dtype=np.float64) + ) + if outside.any(): + raise ValueError( + f"ACS ESR must be one of {sorted(_ESR_TO_A_LFSR)} or blank; " + f"{int(outside.sum())} row(s) are outside the domain." + ) + status = np.zeros(len(codes), dtype=np.int64) + for esr_code, cps_code in _ESR_TO_A_LFSR.items(): + status[codes == esr_code] = cps_code + return status + + def _require_codes(values: pd.Series, *, name: str, where: str) -> np.ndarray: codes = _numeric(values) bad = ~np.isin(codes, np.asarray(_CITIZENSHIP_DOMAIN, dtype=np.float64)) @@ -414,6 +495,7 @@ def _acs_cps_view( "A_HSCOL": np.where( _is_any(person["SCHG"], _COLLEGE_GRADES), _CPS_COLLEGE, 0 ).astype(np.int64), + "A_LFSR": _cps_labor_force_status(person["ESR"]), "WSAL_VAL": np.nan_to_num(_numeric(person["WAGP"]), nan=0.0), "SEMP_VAL": np.nan_to_num(_numeric(person["SEMP"]), nan=0.0), "MCARE": _cps_yes_no(person["HINS3"]), @@ -472,25 +554,44 @@ def _donor_status_masks( ) -> tuple[np.ndarray, np.ndarray, dict[str, object]]: """The kernel's worker and student tests on donor rows, and their columns. + The worker test is the kernel's own (``A_LFSR`` in the labor force at 16 + or older) when the donor carries ``A_LFSR``, else the earnings proxy of + :data:`_DONOR_WORKER_COLUMNS`. + Raises: - ValueError: If no worker or no student column of - :data:`_DONOR_WORKER_COLUMNS` / :data:`_DONOR_STUDENT_COLUMNS` is - present: the donor's delivered counts could not be measured. + ValueError: If neither ``A_LFSR`` nor a worker pair of + :data:`_DONOR_WORKER_COLUMNS`, or no student column of + :data:`_DONOR_STUDENT_COLUMNS`, is present: the donor's delivered + counts could not be measured. """ + labor_force = _DONOR_LABOR_FORCE_COLUMN in person worker_columns = next( (pair for pair in _DONOR_WORKER_COLUMNS if all(c in person for c in pair)), None, ) student_column = next((c for c in _DONOR_STUDENT_COLUMNS if c in person), None) - if worker_columns is None or student_column is None: + if (not labor_force and worker_columns is None) or student_column is None: raise ValueError( - "The donor spine needs a worker pair from " - f"{[list(pair) for pair in _DONOR_WORKER_COLUMNS]} and a student " - f"column from {list(_DONOR_STUDENT_COLUMNS)} to measure the " + f"The donor spine needs {_DONOR_LABOR_FORCE_COLUMN} or a worker pair " + f"from {[list(pair) for pair in _DONOR_WORKER_COLUMNS]} and a " + f"student column from {list(_DONOR_STUDENT_COLUMNS)} to measure the " "undocumented worker and student counts it already delivers." ) - worker = _positive(person[worker_columns[0]]) | _positive(person[worker_columns[1]]) + if labor_force: + worker = _is_any( + person[_DONOR_LABOR_FORCE_COLUMN], _CPS_LABOR_FORCE_STATUS_CODES + ) & (_numeric(person[_AGE]) >= _LABOR_FORCE_MIN_AGE) + worker_test = ( + f"{_DONOR_LABOR_FORCE_COLUMN} in " + f"{list(_CPS_LABOR_FORCE_STATUS_CODES)} and {_AGE} >= " + f"{_LABOR_FORCE_MIN_AGE}" + ) + else: + worker = _positive(person[worker_columns[0]]) | _positive( + person[worker_columns[1]] + ) + worker_test = f"{worker_columns[0]} > 0 or {worker_columns[1]} > 0" if student_column == "A_HSCOL": student = _is_any(person["A_HSCOL"], (_CPS_COLLEGE,)) student_test = "A_HSCOL == 2" @@ -502,14 +603,7 @@ def _donor_status_masks( f"{student_column} (full time only: narrower than A_HSCOL == 2, so " "the donor student count can only be understated)" ) - return ( - worker, - student, - { - "worker": f"{worker_columns[0]} > 0 or {worker_columns[1]} > 0", - "student": student_test, - }, - ) + return worker, student, {"worker": worker_test, "student": student_test} def _donor_delivered( @@ -518,24 +612,86 @@ def _donor_delivered( """Pooled weight of donor rows in each controlled status, and the columns. The donor rows' stored labels on the frame's (pooled) weights: what the - donor already delivers toward each national control. + donor already delivers toward each national control. The controls bind + Pew's broad unauthorized universe, so a donor DACA, TPS or parole worker + counts as well as a ``NONE`` one. """ if not donor.any(): return dict.fromkeys(_CONTROLLED_STATUSES, 0.0), None rows = person.loc[donor] worker, student, columns = _donor_status_masks(rows) - undocumented = rows["ssn_card_type"].astype(object).eq("NONE").to_numpy() + unauthorized = _pew_unauthorized_rows(rows) donor_weights = weights[donor] return ( { - "undocumented_workers": float(donor_weights[undocumented & worker].sum()), - "undocumented_students": float(donor_weights[undocumented & student].sum()), + "undocumented_workers": float(donor_weights[unauthorized & worker].sum()), + "undocumented_students": float(donor_weights[unauthorized & student].sum()), }, columns, ) +def _evidence_profile(person: pd.DataFrame, *, time_period: int, where: str): + """The kernel's source-aware evidence for ``person``, refusals as ValueError.""" + + try: + return _source_aware_immigration_profile(person, time_period=time_period) + except SourceRuntimeError as exc: + raise ValueError(f"{where} immigration evidence is unusable: {exc}") from exc + + +def _humanitarian_delivered( + person: pd.DataFrame, + weights: np.ndarray, + rows: np.ndarray, + draws: Sequence[HumanitarianDraw], + *, + time_period: int, + where: str, +) -> dict[str, float]: + """Pooled weight of ``rows`` emitted as each draw within its cohort. + + The same contract the whole-file composition gate grades: the draw's + status on a row whose origin, arrival and SSN evidence support it. + """ + + if not draws or not rows.any(): + return {draw.label: 0.0 for draw in draws} + subset = person.loc[rows] + profile = _evidence_profile(subset, time_period=time_period, where=where) + selected = weights[rows] + return { + draw.label: float( + selected[ + _humanitarian_emitted_mask(subset, draw=draw, profile=profile) + ].sum() + ) + for draw in draws + } + + +def _humanitarian_reconciliation( + draw: HumanitarianDraw, donor: float, assigned: float +) -> dict[str, object]: + """One humanitarian draw: target, donor, residual, ACS and pooled mass.""" + + pooled = donor + assigned + return { + "category": draw.category, + "origin": draw.origin, + "status": draw.status, + "target": draw.target, + "donor_delivered": donor, + "residual": max(0.0, draw.target - donor), + "acs_assigned": assigned, + "pooled_total": pooled, + "pooled_relative": (pooled / draw.target) if draw.target > 0 else None, + "donor_excess": max(0.0, donor - draw.target), + "source": draw.source, + } + + def _status_reconciliation( control: float, donor: float, capacity: float, assigned: float ) -> dict[str, float]: @@ -627,12 +783,14 @@ def _adjustment_lag_sensitivity( def acs_local_immigration_receipt_failures( receipt: Mapping[str, object], ) -> list[str]: - """Why a staging receipt's worker and student targets are not the residual. + """Why a staging receipt's ACS targets are not the donor residuals. Empty when the receipt carries - :data:`ACS_LOCAL_IMMIGRATION_CONTROLS_METHOD` and, for each controlled - status, ``residual == max(0, control - donor_delivered)`` and - ``pooled_total == donor_delivered + acs_assigned``. + :data:`ACS_LOCAL_IMMIGRATION_CONTROLS_METHOD`; for each controlled status, + ``residual == max(0, control - donor_delivered)`` and + ``pooled_total == donor_delivered + acs_assigned``; and the same two + identities (``target`` for ``control``) for exactly the packaged + humanitarian draws. """ controls = receipt.get("controls") if isinstance(receipt, Mapping) else None @@ -642,7 +800,7 @@ def acs_local_immigration_receipt_failures( ): return [ f"controls.method is not {ACS_LOCAL_IMMIGRATION_CONTROLS_METHOD!r} " - "(a staging run from before the donor-residual targets)" + "(a staging run from before the current donor-residual targets)" ] statuses = controls.get("statuses") if not isinstance(statuses, Mapping): @@ -678,6 +836,46 @@ def acs_local_immigration_receipt_failures( f"{status}: pooled total {pooled:,.1f} is not donor delivered + " f"ACS assigned = {donor + assigned:,.1f}" ) + humanitarian = controls.get("humanitarian") + if not isinstance(humanitarian, Mapping): + return [*failures, "controls.humanitarian is missing"] + expected_labels = {draw.label for draw in us_immigration_controls().humanitarian} + if set(humanitarian) != expected_labels: + failures.append( + "controls.humanitarian draws differ from the packaged draws: " + f"missing {sorted(expected_labels - set(humanitarian))}, " + f"unexpected {sorted(set(humanitarian) - expected_labels)}" + ) + for label in sorted(set(humanitarian) & expected_labels): + entry = humanitarian[label] + try: + target, donor, residual, assigned, pooled = ( + float(entry[key]) + for key in ( + "target", + "donor_delivered", + "residual", + "acs_assigned", + "pooled_total", + ) + ) + except (KeyError, TypeError, ValueError): + failures.append( + f"controls.humanitarian.{label} lacks a numeric target, " + "donor_delivered, residual, acs_assigned or pooled_total" + ) + continue + expected = max(0.0, target - donor) + if not np.isclose(residual, expected, rtol=1e-9, atol=1e-6): + failures.append( + f"{label}: residual {residual:,.1f} is not max(0, target - " + f"donor delivered) = {expected:,.1f}" + ) + if not np.isclose(pooled, donor + assigned, rtol=1e-9, atol=1e-6): + failures.append( + f"{label}: pooled total {pooled:,.1f} is not donor delivered + " + f"ACS assigned = {donor + assigned:,.1f}" + ) return failures @@ -790,8 +988,9 @@ def require_acs_local_immigration_donor(base: Frame, *, time_period: int) -> Non The stage runs after hours of transfer fitting; its donor-side inputs are checked here first: a complete immigration surface, the ASEC citizenship - and entry-year codes, age, and the worker and student columns that - measure the controlled counts the donor already delivers. + and entry-year codes, age, the birth-country evidence the humanitarian + cohorts read, and the worker and student columns that measure the + controlled counts the donor already delivers. """ person = base.table("person") @@ -818,6 +1017,7 @@ def require_acs_local_immigration_donor(base: Frame, *, time_period: int) -> Non _years_since_entry( citizenship, entry_year, _numeric(person[_AGE]), time_period=time_period ) + _evidence_profile(person, time_period=time_period, where="The donor's") _donor_status_masks(person) @@ -830,11 +1030,11 @@ def with_acs_local_immigration_inputs( frame: The local lane's multispine US frame. Its person table must carry complete origin tags, the donor's two immigration columns, the raw ACS fields of :data:`ACS_IMMIGRATION_SOURCE_COLUMNS` on ACS - rows, the ASEC ``PRCITSHP``/``PEINUSYR`` codes and a worker and a - student column (:data:`_DONOR_WORKER_COLUMNS`, + rows, the ASEC ``PRCITSHP``/``PENATVTY``/``PEINUSYR`` codes and a + worker and a student column (:data:`_DONOR_WORKER_COLUMNS`, :data:`_DONOR_STUDENT_COLUMNS`) on donor rows, and ``age``; the household table carries ``SERIALNO``. - seed: The build seed for the EAD draws. + seed: The build seed for the humanitarian and EAD draws. time_period: The dataset year, against which entry years are read. Returns: @@ -886,7 +1086,7 @@ def with_acs_local_immigration_inputs( weights = np.asarray(frame.resolve_weights("person").values, dtype=np.float64) acs_person = person.loc[acs] view, entry_year = _acs_cps_view(acs_person, time_period=time_period) - national = _packaged_controls() + national = us_immigration_controls() total_weight = float(weights.sum()) acs_weight = float(weights[acs].sum()) if not (total_weight > 0 and acs_weight > 0): @@ -894,24 +1094,53 @@ def with_acs_local_immigration_inputs( share = acs_weight / total_weight # The donor was labelled against the full controls on its own mass and # pooled at rescaled weights; measure what it delivers on this frame and - # target the ACS rows at the nonnegative residual (#1052 review). + # target the ACS rows at the nonnegative residual (#1052 review), for the + # worker and student controls and for every humanitarian stock (#767). donor_delivered, donor_columns = _donor_delivered(person, weights, ~acs) - targets = _ResidualTargets( - workers=max(0.0, national.workers - donor_delivered["undocumented_workers"]), - students=max(0.0, national.students - donor_delivered["undocumented_students"]), - ) - codes = _assign_ssn_card_codes( - view, - weights[acs], - seed=int(seed), - controls=targets, - arrival_year=entry_year, + donor_humanitarian = _humanitarian_delivered( + person, + weights, + ~acs, + national.humanitarian, time_period=time_period, - draw_keys=_acs_draw_keys(frame.table("household"), acs_person), + where="The donor's", ) + undocumented = national.undocumented + targets = ImmigrationControls( + undocumented=_ResidualTargets( + workers=max( + 0.0, undocumented.workers - donor_delivered["undocumented_workers"] + ), + students=max( + 0.0, undocumented.students - donor_delivered["undocumented_students"] + ), + ), + humanitarian=tuple( + HumanitarianDraw( + category=draw.category, + origin=draw.origin, + status=draw.status, + target=max(0.0, draw.target - donor_humanitarian[draw.label]), + source=draw.source, + ) + for draw in national.humanitarian + ), + ) + try: + codes, marks = _assign_ssn_card_codes( + view, + weights[acs], + seed=int(seed), + controls=targets, + arrival_year=entry_year, + time_period=time_period, + draw_keys=_acs_draw_keys(frame.table("household"), acs_person), + ) + except SourceRuntimeError as exc: + raise ValueError(f"ACS local immigration evidence is unusable: {exc}") from exc derived_status = np.full(len(person), None, dtype=object) derived_status[acs] = _derive_immigration_status( - view, codes, time_period=time_period, arrival_year=entry_year + view, codes, marks, time_period=time_period, arrival_year=entry_year ) derived_ssn = np.full(len(person), None, dtype=object) derived_ssn[acs] = np.asarray( @@ -972,23 +1201,42 @@ def with_acs_local_immigration_inputs( acs_weights = weights[acs] noncitizen = view["PRCITSHP"].to_numpy() == 5 acs_status = { - "undocumented_workers": (view["WSAL_VAL"].to_numpy() > 0) - | (view["SEMP_VAL"].to_numpy() > 0), + "undocumented_workers": (view["A_AGE"].to_numpy() >= _LABOR_FORCE_MIN_AGE) + & np.isin(view["A_LFSR"].to_numpy(), _CPS_LABOR_FORCE_STATUS_CODES), "undocumented_students": view["A_HSCOL"].to_numpy() == _CPS_COLLEGE, } - # Capacity: ACS residual-pool non-citizens (NONE, or EAD by a spill) - # before the spills; assigned: final NONE labels, stored ones included. - residual_pool = noncitizen & np.isin(codes, (0, 2)) - acs_undocumented = final.loc[acs, "ssn_card_type"].astype(object).eq("NONE") + # Capacity: ACS non-citizens inside Pew's broad universe before the EAD + # spills (the residual pool, NONE or EAD, and the Pew-retained + # humanitarian marks); assigned: final labels inside that universe, + # stored ones included. + in_universe_before_spill = noncitizen & ( + np.isin(codes, (0, 2)) + | np.isin(marks, _PEW_INCLUDED_HUMANITARIAN_STATUS_VALUES) + ) + acs_unauthorized = _pew_unauthorized_rows(final.loc[acs]) statuses = { status: _status_reconciliation( - getattr(national, status.removeprefix("undocumented_")), + getattr(undocumented, status.removeprefix("undocumented_")), donor_delivered[status], - float(acs_weights[residual_pool & in_status].sum()), - float(acs_weights[acs_undocumented.to_numpy() & in_status].sum()), + float(acs_weights[in_universe_before_spill & in_status].sum()), + float(acs_weights[acs_unauthorized & in_status].sum()), ) for status, in_status in acs_status.items() } + acs_humanitarian = _humanitarian_delivered( + final, + weights, + acs, + national.humanitarian, + time_period=time_period, + where="The ACS rows'", + ) + humanitarian = { + draw.label: _humanitarian_reconciliation( + draw, donor_humanitarian[draw.label], acs_humanitarian[draw.label] + ) + for draw in national.humanitarian + } receipt: dict[str, object] = { "issue": ACS_LOCAL_IMMIGRATION_ISSUE, "seed": int(seed), @@ -1009,12 +1257,13 @@ def with_acs_local_immigration_inputs( "donor_status_columns": donor_columns, "acs_person_weight_share": share, "national": { - "undocumented_workers": national.workers, - "undocumented_students": national.students, - "undocumented_population_anchor": national.population_anchor, + "undocumented_workers": undocumented.workers, + "undocumented_students": undocumented.students, + "undocumented_population_anchor": undocumented.population_anchor, }, "statuses": statuses, - "sources": dict(national.sources), + "humanitarian": humanitarian, + "sources": dict(undocumented.sources), }, "indicator_mapping": dict(ACS_TO_CPS_INDICATOR_MAPPING), "unmapped_indicators": dict(ACS_UNMAPPED_INDICATORS), @@ -1158,7 +1407,12 @@ def _measured_citizenship_failures( return failures -def acs_local_immigration_signal_gate(frame: Frame) -> GateResult: +def acs_local_immigration_signal_gate( + frame: Frame, + *, + time_period: int, + controls: ImmigrationControls | None = None, +) -> GateResult: """Require complete, measured, plausible immigration inputs per origin. For each spine, fails when a label or the entry clock is missing or has @@ -1170,7 +1424,9 @@ def acs_local_immigration_signal_gate(frame: Frame) -> GateResult: stage's band. Donor-spine composition is reported but not graded; the donor release graded it. The whole file must pass :func:`~microcosm.build.us_runtime.immigration.us_immigration_composition_gate` - (agreement, non-citizen band and the undocumented anchor band). + at ``time_period`` against ``controls`` (default: the packaged controls): + label agreement, the source-aware cohort contracts, the non-citizen band, + the undocumented anchor band and every humanitarian stock band. """ person = frame.table("person") @@ -1226,7 +1482,11 @@ def acs_local_immigration_signal_gate(frame: Frame) -> GateResult: for column in US_IMMIGRATION_OUTPUT_COLUMNS ) if complete: - whole = us_immigration_composition_gate(frame) + whole = us_immigration_composition_gate( + frame, + time_period=int(time_period), + controls=us_immigration_controls() if controls is None else controls, + ) details["file"] = { "passed": whole.passed, "failures": list(whole.failures), diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_pums.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_pums.py index 266670b0fe..9d295e7a84 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_pums.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_pums.py @@ -101,7 +101,9 @@ # Preserve source hours and their universe/allocation evidence when supplied, # the citizenship, entry-year, birthplace, coverage, class-of-worker, school # and military-service fields the ACS local lane's immigration stage reads -# (microcosm#1020, acs_local_immigration), the six disability-difficulty items +# (microcosm#1020, acs_local_immigration; the ACS transfer's immigration +# evidence reads the same CIT, POBP and YOEP and refuses a row without a +# citizenship source, microcosm#767), the six disability-difficulty items # the local lane maps natively (microcosm#1021, acs_local_work_disability), # all other income (OIP), the local income pass's child-support predictor # (microcosm#1022, acs_local_income), and the public assistance income the diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_transfer.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_transfer.py index 0706cf02ec..d7ffca31e9 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_transfer.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_transfer.py @@ -31,9 +31,14 @@ from microcosm.build.gates import FitWeightRecord from microcosm.build.serialization_dtypes import canonicalize_frame_string_dtypes +from microcosm.build.us_runtime.acs_pums import ACS_2024_1YR_VINTAGE from microcosm.build.us_runtime.capital_gain_distributions import ( capital_gain_distribution_shares_asset_identity, ) +from microcosm.build.us_runtime.immigration import ( + reconcile_us_immigration_humanitarian_transfer, + us_immigration_evidence_features, +) from microcosm.build.us_runtime.puf_support import ( PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS, PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS, @@ -209,6 +214,32 @@ def _pregnancy_structural_policy_identity(*, enabled: bool) -> dict[str, object] "immigration_status_str", ) _IMMIGRATION_STATUS_MODEL_TARGET = "__acs_transfer_immigration_status_pair" +_IMMIGRATION_CITIZENSHIP_FEATURE = "__acs_transfer_is_us_citizen" +_IMMIGRATION_ORIGIN_FEATURE = "__acs_transfer_birth_country_code" +_IMMIGRATION_ARRIVAL_FEATURE = "__acs_transfer_arrival_year" +_IMMIGRATION_REQUIRED_FEATURES = ( + _IMMIGRATION_CITIZENSHIP_FEATURE, + _IMMIGRATION_ORIGIN_FEATURE, + _IMMIGRATION_ARRIVAL_FEATURE, +) +_HUMANITARIAN_IMMIGRATION_STATUSES = frozenset( + { + "PAROLED_ONE_YEAR", + "REFUGEE", + "ASYLEE", + "DEPORTATION_WITHHELD", + "TPS", + } +) +_EVIDENCE_DERIVED_IMMIGRATION_STATUSES = frozenset( + { + "CUBAN_HAITIAN_ENTRANT", + "DACA", + } +) +_CONSTRAINED_IMMIGRATION_STATUSES = ( + _HUMANITARIAN_IMMIGRATION_STATUSES | _EVIDENCE_DERIVED_IMMIGRATION_STATUSES +) # RNTP/GRNTP are not pre-subsidy rent, so this leaf remains donor-transferable. _ADDITIONAL_PERSON_TRANSFER_TARGETS = ("pre_subsidy_rent",) @@ -432,8 +463,9 @@ def acs_transfer_execution_contract_identity( adult_care_enabled = _ADULT_CARE_EXPENSE in requested_targets passive_enabled = _PASSIVE_PARTNERSHIP_S_CORP in requested_targets payload: dict[str, object] = { - "schema_version": 2, + "schema_version": 3, "person_required_predictors": list(ACS_PERSON_TRANSFER_PREDICTORS), + "immigration_required_predictors": list(_IMMIGRATION_REQUIRED_FEATURES), "person_optional_predictors": list(ACS_OPTIONAL_PERSON_TRANSFER_PREDICTORS), "group_required_predictors": list(ACS_GROUP_TRANSFER_PREDICTORS), "group_optional_names": dict(sorted(_GROUP_OPTIONAL_NAMES.items())), @@ -458,6 +490,9 @@ def acs_transfer_execution_contract_identity( "tenure_codes": dict(sorted(_TENURE_CODES.items())), "immigration_status_targets": list(_IMMIGRATION_STATUS_TARGETS), "immigration_status_model_target": _IMMIGRATION_STATUS_MODEL_TARGET, + "immigration_baseline_excluded_statuses": sorted( + _CONSTRAINED_IMMIGRATION_STATUSES + ), "discrete_numeric_targets": sorted(_DISCRETE_NUMERIC_TARGETS), "structural_target_policies": { _PREGNANCY_TARGET: _pregnancy_structural_policy_identity( @@ -485,6 +520,14 @@ def acs_transfer_execution_contract_identity( "tax_unit_link": _ADULT_CARE_UNIT, "mutable_rows": "newly_imputed_expense_cells_only", }, + "humanitarian_immigration": { + "enabled": set(_IMMIGRATION_STATUS_TARGETS).issubset(requested_targets), + "targets": list(_IMMIGRATION_STATUS_TARGETS), + "mutable_rows": "newly_imputed_paired_cells_only", + "target_basis": "manifest_residual_after_immutable_asec_mass", + "selection": "stable_person_hash_in_manifest_draw_order", + "candidate_shortfall": "take_every_candidate_error_below_composition_band_floor", + }, "passive_partnership_s_corp": { "enabled": passive_enabled, "subset": _PASSIVE_PARTNERSHIP_S_CORP, @@ -658,7 +701,7 @@ class AcsImputedInput: derivation: str | None = None #: Post-fit structural reconciliation counts, when the column's surface #: was adjusted to a statute contract after prediction. - reconciliation: Mapping[str, int] | None = None + reconciliation: Mapping[str, object] | None = None #: Hard-domain and source-person fanout proof for structurally constrained #: transfer targets. This is receipt-only and never enters the frame. structural_receipt: Mapping[str, object] | None = None @@ -1108,6 +1151,37 @@ def acs_transfer_donor_requirements( if all(component in person.columns for component in components): required[donor.schema.person_entity].update(components) + person_targets = { + target + for targets in target_families.get(donor.schema.person_entity, {}).values() + for target in targets + } + if set(_IMMIGRATION_STATUS_TARGETS).issubset(person_targets): + evidence_triplets = ( + ("PRCITSHP", "PENATVTY", "PEINUSYR"), + ("CIT", "POBP", "YOEP"), + ) + active_triplets: list[tuple[str, str, str]] = [] + for triplet in evidence_triplets: + citizenship = triplet[0] + if citizenship not in person or not person[citizenship].notna().any(): + continue + missing = sorted(set(triplet) - set(person.columns)) + if missing: + raise ValueError( + "paired immigration targets have an incomplete donor " + f"evidence triplet {triplet!r}; missing={missing}." + ) + active_triplets.append(triplet) + if not active_triplets: + raise ValueError( + "paired immigration targets require a live ASEC " + "PRCITSHP/PENATVTY/PEINUSYR or ACS CIT/POBP/YOEP donor " + "evidence triplet." + ) + for triplet in active_triplets: + required[donor.schema.person_entity].update(triplet) + housing = bool(_HOUSING_TRANSFER_TARGETS.intersection(target_names)) head_source = next( ( @@ -1897,10 +1971,12 @@ def transfer_acs_inputs( _apply_post_transfer_structure( output_tables, provenance, + recipient=recipient, imputed_masks=imputed_masks, donor_spine=donor_spine, resolved_channel=resolved_channel, execution_contract=resolved_execution_contract, + seed=seed, ) tables: dict[str, pd.DataFrame] = dict(output_tables) @@ -1934,10 +2010,12 @@ def _apply_post_transfer_structure( output_tables: dict[str, pd.DataFrame], provenance: list[AcsImputedInput], *, + recipient: Frame, imputed_masks: Mapping[tuple[str, str], np.ndarray], donor_spine: str, resolved_channel: str | None, execution_contract: Mapping[str, object], + seed: int, ) -> None: """Apply the deterministic post-fit steps the base's construction implies. @@ -1948,6 +2026,8 @@ def _apply_post_transfer_structure( capital-gain parents at the packaged share with route exclusivity. - The adult-care expense surface is reconciled to the statute structure (qualifying carriers only, at most one per tax unit). + - The paired immigration baseline is reconciled to manifest humanitarian + residual targets after the immutable ASEC contribution. - The passive partnership/S-corp subset is zeroed where it has no line-32 total under it (microcosm#1239). - Self-employment components drawn onto rows whose self-employment total @@ -1959,6 +2039,43 @@ def _apply_post_transfer_structure( if person is None: return + immigration_masks = [ + imputed_masks.get(("person", target)) for target in _IMMIGRATION_STATUS_TARGETS + ] + if all(mask is not None for mask in immigration_masks): + assert immigration_masks[0] is not None + assert immigration_masks[1] is not None + if not np.array_equal(immigration_masks[0], immigration_masks[1]): + raise ValueError( + "ACS immigration transfer must impute its paired target cells " + "on exactly the same recipient rows." + ) + mutable_immigration = immigration_masks[0] + immigration_provenance = [ + (index, item) + for index, item in enumerate(provenance) + if item.entity == "person" and item.column in _IMMIGRATION_STATUS_TARGETS + ] + source_operator_family = any( + item.family.split("__batch_", 1)[0] == "source_operator_immigration" + for _, item in immigration_provenance + ) + if mutable_immigration.any() and source_operator_family: + reconciled, receipt = reconcile_us_immigration_humanitarian_transfer( + person, + weights=np.asarray( + recipient.resolve_weights("person").values, + dtype=np.float64, + ), + mutable_rows=mutable_immigration, + seed=seed, + time_period=ACS_2024_1YR_VINTAGE, + ) + for target in _IMMIGRATION_STATUS_TARGETS: + person[target] = reconciled[target] + for index, item in immigration_provenance: + provenance[index] = replace(item, reconciliation=receipt) + post_transfer_contract = execution_contract["post_transfer_structure"] assert isinstance(post_transfer_contract, Mapping) schedule_d_contract = post_transfer_contract[ @@ -2918,6 +3035,38 @@ def _transfer_feature_surface( else: surface = _group_feature_surface(donor, recipient, entity=entity) + if set(_IMMIGRATION_STATUS_TARGETS).issubset(targets): + if entity != donor.schema.person_entity: + raise ValueError( + "The paired immigration targets must be transferred on person." + ) + donor_evidence = us_immigration_evidence_features( + donor, + time_period=ACS_2024_1YR_VINTAGE, + ).rename( + columns={ + "is_us_citizen": _IMMIGRATION_CITIZENSHIP_FEATURE, + "birth_country_code": _IMMIGRATION_ORIGIN_FEATURE, + "arrival_year": _IMMIGRATION_ARRIVAL_FEATURE, + } + ) + recipient_evidence = us_immigration_evidence_features( + recipient, + time_period=ACS_2024_1YR_VINTAGE, + ).rename( + columns={ + "is_us_citizen": _IMMIGRATION_CITIZENSHIP_FEATURE, + "birth_country_code": _IMMIGRATION_ORIGIN_FEATURE, + "arrival_year": _IMMIGRATION_ARRIVAL_FEATURE, + } + ) + surface = _FeatureSurface( + donor=pd.concat([surface.donor, donor_evidence], axis=1), + recipient=pd.concat([surface.recipient, recipient_evidence], axis=1), + required=(*surface.required, *_IMMIGRATION_REQUIRED_FEATURES), + optional=surface.optional, + ) + housing = bool(_HOUSING_TRANSFER_TARGETS.intersection(targets)) if not housing: return surface @@ -3961,7 +4110,8 @@ def _target_encodings( result: dict[str, _TargetEncoding] = {} if immigration: pairs = list( - zip( + _baseline_immigration_pair(ssn, status) + for ssn, status in zip( table[_IMMIGRATION_STATUS_TARGETS[0]].to_numpy(dtype=object), table[_IMMIGRATION_STATUS_TARGETS[1]].to_numpy(dtype=object), strict=True, @@ -4008,6 +4158,18 @@ def _target_encodings( return result +def _baseline_immigration_pair(ssn: object, status: object) -> tuple[object, object]: + """Remove evidence-constrained labels from the unconstrained codec.""" + + if status not in _CONSTRAINED_IMMIGRATION_STATUSES: + return ssn, status + if ssn == "CITIZEN": + return "CITIZEN", "CITIZEN" + if ssn == "NONE": + return "NONE", "UNDOCUMENTED" + return ssn, "LEGAL_PERMANENT_RESIDENT" + + def _complete_case_target_encodings( table: pd.DataFrame, *, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_checkpoint.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_checkpoint.py index 0c6ebe0fa2..63bc85593c 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_checkpoint.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_checkpoint.py @@ -57,10 +57,10 @@ ASEC_RAW_STAGE_ARTIFACT_KIND = "populace_us_asec_raw_stage" ASEC_RAW_STAGE_CHECKPOINT_FILENAME = "asec_raw_stage.checkpoint.h5" ASEC_RAW_STAGE_OPERATOR_STATUS = "operator_untouched" -# Version 3 added the PAW_TYP restoration that gates TANF enrollment; older -# artifacts lack the gate column and must fail loudly rather than let -# PAW_VAL-only conflation back in (microcosm#591). -ASEC_RAW_STAGE_SCHEMA_VERSION = 3 +# Version 4 added measured A_LFSR for the Pew civilian-labor-force control; +# older artifacts lack the current labor-force status and must fail loudly +# rather than substitute prior-year earnings (microcosm#767). +ASEC_RAW_STAGE_SCHEMA_VERSION = 4 ASEC_RAW_STAGE_STAGE = "raw_source_mapping" _RAW_STAGE_BINDING_KEYS = frozenset( { @@ -75,9 +75,9 @@ "stage", } ) -_RAW_SOURCE_MAPPING_COLUMNS = frozenset({"ED_VAL", "LKWEEKS", "PAW_TYP"}) +_RAW_SOURCE_MAPPING_COLUMNS = frozenset({"A_LFSR", "ED_VAL", "LKWEEKS", "PAW_TYP"}) _RAW_STAGE_REQUIRED_PERSON_COLUMNS = frozenset( - {"ED_VAL", "LKWEEKS", "PAW_TYP", "PERIDNUM", "source_year"} + {"A_LFSR", "ED_VAL", "LKWEEKS", "PAW_TYP", "PERIDNUM", "source_year"} ) _RAW_SOURCE_MAPPING_KEYS = frozenset( { @@ -458,6 +458,19 @@ def _validate_raw_stage_source_columns(frame: Frame, *, path: Path) -> None: "in {0, 1, 2, 3}." ) + labor_force_status = pd.to_numeric(person["A_LFSR"], errors="coerce").to_numpy( + dtype=np.float64 + ) + valid_labor_force_status = np.isfinite(labor_force_status) & np.isin( + labor_force_status, + (0.0, 1.0, 2.0, 3.0, 4.0, 7.0), + ) + if not valid_labor_force_status.all(): + raise ValueError( + f"ASEC raw-stage checkpoint {path} A_LFSR must be complete integers " + "in {0, 1, 2, 3, 4, 7}." + ) + def _validate_asec_frame( frame: Frame, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/education_assistance_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/education_assistance_source.py index eea8a97de1..3e10525d39 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/education_assistance_source.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/education_assistance_source.py @@ -1,4 +1,4 @@ -"""Measured CPS ASEC educational assistance for the education-inputs stage. +"""Pinned CPS ASEC person sidecar for measured ED_VAL and A_LFSR fields. The retired eCPS build carried person-level educational assistance directly from ASEC ``ED_VAL`` (archived ``datasets/cps/cps.py`` line 1493: @@ -6,8 +6,11 @@ recode, filter, or universe restriction), and the archived ``datasets/cps/census_cps.py`` listed ``ED_VAL`` among the raw ASEC columns it kept. The frozen ``census_cps_*.h5`` inputs microcosm builds from never -carried that column, so this module restores it from the official, immutable -Census ASEC public-use archives — the same repair class as +carried that column. Those frozen inputs also omitted current civilian labor- +force status ``A_LFSR``, which the immigration stage needs to reproduce Pew's +age-16+ working-or-looking-for-work definition without a prior-year earnings +proxy. This module restores both fields from the official, immutable Census +ASEC public-use archives — the same repair class as :mod:`.weeks_unemployed`'s ``LKWEEKS`` sidecar, extended to every pooled income year. @@ -38,11 +41,14 @@ "ASEC_EDUCATION_ASSISTANCE_ARCHIVES", "ASEC_EDUCATION_ASSISTANCE_INCOME_YEARS", "ASEC_EDUCATION_ASSISTANCE_SOURCE_COLUMNS", + "ASEC_LABOR_FORCE_STATUS_COLUMN", + "ASEC_LABOR_FORCE_STATUS_VALID_CODES", "EDUCATION_ASSISTANCE_ARCHIVED_DERIVATION_URL", "EDUCATION_ASSISTANCE_ARCHIVED_SOURCE_URL", "AsecEducationArchive", "fetch_asec_education_assistance_source", "fill_asec_education_assistance_source", + "fill_asec_labor_force_status_source", "load_asec_education_assistance_sources", ] @@ -61,12 +67,15 @@ ) _SOURCE = "ED_VAL" +ASEC_LABOR_FORCE_STATUS_COLUMN = "A_LFSR" +ASEC_LABOR_FORCE_STATUS_VALID_CODES = frozenset({0, 1, 2, 3, 4, 7}) ASEC_EDUCATION_ASSISTANCE_SOURCE_COLUMNS: tuple[str, ...] = ( "PH_SEQ", "P_SEQ", "A_LINENO", "PERIDNUM", _SOURCE, + ASEC_LABOR_FORCE_STATUS_COLUMN, ) _AUDIT_WEIGHT_COLUMN = "A_FNLWGT" @@ -483,6 +492,19 @@ def load_asec_education_assistance_sources( f"ASEC {pins.survey_year} ED_VAL must be finite and nonnegative " f"at row(s): {rows}." ) + labor_force_status = pd.to_numeric( + raw[ASEC_LABOR_FORCE_STATUS_COLUMN], errors="coerce" + ).to_numpy(dtype=np.float64) + valid_labor_force_status = np.isfinite(labor_force_status) & np.isin( + labor_force_status, + sorted(ASEC_LABOR_FORCE_STATUS_VALID_CODES), + ) + if not valid_labor_force_status.all(): + rows = np.flatnonzero(~valid_labor_force_status)[:5].tolist() + raise ValueError( + f"ASEC {pins.survey_year} A_LFSR must be a complete integer in " + f"{sorted(ASEC_LABOR_FORCE_STATUS_VALID_CODES)} at row(s): {rows}." + ) weights = pd.to_numeric(raw[_AUDIT_WEIGHT_COLUMN], errors="coerce").to_numpy( dtype=np.float64 ) @@ -534,6 +556,7 @@ def load_asec_education_assistance_sources( ) audits[income_year] = audit part = raw.loc[:, list(ASEC_EDUCATION_ASSISTANCE_SOURCE_COLUMNS)].copy() + part[ASEC_LABOR_FORCE_STATUS_COLUMN] = labor_force_status.astype("int64") part.insert(0, "source_year", np.int64(income_year)) parts.append(part) result = pd.concat(parts, ignore_index=True) @@ -645,3 +668,117 @@ def fill_asec_education_assistance_source( "source_audit", {} ) return result + + +def fill_asec_labor_force_status_source( + person: pd.DataFrame, + source: pd.DataFrame, +) -> pd.DataFrame: + """Fill measured ``A_LFSR`` via the pinned exact Census identity join.""" + + required_person = ("source_year", "PERIDNUM") + missing_person = [column for column in required_person if column not in person] + if missing_person: + raise ValueError( + "ASEC labor-force-status repair requires person column(s): " + f"{missing_person}." + ) + required_source = ("source_year", *ASEC_EDUCATION_ASSISTANCE_SOURCE_COLUMNS) + missing_source = [column for column in required_source if column not in source] + if missing_source: + raise ValueError( + f"ASEC labor-force-status sidecar missing column(s): {missing_source}." + ) + result = person.copy(deep=True) + person_years = pd.to_numeric(result["source_year"], errors="coerce") + if person_years.isna().any() or (person_years != np.floor(person_years)).any(): + raise ValueError("ASEC labor-force-status person source_year is invalid.") + needed_years = sorted(int(year) for year in person_years.unique()) + covered = set( + pd.to_numeric(source["source_year"], errors="coerce").astype(int).unique() + ) + uncovered = [year for year in needed_years if year not in covered] + if uncovered: + raise ValueError( + "ASEC labor-force-status sidecar does not cover pooled income " + f"year(s): {uncovered}." + ) + + donor = source.copy(deep=True) + donor["PERIDNUM"] = _fixed_width_peridnum( + donor["PERIDNUM"], label="ASEC labor-force-status sidecar" + ) + if ASEC_LABOR_FORCE_STATUS_COLUMN in result.columns: + existing = pd.to_numeric( + result[ASEC_LABOR_FORCE_STATUS_COLUMN], errors="coerce" + ) + if existing.notna().any(): + raise ValueError( + "ASEC labor-force-status fill found a preexisting A_LFSR " + "column with values; refusing to overwrite measured data." + ) + filled = np.zeros(len(result), dtype=np.int64) + + for year in needed_years: + year_mask = person_years.eq(year).to_numpy() + year_donor = donor.loc[ + pd.to_numeric(donor["source_year"], errors="coerce").eq(year) + ].set_index("PERIDNUM") + keys = _fixed_width_peridnum( + result.loc[year_mask, "PERIDNUM"], + label="ASEC labor-force-status frame", + ) + missing_keys = keys[~keys.isin(year_donor.index)].drop_duplicates() + if not missing_keys.empty: + raise ValueError( + "ASEC labor-force-status sidecar does not cover frame " + f"PERIDNUM key(s) for income year {year}: " + f"{missing_keys.tolist()[:5]}." + ) + aligned = year_donor.reindex(keys.to_numpy()) + aligned.index = result.index[year_mask] + identity_pairs = [ + ( + "PH_SEQ", + "source_household_id" if "source_household_id" in result else "PH_SEQ", + ), + ("P_SEQ", "P_SEQ"), + ("A_LINENO", "A_LINENO"), + ] + for donor_column, frame_column in identity_pairs: + if frame_column not in result: + continue + observed = pd.to_numeric( + result.loc[year_mask, frame_column], errors="coerce" + ).to_numpy(dtype=np.float64) + expected = pd.to_numeric(aligned[donor_column], errors="coerce").to_numpy( + dtype=np.float64 + ) + mismatch = ( + ~np.isfinite(observed) | ~np.isfinite(expected) | (observed != expected) + ) + if mismatch.any(): + rows = result.index[year_mask].to_numpy()[mismatch][:5].tolist() + raise ValueError( + "ASEC labor-force-status redundant identity mismatch for " + f"{frame_column} against sidecar {donor_column} in income " + f"year {year} at row(s): {rows}." + ) + values = pd.to_numeric( + aligned[ASEC_LABOR_FORCE_STATUS_COLUMN], errors="coerce" + ).to_numpy(dtype=np.float64) + valid = np.isfinite(values) & np.isin( + values, + sorted(ASEC_LABOR_FORCE_STATUS_VALID_CODES), + ) + if not valid.all(): + raise ValueError( + f"ASEC labor-force-status sidecar A_LFSR is invalid for " + f"income year {year}." + ) + filled[year_mask] = values.astype(np.int64) + result[ASEC_LABOR_FORCE_STATUS_COLUMN] = filled + result.attrs["labor_force_status_source_audit"] = source.attrs.get( + "source_audit", {} + ) + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py b/packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py index ea03a02978..727cb4cc64 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py @@ -94,7 +94,9 @@ US_MULTISPINE_AGREEMENT_DIAGNOSTICS_ARTIFACT_KIND = ( "populace_us_multispine_agreement_diagnostics" ) -# 10 binds the portable authenticated primary-QRF worker identity. +# 10 binds the humanitarian-immigration composition gate into the stacked +# operator order and authenticated terminal-gate surface, and the portable +# authenticated primary-QRF worker identity. # 9 binds the post-assembly household-geography assignment receipt and its # authenticated release-vintage authorities. # 8 binds the nullable-boolean-capable physical H5 materializer in both the @@ -135,6 +137,30 @@ "materialize_multispine_agreement_outputs", "stacked_completeness_gate", "by_origin_battery", + "us_immigration_composition_gate", +) +_STACKED_TERMINAL_GATE_NAMES = frozenset( + { + "us_stacked_completeness", + "us_by_origin_battery", + "immigration_composition", + } +) +#: The stacked operator order attested schema-9 pools sealed, before the +#: humanitarian-immigration composition gate joined it. A literal, not a +#: function of the current order: history must not move when the present does. +_SCHEMA9_STACKED_POOL_OPERATOR_ORDER = ( + "assemble_stacked_spine", + "assign_us_puma_ladder", + "prepare_multispine_source_inputs_for_clone", + "gap_fill_stacked_spine", + "run_stacked_late_producer_dag", + "prepare_stacked_tail_derivation", + "derive_multispine_pool_inputs", + "seed_multispine_pool_inputs", + "materialize_multispine_agreement_outputs", + "stacked_completeness_gate", + "by_origin_battery", ) _LEGACY_POOL_OPERATOR_ORDER = ( "assemble", @@ -1399,6 +1425,7 @@ def _load_authenticated_us_multispine_pool_manifest( diagnostics_path=diagnostics_path, manifest_agreement_gate=manifest_agreement_gate, diagnostics_agreement_gate=diagnostics_agreement_gate, + require_canonical_gate_set=legacy_worker_authentication is None, ) if diagnostics_agreement_gate != manifest_agreement_gate: raise ValueError( @@ -1443,7 +1470,12 @@ def _validate_stacked_late_dag_manifest_binding( if manifest.get("pipeline") != "us-stacked-pool": return - if manifest.get("operator_order") != list(US_STACKED_POOL_OPERATOR_ORDER): + expected_operator_order = ( + US_STACKED_POOL_OPERATOR_ORDER + if legacy_worker_authentication is None + else _SCHEMA9_STACKED_POOL_OPERATOR_ORDER + ) + if manifest.get("operator_order") != list(expected_operator_order): raise ValueError( f"US stacked pool manifest {manifest_path} does not bind the " "canonical late-DAG operator order." @@ -1866,7 +1898,7 @@ def _validate_stacked_geography_h5_binding( manifest_path: Path, pool_path: Path, ) -> None: - """Bind schema-9 manifest geography claims to the authenticated H5.""" + """Bind current-schema manifest geography claims to the authenticated H5.""" if manifest.get("pipeline") != _STACKED_PIPELINE: return @@ -2685,6 +2717,7 @@ def _require_matching_terminal_gate_aliases( diagnostics_path: Path, manifest_agreement_gate: Mapping[str, object], diagnostics_agreement_gate: Mapping[str, object], + require_canonical_gate_set: bool = True, ) -> None: """Bind stacked terminal gates to the legacy compatibility aliases. @@ -2727,6 +2760,29 @@ def _require_matching_terminal_gate_aliases( f"US stacked pool diagnostics {diagnostics_path} terminal_gates do " "not match agreement_gate." ) + manifest_gates = _mapping( + manifest_terminal_gates.get("gates"), + label=f"US stacked pool manifest {manifest_path}.terminal_gates.gates", + ) + diagnostics_gates = _mapping( + diagnostics_terminal_gates.get("gates"), + label=(f"US stacked pool diagnostics {diagnostics_path}.terminal_gates.gates"), + ) + # The canonical gate set arrived with manifest schema 10. Attested + # schema-9 pools sealed whatever terminal gates their build ran, so only + # the alias equalities above apply to them. + if not require_canonical_gate_set: + return + for label, gates in ( + (f"US stacked pool manifest {manifest_path}", manifest_gates), + (f"US stacked pool diagnostics {diagnostics_path}", diagnostics_gates), + ): + if set(gates) != _STACKED_TERMINAL_GATE_NAMES: + raise ValueError( + f"{label} does not carry the canonical terminal gate set; " + f"expected={sorted(_STACKED_TERMINAL_GATE_NAMES)}, " + f"observed={sorted(gates)}." + ) def _publication_run_id(value: object, *, label: str) -> str: diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py b/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py index c6b7cb51b4..62d806e833 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py @@ -12,7 +12,8 @@ ``OTHER_NON_CITIZEN`` / ``NONE`` (likely undocumented, i.e. ITIN-filer territory for tax purposes). - ``immigration_status_str``: ``CITIZEN`` / ``LEGAL_PERMANENT_RESIDENT`` / - ``CUBAN_HAITIAN_ENTRANT`` / ``DACA`` / ``UNDOCUMENTED``. + ``CUBAN_HAITIAN_ENTRANT`` / ``DACA`` / ``UNDOCUMENTED`` plus the supported + humanitarian and temporary-protection statuses listed below. Method — survey measurement first, published residual method second, and one narrowly-scoped control where the survey carries no signal at all: @@ -28,34 +29,59 @@ Residual Methods Robust?", SSRN 4662801). 3. **Work/study authorization split.** The CPS carries no work-authorization variable, so among residual non-citizens the split between EAD holders and - the unauthorized is unidentified from the survey alone. Workers and - students spill to ``NON_CITIZEN_VALID_EAD`` — in deterministic seeded - order — until the *remaining* undocumented worker/student counts match - their published control totals (Pew Research Center; Higher Ed - Immigration Portal). This is the only forced margin. -4. **The total undocumented population is emergent, not forced.** + the unauthorized is unidentified from the survey alone. Workers (measured + with ASEC ``A_LFSR`` at age 16+, not prior-year earnings) and students + spill to ``NON_CITIZEN_VALID_EAD`` in deterministic seeded order until the + broad reported universes match their controls. Pew retains DACA, parole, + TPS, and similar temporary protections in its unauthorized estimates, so + those statuses continue to count after receiving EADs. Residual + Cuban/Haitian rows that receive an EAD also stay in that universe because + the engine's broader ``CUBAN_HAITIAN_ENTRANT`` label carries the modeled + CHNV-parole/Haiti-TPS subset. +4. **The total Pew-defined unauthorized population is emergent, not forced.** Representation is calibration's job, not the label stage's: the release - gate checks the emergent total against a cited published anchor with a - coarse plausibility band, and a follow-up calibration target (Ledger - facts lane) can reconcile the level in the weights. On the 2024 ASEC the - emergent total is ≈13M — inside the range of published 2023–24 estimates - — with 26.6M non-citizens, 18.0M residual after the indicators, and 4.8M - spilled to EAD. -5. **Status tags carry only statutory tests the data can support.** + gate checks the emergent broad-universe total against Pew's cited 2023 + anchor with a coarse plausibility band. The engine's narrower + ``UNDOCUMENTED`` enum remains exactly paired with ``ssn_card_type=NONE``. +5. **Humanitarian/temporary-protection statuses are drawn to published + stocks, never blanketed (microcosm #767).** Blanket ``REFUGEE``/``TPS`` + labels for recent arrivals were originally refused because LPR treatment + gave near-identical means-tested eligibility; H.R.1 repealed exactly that + equivalence (Medicaid §71109 effective 2026-10-01, SNAP §10108 effective + 2025-07-01, ACA §71301/§71302 effective 2026/2027 — all keyed to these + enum values in policyengine-us parameters), so an LPR-only file silently + zeroes every one of those channels. Instead of blankets, the stage now + draws ``PAROLED_ONE_YEAR`` / ``REFUGEE`` / ``ASYLEE`` / + ``DEPORTATION_WITHHELD`` / ``TPS`` — in that order (decreasing target + precision: exact per-origin program admissions first), mutually + exclusive, so a person two cohorts could claim takes the earlier + status — from candidates whose survey signature supports it (origin + country x arrival window x legal-status indicators), each to a + manifest-cited weighted stock target. Cuba/Haiti-born persons are + excluded from every humanitarian draw (the ``CUBAN_HAITIAN_ENTRANT`` + class is the better statutory label and keeps H.R.1 eligibility), as is + the DACA statutory cohort. Draws from the residual pool (TPS and parole + only) flip ``ssn_card_type`` ``NONE`` to ``NON_CITIZEN_VALID_EAD`` — + both programs grant employment authorization — so the two output columns + never disagree. ``CONDITIONAL_ENTRANT`` is deliberately not emitted: the + INA 203(a)(7) class closed in 1980 and surviving holders are + indistinguishable from LPRs at this granularity. +6. **Status tags carry only statutory tests the data can support.** ``DACA`` applies the statutory cohort test (arrived by 2007 before age 16, aged 15+) to EAD holders; ``CUBAN_HAITIAN_ENTRANT`` applies the nationality-plus-arrival class to documented non-citizens; every other documented non-citizen stays ``LEGAL_PERMANENT_RESIDENT`` (the modal true - status). Blanket ``REFUGEE``/``TPS`` labels for recent arrivals or - leftover EAD holders are deliberately not emitted: they would mislabel - millions (true stocks are under a million each) and over-grant - refugee-class exemptions in benefit rules, while LPR treatment gives - near-identical means-tested eligibility for those populations. + status). Selection draws are seeded blake2b hashes keyed by the person's stable source -identity (``source_year``/``source_person_id`` when present), so +identity (``source_year``/``source_household_id``/``source_person_id`` when +present), so support-channel clones of one source person always receive the same status -and reruns are bit-reproducible without global RNG state. +and reruns are bit-reproducible without global RNG state. A caller whose rows +carry no unique source identity (the ACS local lane, where ``SPORDER`` +repeats in every household) passes explicit draw keys, and a caller with an +exact entry year (the ACS ``YOEP``) passes it in place of the ``PEINUSYR`` +bins (:mod:`~microcosm.build.us_runtime.acs_local_immigration`). Control totals and the gate anchor are data, not code: they live in the ``immigration_status`` stage of ``microcosm/build/us/source_stages.json`` with @@ -89,16 +115,24 @@ from microcosm.frame.units import US_SCHEMA __all__ = [ + "HUMANITARIAN_STATUS_CATEGORIES", "IMMIGRATION_STATUS_VALUES", "SSN_CARD_TYPE_VALUES", "US_IMMIGRATION_NONCONSTANT_PERSON_COLUMNS", "US_IMMIGRATION_OUTPUT_COLUMNS", "US_IMMIGRATION_REQUIRED_SOURCE_COLUMNS", "US_IMMIGRATION_STAGE_NAME", + "HumanitarianDraw", + "ImmigrationControls", "UndocumentedControls", "derive_us_immigration_status_from_manifest", + "reconcile_us_immigration_humanitarian_transfer", "us_immigration_composition_gate", "us_immigration_composition_summary", + "us_immigration_controls", + "us_immigration_evidence_features", + "us_immigration_humanitarian_draw_mask", + "us_immigration_humanitarian_transfer_selection_masks", "us_immigration_stage_spec", "with_us_immigration_inputs", ] @@ -126,13 +160,18 @@ #: PolicyEngine-US ``ImmigrationStatus`` enum member names this stage emits #: (a deliberate subset of the engine's full enum domain; see module -#: docstring for why blanket REFUGEE/TPS labels are not fabricated). +#: docstring for why ``CONDITIONAL_ENTRANT`` is not fabricated). IMMIGRATION_STATUS_VALUES: tuple[str, ...] = ( "CITIZEN", "LEGAL_PERMANENT_RESIDENT", "CUBAN_HAITIAN_ENTRANT", "DACA", "UNDOCUMENTED", + "PAROLED_ONE_YEAR", + "REFUGEE", + "ASYLEE", + "DEPORTATION_WITHHELD", + "TPS", ) #: Raw CPS ASEC person columns the derivation reads. All of them ship in the @@ -146,8 +185,7 @@ "A_MARITL", "A_SPOUSE", "A_HSCOL", - "WSAL_VAL", - "SEMP_VAL", + "A_LFSR", "MCARE", "CAID", "IHSFLG", @@ -169,11 +207,13 @@ _FORCED_CONTROL_KEYS = ("undocumented_workers", "undocumented_students") _ANCHOR_KEY = "undocumented_population_anchor" +_HUMANITARIAN_KEY = "humanitarian_status_stocks" _DERIVE_IMMIGRATION_STATUS_PARAMETER_KEYS = frozenset( { *_FORCED_CONTROL_KEYS, _ANCHOR_KEY, + _HUMANITARIAN_KEY, "seed_from_build_config", "time_period_from_build_config", } @@ -236,15 +276,219 @@ _DACA_MAX_AGE_AT_ENTRY = 16 _DACA_MIN_CURRENT_AGE = 15 +#: Humanitarian draw order: decreasing target precision (exact per-origin +#: program admissions first, national flow aggregates next, per-country TPS +#: registrations last). Draws are sequential and mutually exclusive. +HUMANITARIAN_STATUS_CATEGORIES: tuple[str, ...] = ( + "paroled_one_year", + "refugee", + "asylee", + "deportation_withheld", + "tps", +) + +_HUMANITARIAN_STATUS_BY_CATEGORY: Mapping[str, str] = { + "paroled_one_year": "PAROLED_ONE_YEAR", + "refugee": "REFUGEE", + "asylee": "ASYLEE", + "deportation_withheld": "DEPORTATION_WITHHELD", + "tps": "TPS", +} + +# Pew's residual estimates use a deliberately broader universe than the +# engine's ``UNDOCUMENTED`` enum. In particular, Pew retains DACA recipients +# and people with temporary protection (including parole and TPS) in its +# unauthorized-immigrant totals. Refugees and people already granted asylum +# are lawfully admitted statuses and therefore stay outside this universe. +# ``CUBAN_HAITIAN_ENTRANT`` is conditional: only EAD rows that came from the +# residual pool count, not assumed-documented entrants sharing that broad +# engine label. +_PEW_UNAUTHORIZED_STATUS_VALUES: tuple[str, ...] = ( + "UNDOCUMENTED", + "DACA", + "PAROLED_ONE_YEAR", + "DEPORTATION_WITHHELD", + "TPS", +) +_PEW_INCLUDED_HUMANITARIAN_STATUS_VALUES: tuple[str, ...] = ( + "PAROLED_ONE_YEAR", + "DEPORTATION_WITHHELD", + "TPS", +) + +# CPS ASEC A_LFSR: 1 working, 2 with a job/not at work, 3 unemployed/looking, +# 4 unemployed/on layoff. Code 0 covers children and Armed Forces; 7 is not +# in the labor force. Pew's labor-force total is for people age 16 or older. +_CPS_LABOR_FORCE_STATUS_CODES = (1, 2, 3, 4) +_CPS_LABOR_FORCE_STATUS_DOMAIN = (0, 1, 2, 3, 4, 7) + +#: PEINUSYR codes for 2020+ arrivals. Census re-bins the newest codes each +#: survey year. Code 27 is 2020-2021 in the 2024, 2025 and 2026 ASEC; code 28 +#: is 2022-2024, 2022-2025 and 2022-2023 in turn; the 2026 ASEC adds code 29 +#: for 2024-2026. Foreign-born infants agree: in each of those files they sit +#: only in the newest code. Every window below holds in all three, the +#: default pool (``asec_sources.ASEC_DEFAULT_POOL_INCOME_YEARS``). Operation +#: Allies Welcome (2021-2022 arrivals) draws from 27 and 28. U4U and CHNV +#: (2022 to early 2025) and refugees admitted in FY2023-FY2024 draw from 28 +#: and 29, never from 27. The 2023 ASEC codes 2020-2023 as 27 and has no 28, +#: so the 2022+ windows find no candidates there; it is outside the default +#: pool. +_RECENT_ARRIVAL_CODES = (27, 28) +#: Codes holding only 2022+ arrivals: 28 in each default-pool file, and the +#: 2026 ASEC's 29. +_POST_2021_ARRIVAL_CODES = (28, 29) +_PAROLE_ASEC_ARRIVAL_CODES: Mapping[str, tuple[int, ...]] = { + "afghanistan": _RECENT_ARRIVAL_CODES, + "ukraine": _POST_2021_ARRIVAL_CODES, + "nicaragua": _POST_2021_ARRIVAL_CODES, + "venezuela": _POST_2021_ARRIVAL_CODES, +} +#: First exact (ACS YOEP) arrival year of each parole program. +_PAROLE_ACS_MIN_ARRIVAL_YEAR: Mapping[str, int] = { + "afghanistan": 2021, + "ukraine": 2022, + "nicaragua": 2023, + "venezuela": 2022, +} +#: Last exact (ACS YOEP) arrival year of each parole program that ended +#: before the period. Operation Allies Welcome paroled Afghans who arrived in +#: 2021 and 2022, so exact-year rows take only those arrivals. ASEC rows keep +#: their coarser codes: the 2024 ASEC's code 28 spans 2022-2024. A program +#: missing here has no last year. +_PAROLE_ACS_MAX_ARRIVAL_YEAR: Mapping[str, int] = { + "afghanistan": 2022, +} +#: Asylum grants lag arrival by filing queues plus court backlog; grantees +#: in 2022-2024 overwhelmingly arrived 2016+ (OHSS asylee flow reports). The +#: 2026 ASEC's code 29 (2024-2026) is left out: its arrivals come after all +#: but the last months of the FY2022-FY2024 grants the target counts. +_ASYLEE_ARRIVAL_CODES = (25, 26, 27, 28) +#: Withholding of removal follows years of proceedings: settled arrivals only. +_WITHHELD_MAX_ARRIVAL_CODE = 26 + +#: PENATVTY birth-country codes, verified against Census CPS technical +#: documentation (cpsmar24.pdf) Appendix I "Countries and Areas of the +#: World". Cuba (327) and Haiti (332) stay out of every humanitarian draw: +#: the Cuban/Haitian-entrant class is the better statutory label. +#: +#: Parole per-origin keys: Operation Allies Welcome (Afghanistan), Uniting +#: for Ukraine, and the non-Cuban/Haitian CHNV nationalities. +_PAROLE_ORIGIN_CODES: Mapping[str, tuple[int, ...]] = { + "afghanistan": (200,), + "ukraine": (164,), + "nicaragua": (315,), + "venezuela": (373,), +} + +#: Top refugee-resettlement nationalities, OHSS Refugees Annual Flow Report +#: FY2023 Table 3 and the OHSS FY-24 report (DR Congo, Syria, Afghanistan, +#: Burma, Venezuela lead; Congo appears as both 412 Congo and 459 Zaire). +_REFUGEE_ORIGIN_CODES: tuple[int, ...] = ( + 412, # Congo + 459, # Zaire (Democratic Republic of the Congo) + 239, # Syria + 200, # Afghanistan + 205, # Myanmar (Burma) + 373, # Venezuela + 448, # Somalia + 451, # Sudan + 417, # Eritrea + 213, # Iraq + 313, # Guatemala + 164, # Ukraine +) + +#: Top asylum-grant nationalities, OHSS Asylees flow reports FY2022-FY2024 +#: (Afghanistan, China, Venezuela, Russia lead; Central/South America and +#: Egypt/Cameroon/Turkey round out the recurring top grant countries). +_ASYLEE_ORIGIN_CODES: tuple[int, ...] = ( + 200, # Afghanistan + 207, # China + 373, # Venezuela + 163, # Russia + 312, # El Salvador + 313, # Guatemala + 314, # Honduras + 315, # Nicaragua + 210, # India + 414, # Egypt + 407, # Cameroon + 364, # Colombia + 365, # Ecuador + 370, # Peru + 243, # Turkey + 239, # Syria +) + +#: TPS per-country keys: birth codes plus the latest PEINUSYR arrival code +#: compatible with the designation's required continuous-residence date +#: (CRS RS20844 Table 1). Legacy designations bind hard (El Salvador +#: 2001-02-13 -> code 17; Honduras/Nicaragua 1998-12-30 -> code 16; Nepal +#: 2015-06-24 -> code 24); 2021+ designations reach code 28, where the 2024 +#: ASEC's 2022-2024 bin unavoidably includes some post-cutoff arrivals. The +#: 2026 ASEC's code 29 (2024-2026) is left out: no designation's residence +#: date falls after 2024 (``_TPS_ACS_MAX_ARRIVAL_YEAR_BY_BIRTH``), so at most +#: the bin's first year could qualify. +#: Ukraine and Afghanistan TPS registrants are carried by the parole draw +#: (same U4U/OAW people); Haiti TPS is carried by CUBAN_HAITIAN_ENTRANT. +_TPS_ORIGIN_CODES: Mapping[str, tuple[tuple[int, ...], int]] = { + "venezuela": ((373,), 28), + "el_salvador": ((312,), 17), + "honduras": ((314,), 16), + "nicaragua": ((315,), 16), + "nepal": ((229,), 24), + # Burma, Syria, Yemen, Lebanon, Cameroon, Ethiopia, Somalia, Sudan + # (South Sudan shares 451 in the CPS codebook). + "other_designated": ((205, 239, 248, 224, 407, 416, 448, 451), 28), +} + +#: ACS YOEP preserves an exact year, unlike the ASEC arrival bins. Use that +#: extra information for continuous-residence compatibility rather than +#: deliberately coarsening YOEP back to PEINUSYR. The per-country cutoffs are +#: calendar-year approximations of the designation dates; ASEC retains the +#: published coarse-bin contract above. +_TPS_ACS_MAX_ARRIVAL_YEAR_BY_BIRTH: Mapping[int, int] = { + 373: 2023, # Venezuela (2021 and 2023 designations) + 312: 2001, # El Salvador + 314: 1998, # Honduras + 315: 1998, # Nicaragua + 229: 2015, # Nepal + 205: 2024, # Burma + 239: 2024, # Syria + 248: 2024, # Yemen + 224: 2024, # Lebanon + 407: 2023, # Cameroon + 416: 2024, # Ethiopia + 448: 2024, # Somalia + 451: 2023, # Sudan / South Sudan (shared survey code) +} + +#: Categories whose flat manifest block carries one national target. +_FLAT_HUMANITARIAN_CATEGORIES = ("refugee", "asylee", "deportation_withheld") +#: Categories whose manifest block carries per-origin targets, and the +#: module table each block's keys must match exactly. +_PER_ORIGIN_HUMANITARIAN_CATEGORIES: Mapping[str, tuple[str, ...]] = { + "paroled_one_year": tuple(_PAROLE_ORIGIN_CODES), + "tps": tuple(_TPS_ORIGIN_CODES), +} + #: Weighted share of persons whose SSN card type is not ``CITIZEN`` must land #: in this band (Census counts ~22–27M non-citizens of ~336M residents; a #: share outside it means the imputation collapsed or exploded). _NON_CITIZEN_SHARE_BAND = (0.03, 0.12) -#: Emergent weighted ``NONE`` (undocumented) population relative to the cited -#: published anchor. Coarse by design — a release-blocking backstop against -#: collapse or explosion, not a calibration objective; the level belongs to -#: the calibration lane. +#: Emergent Pew-defined unauthorized population relative to the cited +#: published anchor. This includes DACA and specified temporary protections, +#: not only the engine's ``NONE``/``UNDOCUMENTED`` pair. Coarse by design — a +#: release-blocking backstop against collapse or explosion, not a calibration +#: objective; the level belongs to the calibration lane. _UNDOCUMENTED_ANCHOR_RELATIVE_BAND = (0.5, 1.6) +#: Emitted humanitarian mass per category relative to its cited target. +#: The draw forces the target when candidates suffice, so the band only +#: bites on pool exhaustion (the ASEC undercovers 2022-24 arrivals — the +#: Ukraine parole pool saturates below its admin count) or collapse. A +#: category with an explicit zero target must emit exactly zero. The stacked +#: transfer's reconciliation refuses an exhausted draw below the band's floor. +_HUMANITARIAN_TARGET_RELATIVE_BAND = (0.35, 1.5) @dataclass(frozen=True) @@ -252,11 +496,12 @@ class UndocumentedControls: """Published totals the stage forces or gates against. Attributes: - workers: Undocumented-worker control the EAD spill enforces. - students: Undocumented-student control the EAD spill enforces. - population_anchor: Published total undocumented population the - composition gate checks the *emergent* total against (never - forced in the labels). + workers: Pew unauthorized-immigrant labor-force control the EAD spill + enforces, including DACA and temporary protections Pew retains. + students: Broad undocumented-student control the EAD spill enforces. + population_anchor: Published Pew unauthorized population the + composition gate checks the *emergent broad-universe* total + against (never forced in the labels). sources: Citation URL per key, straight from the manifest. """ @@ -283,6 +528,91 @@ def __post_init__(self) -> None: ) +@dataclass(frozen=True) +class HumanitarianDraw: + """One seeded humanitarian draw: a cited stock the stage selects toward. + + Attributes: + category: Manifest category key (``paroled_one_year`` … ``tps``). + origin: Per-origin key inside the category, or ``None`` for a + national draw. + status: ``ImmigrationStatus`` enum member name the draw emits. + target: Cited weighted stock. Zero is allowed and means the + category is explicitly not imputed (the citation documents why). + source: Citation URL or reference for the target. + """ + + category: str + origin: str | None + status: str + target: float + source: str + + def __post_init__(self) -> None: + if not np.isfinite(self.target) or self.target < 0: + raise ValueError( + f"Humanitarian target {self.label!r} must be non-negative, " + f"got {self.target!r}." + ) + if not self.source: + raise ValueError( + f"Humanitarian target {self.label!r} requires a source citation." + ) + + @property + def label(self) -> str: + return ( + self.category if self.origin is None else f"{self.category}:{self.origin}" + ) + + @property + def salt(self) -> str: + return f"immigration:{self.label}" + + +@dataclass(frozen=True) +class ImmigrationControls: + """Every manifest-sourced control the immigration stage consumes. + + Attributes: + undocumented: The legacy forced margins and the composition-gate + anchor for the residual pool. + humanitarian: The humanitarian draws in assignment order — the + per-origin draws of a category are adjacent, categories follow + ``HUMANITARIAN_STATUS_CATEGORIES``. + """ + + undocumented: UndocumentedControls + humanitarian: tuple[HumanitarianDraw, ...] + + def humanitarian_target(self, category: str) -> float: + """Summed cited target for one category across its origins.""" + + return float( + sum(draw.target for draw in self.humanitarian if draw.category == category) + ) + + +@dataclass(frozen=True) +class _ImmigrationEvidenceProfile: + """Canonical immigration evidence shared by ASEC and ACS rows. + + ``exact_arrival`` marks rows whose arrival is an exact year (the ACS + ``YOEP``, or exact years a caller supplies) rather than an ASEC + ``PEINUSYR`` bin; the humanitarian arrival windows read the year itself + on those rows and the bin codes on the rest. + """ + + exact_arrival: np.ndarray + is_citizen: np.ndarray + birth_country: np.ndarray + arrival_code: np.ndarray + arrival_year: np.ndarray + age: np.ndarray + age_at_entry: np.ndarray + tps_acs_max_arrival_year: np.ndarray + + def us_immigration_stage_spec() -> SourceStageSpec: """Load the packaged ``immigration_status`` source-stage manifest entry.""" @@ -356,15 +686,17 @@ def derive_us_immigration_status_from_manifest( raise SourceRuntimeError( "US immigration derivation requires finite non-negative person weights." ) - ssn_codes = _assign_ssn_card_codes( + ssn_codes, humanitarian_marks = _assign_ssn_card_codes( result, weights.to_numpy(dtype=np.float64), seed=int(context.config.seed), controls=controls, + time_period=int(context.config.target_year), ) status = _derive_immigration_status( result, ssn_codes, + humanitarian_marks, time_period=int(context.config.target_year), ) result["ssn_card_type"] = pd.Series(ssn_codes, index=result.index).map( @@ -374,7 +706,118 @@ def derive_us_immigration_status_from_manifest( return result -def _controls_from_parameters(params: Mapping[str, object]) -> UndocumentedControls: +def _target_and_source( + block: object, + *, + label: str, + value_key: str, + allow_zero: bool, +) -> tuple[float, str]: + if not isinstance(block, Mapping): + raise SourceRuntimeError( + f"US immigration control {label!r} requires an object with " + f"{value_key!r} and 'source'." + ) + unexpected = sorted(set(block) - {value_key, "source"}) + if unexpected: + raise SourceRuntimeError( + f"US immigration control {label!r} has unsupported key(s): {unexpected}." + ) + value = block.get(value_key) + source = block.get("source") + valid_number = isinstance(value, int | float) and not isinstance(value, bool) + if not valid_number or (float(value) <= 0 and not (allow_zero and value == 0)): + requirement = "non-negative" if allow_zero else "positive" + raise SourceRuntimeError( + f"US immigration control {label!r} requires a {requirement} numeric " + f"{value_key!r}." + ) + if not isinstance(source, str) or not source: + raise SourceRuntimeError( + f"US immigration control {label!r} requires a source citation." + ) + return float(value), source + + +def _humanitarian_draws_from_parameters( + params: Mapping[str, object], +) -> tuple[HumanitarianDraw, ...]: + block = params.get(_HUMANITARIAN_KEY) + if not isinstance(block, Mapping): + raise SourceRuntimeError( + f"US immigration derivation requires a {_HUMANITARIAN_KEY!r} object " + f"with one block per category {list(HUMANITARIAN_STATUS_CATEGORIES)}." + ) + unexpected = sorted(set(block) - set(HUMANITARIAN_STATUS_CATEGORIES)) + if unexpected: + raise SourceRuntimeError( + f"US immigration {_HUMANITARIAN_KEY} has unsupported category(ies): " + f"{unexpected}." + ) + missing = sorted(set(HUMANITARIAN_STATUS_CATEGORIES) - set(block)) + if missing: + raise SourceRuntimeError( + f"US immigration {_HUMANITARIAN_KEY} is missing category(ies): {missing}." + ) + draws: list[HumanitarianDraw] = [] + for category in HUMANITARIAN_STATUS_CATEGORIES: + status = _HUMANITARIAN_STATUS_BY_CATEGORY[category] + category_block = block[category] + if category in _PER_ORIGIN_HUMANITARIAN_CATEGORIES: + origin_keys = _PER_ORIGIN_HUMANITARIAN_CATEGORIES[category] + if not isinstance(category_block, Mapping): + raise SourceRuntimeError( + f"US immigration control {category!r} requires per-origin " + f"blocks {list(origin_keys)}." + ) + unexpected_origins = sorted(set(category_block) - set(origin_keys)) + if unexpected_origins: + raise SourceRuntimeError( + f"US immigration control {category!r} has origin(s) outside " + f"the stage's codebook table: {unexpected_origins}." + ) + missing_origins = sorted(set(origin_keys) - set(category_block)) + if missing_origins: + raise SourceRuntimeError( + f"US immigration control {category!r} is missing origin(s): " + f"{missing_origins}." + ) + for origin in origin_keys: + target, source = _target_and_source( + category_block[origin], + label=f"{category}:{origin}", + value_key="target", + allow_zero=True, + ) + draws.append( + HumanitarianDraw( + category=category, + origin=origin, + status=status, + target=target, + source=source, + ) + ) + else: + target, source = _target_and_source( + category_block, + label=category, + value_key="target", + allow_zero=True, + ) + draws.append( + HumanitarianDraw( + category=category, + origin=None, + status=status, + target=target, + source=source, + ) + ) + return tuple(draws) + + +def _controls_from_parameters(params: Mapping[str, object]) -> ImmigrationControls: values: dict[str, float] = {} sources: dict[str, str] = {} for key, value_key in ( @@ -382,49 +825,29 @@ def _controls_from_parameters(params: Mapping[str, object]) -> UndocumentedContr ("undocumented_students", "target"), (_ANCHOR_KEY, "value"), ): - block = params.get(key) - if not isinstance(block, Mapping): - raise SourceRuntimeError( - f"US immigration derivation requires a {key!r} object with " - f"{value_key!r} and 'source'." - ) - unexpected = sorted(set(block) - {value_key, "source"}) - if unexpected: - raise SourceRuntimeError( - f"US immigration control {key!r} has unsupported key(s): {unexpected}." - ) - value = block.get(value_key) - source = block.get("source") - if ( - not isinstance(value, int | float) - or isinstance(value, bool) - or float(value) <= 0 - ): - raise SourceRuntimeError( - f"US immigration control {key!r} requires a positive numeric " - f"{value_key!r}." - ) - if not isinstance(source, str) or not source: - raise SourceRuntimeError( - f"US immigration control {key!r} requires a source citation." - ) - values[key] = float(value) + value, source = _target_and_source( + params.get(key), label=key, value_key=value_key, allow_zero=False + ) + values[key] = value sources[key] = source - return UndocumentedControls( + undocumented = UndocumentedControls( workers=values["undocumented_workers"], students=values["undocumented_students"], population_anchor=values[_ANCHOR_KEY], sources=sources, ) + return ImmigrationControls( + undocumented=undocumented, + humanitarian=_humanitarian_draws_from_parameters(params), + ) -def _packaged_controls() -> UndocumentedControls: - """The cited controls of the packaged ``immigration_status`` stage.""" +def us_immigration_controls() -> ImmigrationControls: + """Return the controls bound to the packaged immigration manifest stage.""" - stage = us_immigration_stage_spec() derive = [ operation - for operation in stage.operations + for operation in us_immigration_stage_spec().operations if operation.kind == "derive_immigration_status" ] if len(derive) != 1: @@ -451,6 +874,216 @@ def _float_column(person: pd.DataFrame, column: str) -> np.ndarray: ) +def _numeric_evidence_column(person: pd.DataFrame, column: str) -> np.ndarray: + if column not in person.columns: + return np.full(len(person), np.nan, dtype=np.float64) + return pd.to_numeric(person[column], errors="coerce").to_numpy( + dtype=np.float64, + na_value=np.nan, + ) + + +def _source_aware_immigration_profile( + person: pd.DataFrame, + *, + time_period: int, + arrival_year: np.ndarray | None = None, +) -> _ImmigrationEvidenceProfile: + """Normalize ASEC PRCITSHP/PENATVTY/PEINUSYR and ACS CIT/POBP/YOEP. + + A stacked recipient carries the union of both raw survey schemas, with the + non-owning source null on each row. Exactly one citizenship source must be + present. Relevant POBP country codes share the Census country codebook with + PENATVTY, so the normalized origin is lossless; ACS YOEP retains its exact + year while ASEC necessarily uses the documented interval midpoint. + + ``arrival_year`` (one exact year per row, NaN where there is none) + replaces both survey arrival fields: every row then reads as exact-year + evidence, which is how the ACS local lane runs the kernel on its + CPS-named view of ACS persons. + """ + + if isinstance(time_period, bool) or int(time_period) != time_period: + raise ValueError( + f"US immigration time_period must be an integer, got {time_period!r}." + ) + time_period = int(time_period) + asec_citizenship = _numeric_evidence_column(person, "PRCITSHP") + acs_citizenship = _numeric_evidence_column(person, "CIT") + has_asec = np.isfinite(asec_citizenship) + has_acs = np.isfinite(acs_citizenship) + ambiguous = has_asec & has_acs + missing = ~has_asec & ~has_acs + if ambiguous.any() or missing.any(): + raise SourceRuntimeError( + "US immigration evidence requires exactly one row-level citizenship " + "source (ASEC PRCITSHP or ACS CIT); " + f"ambiguous_rows={int(ambiguous.sum())}, missing_rows={int(missing.sum())}." + ) + + for label, values, present in ( + ("PRCITSHP", asec_citizenship, has_asec), + ("CIT", acs_citizenship, has_acs), + ): + invalid = present & ( + ~np.equal(values, np.rint(values)) | ~np.isin(values, [1, 2, 3, 4, 5]) + ) + if invalid.any(): + bad = sorted(set(values[invalid].tolist()))[:5] + raise SourceRuntimeError( + f"{label} carries value(s) outside the Census domain 1..5: {bad}." + ) + + citizenship = np.where(has_acs, acs_citizenship, asec_citizenship).astype(np.int64) + is_citizen = np.isin(citizenship, [1, 2, 3, 4]) + + asec_birth = _numeric_evidence_column(person, "PENATVTY") + acs_birth = _numeric_evidence_column(person, "POBP") + birth = np.where(has_acs, acs_birth, asec_birth) + invalid_birth = ~np.isfinite(birth) | ~np.equal(birth, np.rint(birth)) | (birth < 1) + if invalid_birth.any(): + raise SourceRuntimeError( + "US immigration evidence requires a positive integral PENATVTY/POBP " + f"country code on every row; invalid_rows={int(invalid_birth.sum())}." + ) + birth_country = birth.astype(np.int64) + + if arrival_year is not None: + exact_year = np.asarray(arrival_year, dtype=np.float64) + if exact_year.shape != (len(person),): + raise ValueError( + f"Exact arrival years cover {exact_year.shape} row(s); the person " + f"table has {len(person)}." + ) + invalid_exact = np.isin(citizenship, [4, 5]) & ( + ~np.isfinite(exact_year) + | ~np.equal(exact_year, np.rint(exact_year)) + | (exact_year < 1900) + | (exact_year > time_period) + ) + if invalid_exact.any(): + raise SourceRuntimeError( + "An exact arrival year must be an integral 1900..time_period year " + "for every foreign-born row; " + f"invalid_rows={int(invalid_exact.sum())}." + ) + exact_arrival = np.ones(len(person), dtype=bool) + arrival_code = np.zeros(len(person), dtype=np.int64) + arrival_years = np.where( + np.isfinite(exact_year), exact_year, float(time_period) + ).astype(np.int64) + else: + exact_arrival, arrival_code, arrival_years = _survey_arrival( + person, + has_asec=has_asec, + has_acs=has_acs, + citizenship=citizenship, + time_period=time_period, + ) + + age_values = _numeric_evidence_column(person, "A_AGE") + if "age" in person.columns: + mapped_age = _numeric_evidence_column(person, "age") + age_values = np.where(np.isfinite(age_values), age_values, mapped_age) + invalid_age = ( + ~np.isfinite(age_values) + | ~np.equal(age_values, np.rint(age_values)) + | (age_values < 0) + ) + if invalid_age.any(): + raise SourceRuntimeError( + "US immigration evidence requires a finite non-negative integral " + f"A_AGE/age on every row; invalid_rows={int(invalid_age.sum())}." + ) + age = age_values.astype(np.int64) + age_at_entry = np.maximum(0, age - (time_period - arrival_years)) + tps_acs_max_arrival_year = np.full(len(person), -1, dtype=np.int64) + for birth_code, cutoff in _TPS_ACS_MAX_ARRIVAL_YEAR_BY_BIRTH.items(): + tps_acs_max_arrival_year[birth_country == birth_code] = cutoff + return _ImmigrationEvidenceProfile( + exact_arrival=exact_arrival, + is_citizen=is_citizen, + birth_country=birth_country, + arrival_code=arrival_code, + arrival_year=arrival_years, + age=age, + age_at_entry=age_at_entry, + tps_acs_max_arrival_year=tps_acs_max_arrival_year, + ) + + +def _survey_arrival( + person: pd.DataFrame, + *, + has_asec: np.ndarray, + has_acs: np.ndarray, + citizenship: np.ndarray, + time_period: int, +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """(exact-arrival flag, PEINUSYR code, arrival year) from the survey fields. + + ASEC rows read the ``PEINUSYR`` bin midpoint; ACS rows keep their exact + ``YOEP`` year. A row with neither arrives at ``time_period``. + """ + + arrival_code_values = _numeric_evidence_column(person, "PEINUSYR") + invalid_asec_arrival = has_asec & ( + ~np.isfinite(arrival_code_values) + | ~np.equal(arrival_code_values, np.rint(arrival_code_values)) + | (arrival_code_values < 0) + | (arrival_code_values > max(_ARRIVAL_YEAR_MIDPOINTS)) + ) + if invalid_asec_arrival.any(): + bad = sorted(set(arrival_code_values[invalid_asec_arrival].tolist()))[:5] + raise SourceRuntimeError( + "PEINUSYR carries value(s) outside the 2024-2026 ASEC domain " + f"0..{max(_ARRIVAL_YEAR_MIDPOINTS)}: {bad}." + ) + arrival_code = np.zeros(len(person), dtype=np.int64) + arrival_code[has_asec] = arrival_code_values[has_asec].astype(np.int64) + arrival_year = np.full(len(person), time_period, dtype=np.int64) + for code, midpoint in _ARRIVAL_YEAR_MIDPOINTS.items(): + arrival_year[has_asec & (arrival_code == code)] = midpoint + + acs_arrival = _numeric_evidence_column(person, "YOEP") + acs_foreign_born = has_acs & np.isin(citizenship, [4, 5]) + invalid_acs_arrival = acs_foreign_born & ( + ~np.isfinite(acs_arrival) + | ~np.equal(acs_arrival, np.rint(acs_arrival)) + | (acs_arrival < 1900) + | (acs_arrival > time_period) + ) + if invalid_acs_arrival.any(): + raise SourceRuntimeError( + "ACS YOEP must be an integral 1900..time_period year for every " + f"foreign-born CIT=4/5 row; invalid_rows={int(invalid_acs_arrival.sum())}." + ) + observed_acs_arrival = has_acs & np.isfinite(acs_arrival) + arrival_year[observed_acs_arrival] = acs_arrival[observed_acs_arrival].astype( + np.int64 + ) + return has_acs.copy(), arrival_code, arrival_year + + +def us_immigration_evidence_features( + frame: Frame, + *, + time_period: int = 2024, +) -> pd.DataFrame: + """Return finite, source-harmonized immigration predictors per person.""" + + person = frame.table(frame.schema.person_entity) + profile = _source_aware_immigration_profile(person, time_period=time_period) + return pd.DataFrame( + { + "is_us_citizen": profile.is_citizen.astype(np.float64), + "birth_country_code": profile.birth_country.astype(np.float64), + "arrival_year": profile.arrival_year.astype(np.float64), + }, + index=person.index, + ) + + def _stable_person_draws( person: pd.DataFrame, *, @@ -461,13 +1094,20 @@ def _stable_person_draws( """Deterministic uniform draws keyed by stable person identity. Support-channel clones carry their source person's ``source_year`` / - ``source_person_id``, so keying on those gives every clone of one source - person the same draw; frames without source ids fall back to - ``person_id``. A caller whose source ids are not unique per person (the - ACS ``SPORDER`` repeats in every household) passes explicit ``keys``, one - per row. + ``source_household_id`` / ``source_person_id``, so keying on those gives + every clone of one source person the same draw; frames without full source + ids fall back to the legacy source pair or ``person_id``. A caller whose + rows carry no unique source identity (the ACS ``SPORDER`` repeats in every + household) passes explicit ``keys``, one per row. """ + def complete(columns: tuple[str, ...]) -> bool: + return set(columns).issubset(person.columns) and bool( + person.loc[:, list(columns)].notna().to_numpy(dtype=bool).all() + ) + + full_lineage = ("source_year", "source_household_id", "source_person_id") + legacy_lineage = ("source_year", "source_person_id") if keys is not None: if len(keys) != len(person): raise ValueError( @@ -475,14 +1115,28 @@ def _stable_person_draws( f"table has {len(person)}." ) keys = pd.Series(keys).astype(str) - elif {"source_year", "source_person_id"}.issubset(person.columns): + elif complete(full_lineage): keys = ( person["source_year"].astype(str) + ":" + + person["source_household_id"].astype(str) + + ":" + person["source_person_id"].astype(str) ) - else: + elif complete(legacy_lineage): + keys = ( + person["source_year"].astype(str) + + ":" + + person["source_person_id"].astype(str) + ) + elif complete(("person_id",)): keys = person["person_id"].astype(str) + else: + raise SourceRuntimeError( + "US immigration deterministic draws require one complete stable " + "person-lineage alternative: source_year/source_household_id/" + "source_person_id, source_year/source_person_id, or person_id." + ) denominator = float(2**64) return np.fromiter( ( @@ -538,11 +1192,55 @@ def _select_weight_to_target( return selected +def _asec_arrival_year(person: pd.DataFrame, *, time_period: int) -> np.ndarray: + """Arrival year per person from the ASEC ``PEINUSYR`` bins. + + Each code maps to its :data:`_ARRIVAL_YEAR_MIDPOINTS` value: the later + year of a two-year bin (code 8, 1982-1983, maps to 1983) and the middle of + the wider pre-1980 bins. Code 0 (native-born) or any unknown code maps to + ``time_period``, i.e. zero years in the US. + """ + + arrival_code = _integer_column(person, "PEINUSYR") + arrival_year = np.full(len(person), time_period, dtype=np.int64) + for code, midpoint in _ARRIVAL_YEAR_MIDPOINTS.items(): + arrival_year[arrival_code == code] = midpoint + return arrival_year + + +def _arrival_profile( + person: pd.DataFrame, + *, + time_period: int, + arrival_year: np.ndarray | None = None, +) -> tuple[np.ndarray, np.ndarray]: + """(arrival year, age at entry) per person; ``arrival_year`` replaces the bins. + + An exact ``arrival_year`` (NaN where there is none) is read the way the + ASEC bins are: no entry year means ``time_period``. + """ + + if arrival_year is None: + year = _asec_arrival_year(person, time_period=time_period) + else: + exact = np.asarray(arrival_year, dtype=np.float64) + if exact.shape != (len(person),): + raise ValueError( + f"Arrival years cover {exact.shape} row(s); the person table has " + f"{len(person)}." + ) + year = np.where(np.isfinite(exact), exact, float(time_period)).astype(np.int64) + years_in_us = time_period - year + age = _integer_column(person, "A_AGE") + age_at_entry = np.maximum(0, age - years_in_us) + return year, age_at_entry + + def _arrival_indicators( person: pd.DataFrame, *, arrival_year: np.ndarray | None, - time_period: int | None, + time_period: int, ) -> tuple[np.ndarray, np.ndarray, np.ndarray]: """IRCA-cohort, five-plus-year and three-plus-year arrival indicators. @@ -561,8 +1259,6 @@ def _arrival_indicators( np.isin(arrival_code, list(range(8, 27))), np.isin(arrival_code, list(range(8, 28))), ) - if time_period is None: - raise ValueError("An exact arrival year requires the time period.") year = np.asarray(arrival_year, dtype=np.float64) if year.shape != (len(person),): raise ValueError( @@ -579,21 +1275,804 @@ def _arrival_indicators( ) -def _assign_ssn_card_codes( +def _daca_statutory_cohort( + arrival_year: np.ndarray, age_at_entry: np.ndarray, age: np.ndarray +) -> np.ndarray: + return ( + (arrival_year <= _DACA_LATEST_ARRIVAL_YEAR) + & (age_at_entry < _DACA_MAX_AGE_AT_ENTRY) + & (age >= _DACA_MIN_CURRENT_AGE) + ) + + +def _special_status_masks( + *, + ssn_codes: np.ndarray, + birth_country: np.ndarray, + arrival_year: np.ndarray, + age_at_entry: np.ndarray, + age: np.ndarray, +) -> tuple[np.ndarray, np.ndarray]: + """Return final Cuban/Haitian-entrant and DACA cohort masks. + + The source derivation applies DACA after the Cuban/Haitian entrant rule, + so a qualifying EAD holder in both cohorts retains DACA. Keeping that + precedence in one helper lets source derivation, recipient reconciliation, + and release validation share the same exact contract. + """ + + documented_noncitizen = np.isin(ssn_codes, [2, 3]) + daca = (ssn_codes == 2) & _daca_statutory_cohort( + arrival_year, + age_at_entry, + age, + ) + cuban_haitian = ( + documented_noncitizen + & np.isin(birth_country, _CUBAN_HAITIAN_BIRTH_CODES) + & (arrival_year >= _CUBAN_HAITIAN_ARRIVAL_CUTOFF) + & ~daca + ) + return cuban_haitian, daca + + +def _pew_unauthorized_projection_mask( + *, + ssn_codes: np.ndarray, + humanitarian_marks: np.ndarray, + retained_ead_cohort: np.ndarray, +) -> np.ndarray: + """Project final membership in Pew's broad unauthorized universe. + + This helper runs while EAD assignment is still in progress. Residual + ``NONE`` rows count directly; DACA and residual Cuban/Haitian cohort rows + continue to count if an EAD spill gives them a protected engine label; + and the temporary-protection statuses Pew explicitly retains count + regardless of their non-``NONE`` SSN code. + """ + + return ( + (ssn_codes == 0) + | ((ssn_codes == 2) & retained_ead_cohort) + | np.isin( + humanitarian_marks, + _PEW_INCLUDED_HUMANITARIAN_STATUS_VALUES, + ) + ) + + +_SPILL_CLOSURE_RELATIVE_TOLERANCE = 1e-9 + + +def _spill_pew_unauthorized_excess( person: pd.DataFrame, + ssn_codes: np.ndarray, + humanitarian_marks: np.ndarray, weights: np.ndarray, *, + noncitizens: np.ndarray, + scope: np.ndarray, + preserve_scope: np.ndarray, + retained_ead_cohort: np.ndarray, + target: float, seed: int, - controls: UndocumentedControls, + salt: str, + draw_keys: pd.Series | None = None, +) -> None: + """Spill a broad Pew-universe margin to EAD in deterministic order. + + Rows outside the other controlled margin are exhausted before any + overlapping row is touched, so the student and worker controls disturb + one another only when their overlap makes that unavoidable. Within each + of those two tiers the first draw preserves the prior EAD/DACA allocation + behavior by selecting from the whole residual scope; a selected DACA or + residual Cuban/Haitian cohort row still belongs to Pew's universe, so a + corrective draw then spills rows outside those retained cohorts before + the next tier is considered. + """ + + draws = _stable_person_draws(person, seed=seed, salt=salt, keys=draw_keys) + # A selection that closes the gap exactly can leave floating-point + # residue in the recomputed sum (it adds the weights in a different order + # than the selection's cumulative sum). Treat anything within a + # billionth of the in-scope weight as closed, so residue never opens the + # next tier and spills a row the ordering promised to keep. + closed_within = _SPILL_CLOSURE_RELATIVE_TOLERANCE * max( + 1.0, float(weights[scope].sum()) + ) + + def current_excess() -> float: + included = _pew_unauthorized_projection_mask( + ssn_codes=ssn_codes, + humanitarian_marks=humanitarian_marks, + retained_ead_cohort=retained_ead_cohort, + ) + return float(weights[scope & included].sum()) - target + + # Exhaust each tier before the next: a row inside the other margin is + # spilled only once no row outside it can close the gap, because that + # margin's own pass cannot restore a row this one removed. + for priority in (~preserve_scope, preserve_scope): + # Preserve the seeded EAD allocation surface. DACA and residual + # Cuban/Haitian cohort members keep protected engine labels after an + # EAD and so remain inside Pew's estimate; only rows outside those + # cohorts reduce the count, which the corrective draw supplies. + for retained_allowed in (True, False): + if current_excess() <= closed_within: + return + candidates = (ssn_codes == 0) & noncitizens & scope & priority + if not retained_allowed: + candidates &= ~retained_ead_cohort + selected = _select_weight_to_target( + candidates, + weights, + draws, + current_excess(), + ) + ssn_codes[selected] = 2 + + +def _humanitarian_draw_candidates( + draw: HumanitarianDraw, + *, + ssn_codes: np.ndarray, + profile: _ImmigrationEvidenceProfile, +) -> np.ndarray: + """Candidate mask for one draw, before exclusions shared by all draws. + + ``REFUGEE``/``ASYLEE``/``DEPORTATION_WITHHELD`` draw from the + indicator-documented pool only (code 3: those statuses carry immediate + federal benefits access, the same signals the ASEC-UA method reads); + parole and TPS may also draw from the still-unspilled residual pool + (code 0), whose members then receive EAD-backed SSN cards. + """ + + noncitizen = ~profile.is_citizen + excluded = np.isin(profile.birth_country, _CUBAN_HAITIAN_BIRTH_CODES) + excluded |= _daca_statutory_cohort( + profile.arrival_year, + profile.age_at_entry, + profile.age, + ) + + if draw.category == "paroled_one_year": + origins = ( + (draw.origin,) if draw.origin is not None else tuple(_PAROLE_ORIGIN_CODES) + ) + cohort = np.zeros(len(ssn_codes), dtype=bool) + for origin in origins: + if origin not in _PAROLE_ORIGIN_CODES: + raise SourceRuntimeError( + f"US immigration has no parole origin rule for {origin!r}." + ) + origin_match = np.isin( + profile.birth_country, + _PAROLE_ORIGIN_CODES[origin], + ) + asec_window = (~profile.exact_arrival) & np.isin( + profile.arrival_code, + _PAROLE_ASEC_ARRIVAL_CODES[origin], + ) + acs_window = profile.exact_arrival & ( + profile.arrival_year >= _PAROLE_ACS_MIN_ARRIVAL_YEAR[origin] + ) + if origin in _PAROLE_ACS_MAX_ARRIVAL_YEAR: + acs_window &= ( + profile.arrival_year <= _PAROLE_ACS_MAX_ARRIVAL_YEAR[origin] + ) + cohort |= origin_match & (asec_window | acs_window) + return noncitizen & np.isin(ssn_codes, (0, 3)) & cohort & ~excluded + if draw.category == "refugee": + return ( + noncitizen + & (ssn_codes == 3) + & np.isin(profile.birth_country, _REFUGEE_ORIGIN_CODES) + & ( + ( + (~profile.exact_arrival) + & np.isin(profile.arrival_code, _POST_2021_ARRIVAL_CODES) + ) + | (profile.exact_arrival & (profile.arrival_year >= 2022)) + ) + & ~excluded + ) + if draw.category == "asylee": + return ( + noncitizen + & (ssn_codes == 3) + & np.isin(profile.birth_country, _ASYLEE_ORIGIN_CODES) + & ( + ( + (~profile.exact_arrival) + & np.isin(profile.arrival_code, _ASYLEE_ARRIVAL_CODES) + ) + | (profile.exact_arrival & (profile.arrival_year >= 2016)) + ) + & ~excluded + ) + if draw.category == "deportation_withheld": + return ( + noncitizen + & (ssn_codes == 3) + & ( + ( + (~profile.exact_arrival) + & (profile.arrival_code >= 1) + & (profile.arrival_code <= _WITHHELD_MAX_ARRIVAL_CODE) + ) + | (profile.exact_arrival & (profile.arrival_year <= 2019)) + ) + & ~excluded + ) + if draw.category == "tps": + origins = ( + (draw.origin,) if draw.origin is not None else tuple(_TPS_ORIGIN_CODES) + ) + cohort = np.zeros(len(ssn_codes), dtype=bool) + for origin in origins: + if origin not in _TPS_ORIGIN_CODES: + raise SourceRuntimeError( + f"US immigration has no TPS origin rule for {origin!r}." + ) + codes, max_arrival_code = _TPS_ORIGIN_CODES[origin] + origin_match = np.isin(profile.birth_country, codes) + asec_window = ( + (~profile.exact_arrival) + & (profile.arrival_code >= 1) + & (profile.arrival_code <= max_arrival_code) + ) + acs_window = ( + profile.exact_arrival + & (profile.tps_acs_max_arrival_year >= 0) + & (profile.arrival_year <= profile.tps_acs_max_arrival_year) + ) + cohort |= origin_match & (asec_window | acs_window) + return noncitizen & np.isin(ssn_codes, (0, 3)) & cohort & ~excluded + raise SourceRuntimeError( + f"US immigration derivation has no candidate rule for draw {draw.label!r}." + ) + + +def us_immigration_humanitarian_draw_mask( + frame: Frame, + draw: HumanitarianDraw, + *, + time_period: int = 2024, +) -> np.ndarray: + """Rows emitted as ``draw`` whose source evidence supports that label. + + This is the shared hard-cohort contract for transfer reconciliation, + calibration, and the release gate. It intentionally includes the emitted + status test: callers receive the achieved population for one manifest + draw, not the larger pool of possible candidates. + """ + + person = frame.table(frame.schema.person_entity) + missing = sorted(set(US_IMMIGRATION_OUTPUT_COLUMNS) - set(person.columns)) + if missing: + raise ValueError( + f"US humanitarian draw compatibility requires output column(s) {missing}." + ) + profile = _source_aware_immigration_profile(person, time_period=time_period) + return _humanitarian_emitted_mask(person, draw=draw, profile=profile) + + +def us_immigration_humanitarian_transfer_selection_masks( + frame: Frame, + *, + mutable_rows: np.ndarray, + seed: int, + time_period: int = 2024, + controls: ImmigrationControls | None = None, +) -> dict[str, np.ndarray]: + """Replay the exact deterministic mutable-row selection for every draw. + + The post-transfer frame retains everything needed to reconstruct the + reconciliation decision: source evidence, final SSN codes, stable person + lineage, resolved person weights, manifest controls, and draw order. For + parole and TPS, final EAD codes are mapped back to the residual-pool code + exactly as they are during reconciliation. Earlier expected selections, + rather than observed status labels, exclude rows from later draws, so a + downstream equal-mass status swap cannot redefine the replayed candidate + surface. + """ + + person = frame.table(frame.schema.person_entity) + missing = sorted(set(US_IMMIGRATION_OUTPUT_COLUMNS) - set(person.columns)) + if missing: + raise ValueError( + "Humanitarian transfer selection replay requires person column(s) " + f"{missing}." + ) + mutable = np.asarray(mutable_rows, dtype=bool) + if mutable.shape != (len(person),): + raise ValueError( + "Humanitarian transfer selection replay mutable_rows must align " + "one-to-one with the person table." + ) + weights = np.asarray( + frame.resolve_weights(frame.schema.person_entity).values, + dtype=np.float64, + ) + if ( + weights.shape != (len(person),) + or not np.isfinite(weights).all() + or (weights < 0).any() + ): + raise ValueError( + "Humanitarian transfer selection replay requires finite " + "non-negative person weights." + ) + controls = us_immigration_controls() if controls is None else controls + profile = _source_aware_immigration_profile(person, time_period=time_period) + ssn_codes = _ssn_name_codes(person) + immutable = ~mutable + selected_once = np.zeros(len(person), dtype=bool) + selections: dict[str, np.ndarray] = {} + for draw in controls.humanitarian: + immutable_draw = ( + _humanitarian_emitted_mask( + person, + draw=draw, + profile=profile, + ) + & immutable + ) + residual_target = max( + 0.0, + float(draw.target) - float(weights[immutable_draw].sum()), + ) + candidate_ssn = ssn_codes + if draw.category in {"paroled_one_year", "tps"}: + candidate_ssn = np.where(candidate_ssn == 2, 0, candidate_ssn) + candidates = ( + _humanitarian_draw_candidates( + draw, + ssn_codes=candidate_ssn, + profile=profile, + ) + & mutable + & ~selected_once + ) + stable_draws = _stable_person_draws( + person, + seed=seed, + salt=f"acs_transfer:{draw.salt}", + ) + selected = _select_weight_to_target( + candidates, + weights, + stable_draws, + residual_target, + ) + selections[draw.label] = selected + selected_once |= selected + return selections + + +def _ssn_name_codes(person: pd.DataFrame) -> np.ndarray: + ssn_lookup = { + "NONE": 0, + "CITIZEN": 1, + "NON_CITIZEN_VALID_EAD": 2, + "OTHER_NON_CITIZEN": 3, + } + return ( + person["ssn_card_type"] + .astype(str) + .map(ssn_lookup) + .fillna(-1) + .to_numpy(dtype=np.int64) + ) + + +def _humanitarian_emitted_mask( + person: pd.DataFrame, + *, + draw: HumanitarianDraw, + profile: _ImmigrationEvidenceProfile, +) -> np.ndarray: + ssn_codes = _ssn_name_codes(person) + candidates = _humanitarian_draw_candidates( + draw, + ssn_codes=ssn_codes, + profile=profile, + ) + # Parole and TPS selections from the residual pool are upgraded from NONE + # to EAD after candidacy is evaluated. Their emitted rows may therefore + # carry code 2 even though only code 0/3 was eligible before selection. + if draw.category in {"paroled_one_year", "tps"}: + candidates |= _humanitarian_draw_candidates( + draw, + ssn_codes=np.where(ssn_codes == 2, 0, ssn_codes), + profile=profile, + ) & (ssn_codes == 2) + status = person["immigration_status_str"].astype(str).to_numpy() + return np.asarray(candidates & (status == draw.status), dtype=bool) + + +def reconcile_us_immigration_humanitarian_transfer( + person: pd.DataFrame, + *, + weights: np.ndarray, + mutable_rows: np.ndarray, + seed: int, + time_period: int = 2024, + controls: ImmigrationControls | None = None, +) -> tuple[pd.DataFrame, dict[str, object]]: + """Reconcile a transferred immigration pair to manifest draw targets. + + Immutable rows carry the ASEC source contribution; the reconciliation + repairs or selects only rows whose paired target cells the transfer + imputed. In manifest order, each draw asks those rows for the residual of + its national target after its compatible, immutable contribution, and + only rows its origin, arrival and SSN rules admit can take its label. + When the admitted rows hold less weight than the residual, the draw takes + every one of them, as the ASEC stage's own draws do, and its receipt + records the shortfall. The reconciliation refuses only a draw that then + reaches less than the floor of the composition gate's band + (``_HUMANITARIAN_TARGET_RELATIVE_BAND``) of its target. + """ + + result = person.copy() + missing = sorted(set(US_IMMIGRATION_OUTPUT_COLUMNS) - set(result.columns)) + if missing: + raise ValueError( + f"Humanitarian transfer reconciliation requires person column(s) {missing}." + ) + weights = np.asarray(weights, dtype=np.float64) + mutable = np.asarray(mutable_rows, dtype=bool) + if weights.shape != (len(result),) or mutable.shape != (len(result),): + raise ValueError( + "Humanitarian transfer weights and mutable_rows must align one-to-one " + "with the person table." + ) + if not np.isfinite(weights).all() or (weights < 0).any(): + raise ValueError( + "Humanitarian transfer reconciliation requires finite non-negative " + "person weights." + ) + controls = us_immigration_controls() if controls is None else controls + profile = _source_aware_immigration_profile(result, time_period=time_period) + original_ssn = result["ssn_card_type"].astype(str).to_numpy(copy=True) + original_status = result["immigration_status_str"].astype(str).to_numpy(copy=True) + ssn = original_ssn.copy() + status = original_status.copy() + valid_ssn = np.isin(ssn, SSN_CARD_TYPE_VALUES) + valid_status = np.isin(status, IMMIGRATION_STATUS_VALUES) + if (~valid_ssn).any() or (~valid_status).any(): + raise ValueError( + "Humanitarian transfer reconciliation received values outside the " + "PolicyEngine SSN/immigration enum domains." + ) + + humanitarian_statuses = set(_HUMANITARIAN_STATUS_BY_CATEGORY.values()) + evidence_derived_statuses = {"CUBAN_HAITIAN_ENTRANT", "DACA"} + constrained_statuses = humanitarian_statuses | evidence_derived_statuses + baseline_constrained = mutable & np.isin(status, list(constrained_statuses)) + if baseline_constrained.any(): + raise ValueError( + "ACS joint-QRF baseline emitted evidence-constrained labels before " + "reconciliation." + ) + + # Source citizenship is observed on both surveys. Normalize only mutable + # pairs; immutable ASEC disagreements remain an error below. + citizen_mutable = mutable & profile.is_citizen + ssn[citizen_mutable] = "CITIZEN" + status[citizen_mutable] = "CITIZEN" + noncitizen_mutable = mutable & ~profile.is_citizen + ssn[noncitizen_mutable & (ssn == "CITIZEN")] = "OTHER_NON_CITIZEN" + status[noncitizen_mutable & (status == "CITIZEN")] = "LEGAL_PERMANENT_RESIDENT" + status[noncitizen_mutable & (ssn == "NONE")] = "UNDOCUMENTED" + status[noncitizen_mutable & (ssn != "NONE") & (status == "UNDOCUMENTED")] = ( + "LEGAL_PERMANENT_RESIDENT" + ) + citizenship_repairs = int( + ( + mutable + & ((ssn != original_ssn) | (status != original_status)) + & ( + profile.is_citizen + | (original_ssn == "CITIZEN") + | (original_status == "CITIZEN") + ) + ).sum() + ) + pair_repairs = int( + (mutable & ((ssn != original_ssn) | (status != original_status))).sum() + ) + result["ssn_card_type"] = ssn + result["immigration_status_str"] = status + + # CHE and DACA have hard source-evidence rules just like the manifest + # draws, but no stock target. Rebuild them deterministically after the + # unconstrained codec and citizenship/pair repair rather than accepting a + # donor label on an incompatible ACS cohort. + special_ssn_codes = _ssn_name_codes(result) + cuban_haitian, daca = _special_status_masks( + ssn_codes=special_ssn_codes, + birth_country=profile.birth_country, + arrival_year=profile.arrival_year, + age_at_entry=profile.age_at_entry, + age=profile.age, + ) + special_before = status.copy() + status[mutable & cuban_haitian] = "CUBAN_HAITIAN_ENTRANT" + status[mutable & daca] = "DACA" + special_status_assignments = int((mutable & (status != special_before)).sum()) + result["immigration_status_str"] = status + + immutable = ~mutable + immutable_pair_invalid = immutable & ( + ((ssn == "CITIZEN") != (status == "CITIZEN")) + | ((ssn == "NONE") != (status == "UNDOCUMENTED")) + | ((ssn == "CITIZEN") != profile.is_citizen) + ) + if immutable_pair_invalid.any(): + raise ValueError( + "Immutable ASEC immigration rows violate citizenship/paired-status " + f"invariants; invalid_rows={int(immutable_pair_invalid.sum())}." + ) + immutable_special_invalid = immutable & ( + ((status == "CUBAN_HAITIAN_ENTRANT") != cuban_haitian) + | ((status == "DACA") != daca) + ) + if immutable_special_invalid.any(): + raise ValueError( + "Immutable ASEC immigration rows violate Cuban/Haitian entrant or " + f"DACA cohort invariants; invalid_rows={int(immutable_special_invalid.sum())}." + ) + + selected_once = np.zeros(len(result), dtype=bool) + compatible_humanitarian = np.zeros(len(result), dtype=bool) + draw_receipts: dict[str, object] = {} + largest_target = max((draw.target for draw in controls.humanitarian), default=0.0) + tolerance = max(1e-6, largest_target * 1e-12) + band_floor = _HUMANITARIAN_TARGET_RELATIVE_BAND[0] + for draw in controls.humanitarian: + immutable_draw = ( + _humanitarian_emitted_mask( + result, + draw=draw, + profile=profile, + ) + & immutable + ) + compatible_humanitarian |= immutable_draw + immutable_population = float(weights[immutable_draw].sum()) + if draw.target <= 0 and immutable_population > tolerance: + raise ValueError( + f"Immutable ASEC rows emit {immutable_population:,.6f} weighted " + f"persons for explicit-zero humanitarian draw {draw.label!r}." + ) + residual_target = max(0.0, float(draw.target) - immutable_population) + + candidate_ssn = _ssn_name_codes(result) + if draw.category in {"paroled_one_year", "tps"}: + candidate_ssn = np.where(candidate_ssn == 2, 0, candidate_ssn) + candidates = ( + _humanitarian_draw_candidates( + draw, + ssn_codes=candidate_ssn, + profile=profile, + ) + & mutable + & ~selected_once + ) + available_population = float(weights[candidates].sum()) + # Short of candidates, the selection below takes every one of them. + exhausted = available_population + tolerance < residual_target + stable_draws = _stable_person_draws( + result, + seed=seed, + salt=f"acs_transfer:{draw.salt}", + ) + selected = _select_weight_to_target( + candidates, + weights, + stable_draws, + residual_target, + ) + if exhausted and not np.array_equal(selected, candidates): + raise RuntimeError( + f"Humanitarian transfer selection for {draw.label!r} left " + "candidates unselected although they hold less than its residual." + ) + selected_population = float(weights[selected].sum()) + if not exhausted and selected_population + tolerance < residual_target: + raise RuntimeError( + f"Humanitarian transfer selection underfilled {draw.label!r}." + ) + status[selected] = draw.status + if draw.category in {"paroled_one_year", "tps"}: + ssn[selected & (ssn == "NONE")] = "NON_CITIZEN_VALID_EAD" + selected_once |= selected + result["ssn_card_type"] = ssn + result["immigration_status_str"] = status + achieved_mask = _humanitarian_emitted_mask( + result, + draw=draw, + profile=profile, + ) + compatible_humanitarian |= achieved_mask + achieved_population = float(weights[achieved_mask].sum()) + # An exhausted draw crosses no threshold: it takes every candidate. + threshold_tie_population = 0.0 + if selected.any() and not exhausted: + threshold = float(np.max(stable_draws[selected])) + threshold_tie_population = float( + weights[candidates & (stable_draws == threshold)].sum() + ) + residual_selection_error = abs(selected_population - residual_target) + if ( + not exhausted + and residual_selection_error > threshold_tie_population + tolerance + ): + raise RuntimeError( + f"Humanitarian transfer residual selection for {draw.label!r} " + "missed its target by more than the threshold tie mass." + ) + candidate_shortfall = ( + residual_target - available_population if exhausted else 0.0 + ) + if exhausted and achieved_population + tolerance < band_floor * float( + draw.target + ): + raise ValueError( + "Humanitarian transfer candidate shortfall for " + f"{draw.label!r} leaves it below {band_floor} of its target, " + "the floor of the composition gate's band: " + f"target={float(draw.target):,.6f}, " + f"achieved_population={achieved_population:,.6f}, " + f"residual_target={residual_target:,.6f}, " + f"eligible_recipient_population={available_population:,.6f}, " + f"immutable_population={immutable_population:,.6f}." + ) + absolute_error = abs(achieved_population - float(draw.target)) + immutable_overshoot = max(0.0, immutable_population - float(draw.target)) + discrete_bound = immutable_overshoot + threshold_tie_population + tolerance + if not exhausted and absolute_error > discrete_bound: + raise RuntimeError( + f"Humanitarian transfer reconciliation for {draw.label!r} " + "missed its target by more than the discrete selection bound." + ) + draw_receipts[draw.label] = { + "target": float(draw.target), + "immutable_population": immutable_population, + "residual_target": residual_target, + "eligible_recipient_population": available_population, + "selected_recipient_population": selected_population, + "achieved_population": achieved_population, + "absolute_error": absolute_error, + "selection_threshold_tie_population": threshold_tie_population, + "residual_selection_error": residual_selection_error, + "within_residual_discrete_weight_bound": not exhausted, + "immutable_overshoot": immutable_overshoot, + "within_discrete_weight_bound": not exhausted, + "candidates_exhausted": exhausted, + "candidate_shortfall_population": candidate_shortfall, + } + + result["ssn_card_type"] = ssn + result["immigration_status_str"] = status + all_humanitarian = np.isin(status, list(humanitarian_statuses)) + incompatible_humanitarian = all_humanitarian & ~compatible_humanitarian + if incompatible_humanitarian.any(): + raise ValueError( + "Humanitarian transfer output contains status/origin/arrival/SSN " + f"incompatible rows: {int(incompatible_humanitarian.sum())}." + ) + final_ssn_codes = _ssn_name_codes(result) + final_cuban_haitian, final_daca = _special_status_masks( + ssn_codes=final_ssn_codes, + birth_country=profile.birth_country, + arrival_year=profile.arrival_year, + age_at_entry=profile.age_at_entry, + age=profile.age, + ) + special_status_invalid = ( + (status == "CUBAN_HAITIAN_ENTRANT") != final_cuban_haitian + ) | ((status == "DACA") != final_daca) + if special_status_invalid.any(): + raise RuntimeError( + "Humanitarian transfer reconciliation left Cuban/Haitian entrant " + "or DACA cohort invariants invalid on " + f"{int(special_status_invalid.sum())} row(s)." + ) + final_pair_invalid = ( + ((ssn == "CITIZEN") != (status == "CITIZEN")) + | ((ssn == "NONE") != (status == "UNDOCUMENTED")) + | ((ssn == "CITIZEN") != profile.is_citizen) + ) + if final_pair_invalid.any(): + raise RuntimeError( + "Humanitarian transfer reconciliation left citizenship/paired-status " + f"invariants invalid on {int(final_pair_invalid.sum())} row(s)." + ) + if not np.array_equal( + result.loc[immutable, "ssn_card_type"].astype(str).to_numpy(), + original_ssn[immutable], + ) or not np.array_equal( + result.loc[immutable, "immigration_status_str"].astype(str).to_numpy(), + original_status[immutable], + ): + raise RuntimeError( + "Humanitarian transfer reconciliation changed immutable ASEC rows." + ) + receipt: dict[str, object] = { + "kind": "deterministic_humanitarian_residual_target", + "seed": int(seed), + "time_period": int(time_period), + "mutable_rows": int(mutable.sum()), + "immutable_rows": int(immutable.sum()), + "citizenship_repairs": citizenship_repairs, + "pair_repairs": pair_repairs, + "special_status_assignments": special_status_assignments, + "floating_tolerance": tolerance, + "selection_order": [draw.label for draw in controls.humanitarian], + "draws": draw_receipts, + } + return result, receipt + + +def _assign_humanitarian_statuses( + person: pd.DataFrame, + ssn_codes: np.ndarray, + weights: np.ndarray, + *, + seed: int, + controls: ImmigrationControls, + time_period: int, arrival_year: np.ndarray | None = None, - time_period: int | None = None, draw_keys: pd.Series | None = None, ) -> np.ndarray: - """SSN card codes (0 NONE, 1 CITIZEN, 2 EAD, 3 OTHER) for CPS-named rows. + """Mark humanitarian statuses and upgrade residual draws' SSN codes. + + Runs after the legal-status indicators and before the EAD worker/student + spill. Pew-included temporary protections remain inside the broad control + universe even after receiving EADs. Draws are sequential over + ``controls.humanitarian``; a person marked by an earlier draw is out of + every later candidate pool. Residual-pool selections (parole/TPS only) + move to ``NON_CITIZEN_VALID_EAD`` — both programs confer employment + authorization — keeping the ``NONE`` ⇔ ``UNDOCUMENTED`` invariant. + """ + + marks = np.full(len(person), "", dtype="U24") + profile = _source_aware_immigration_profile( + person, time_period=time_period, arrival_year=arrival_year + ) + for draw in controls.humanitarian: + if draw.target <= 0: + continue + candidates = _humanitarian_draw_candidates( + draw, + ssn_codes=ssn_codes, + profile=profile, + ) & (marks == "") + draws = _stable_person_draws(person, seed=seed, salt=draw.salt, keys=draw_keys) + selected = _select_weight_to_target(candidates, weights, draws, draw.target) + marks[selected] = draw.status + ssn_codes[selected & (ssn_codes == 0)] = 2 + return marks - ``arrival_year``/``time_period`` replace the ``PEINUSYR`` bins with an - exact entry year, and ``draw_keys`` replaces the source-identity draw - keys; left unset, the ASEC behaviour is unchanged. + +def _assign_ssn_card_codes( + person: pd.DataFrame, + weights: np.ndarray, + *, + seed: int, + controls: ImmigrationControls, + time_period: int, + arrival_year: np.ndarray | None = None, + draw_keys: pd.Series | None = None, +) -> tuple[np.ndarray, np.ndarray]: + """SSN codes (0 NONE, 1 CITIZEN, 2 EAD, 3 OTHER) and humanitarian marks. + + ``arrival_year`` replaces the ``PEINUSYR`` bins with an exact entry year + (NaN where there is none), and ``draw_keys`` replaces the source-identity + draw keys; left unset, the ASEC behaviour is unchanged. The marks are the + humanitarian status names, ``""`` for unmarked rows. """ citizenship = _integer_column(person, "PRCITSHP") @@ -654,61 +2133,90 @@ def _assign_ssn_card_codes( ) ssn_codes[(ssn_codes == 0) & assumed_documented] = 3 - is_worker = (_float_column(person, "WSAL_VAL") > 0) | ( - _float_column(person, "SEMP_VAL") > 0 + # Humanitarian draws run between the indicators and the EAD spill. Pew + # retains temporary protection in its unauthorized estimate, so those + # rows contribute to the broad controls even though they receive EADs. + humanitarian_marks = _assign_humanitarian_statuses( + person, + ssn_codes, + weights, + seed=seed, + controls=controls, + time_period=time_period, + arrival_year=arrival_year, + draw_keys=draw_keys, ) - is_student = _integer_column(person, "A_HSCOL") == 2 - # The CPS has no work-authorization variable, so the EAD-vs-unauthorized - # split inside the residual pool is unidentified from the survey. Spill - # excess undocumented workers/students to EAD so the *remaining* counts - # match their published controls; counts already at or below a control - # spill nothing. The total undocumented population is emergent — the - # composition gate checks it against a cited anchor, and reconciling the - # level is the calibration lane's job, not this label stage's. - worker_candidates = (ssn_codes == 0) & noncitizens & is_worker - worker_excess = float(weights[worker_candidates].sum()) - controls.workers - worker_draws = _stable_person_draws( - person, seed=seed, salt="immigration:ead_workers", keys=draw_keys + labor_force_status = _integer_column(person, "A_LFSR") + invalid_labor_force_status = ~np.isin( + labor_force_status, + _CPS_LABOR_FORCE_STATUS_DOMAIN, ) - ssn_codes[ - _select_weight_to_target( - worker_candidates, weights, worker_draws, worker_excess + if invalid_labor_force_status.any(): + bad = sorted(set(labor_force_status[invalid_labor_force_status].tolist()))[:5] + raise SourceRuntimeError( + "A_LFSR carries value(s) outside the CPS ASEC domain " + f"{list(_CPS_LABOR_FORCE_STATUS_DOMAIN)}: {bad}." ) - ] = 2 - - student_candidates = (ssn_codes == 0) & noncitizens & is_student - student_excess = float(weights[student_candidates].sum()) - controls.students - student_draws = _stable_person_draws( - person, seed=seed, salt="immigration:ead_students", keys=draw_keys + is_worker = (age >= 16) & np.isin( + labor_force_status, + _CPS_LABOR_FORCE_STATUS_CODES, ) - ssn_codes[ - _select_weight_to_target( - student_candidates, weights, student_draws, student_excess - ) - ] = 2 - return ssn_codes - - -def _asec_arrival_year(person: pd.DataFrame, *, time_period: int) -> np.ndarray: - """Arrival year per person from the ASEC ``PEINUSYR`` bins. - - Each code maps to its :data:`_ARRIVAL_YEAR_MIDPOINTS` value: the later - year of a two-year bin (code 8, 1982-1983, maps to 1983) and the middle of - the wider pre-1980 bins. Code 0 (native-born) or any unknown code maps to - ``time_period``, i.e. zero years in the US. - """ + is_student = _integer_column(person, "A_HSCOL") == 2 - arrival_code = _integer_column(person, "PEINUSYR") - arrival_year = np.full(len(person), time_period, dtype=np.int64) - for code, midpoint in _ARRIVAL_YEAR_MIDPOINTS.items(): - arrival_year[arrival_code == code] = midpoint - return arrival_year + entry_year, age_at_entry = _arrival_profile( + person, time_period=time_period, arrival_year=arrival_year + ) + daca_cohort = _daca_statutory_cohort(entry_year, age_at_entry, age) + cuban_haitian_cohort = ( + np.isin(_integer_column(person, "PENATVTY"), _CUBAN_HAITIAN_BIRTH_CODES) + & (entry_year >= _CUBAN_HAITIAN_ARRIVAL_CUTOFF) + & ~daca_cohort + ) + retained_ead_cohort = daca_cohort | cuban_haitian_cohort + + # The CPS has no work-authorization variable, so the EAD split inside the + # residual pool is unidentified. The two controls bind the same broad + # unauthorized universe their publishers report: residual undocumented + # people plus DACA and Pew-included temporary protections. Student spill + # runs first; the worker spill runs last and is therefore authoritative + # for Pew's published labor-force total. Each pass prefers rows outside + # the other margin so their overlap is disturbed only when unavoidable. + _spill_pew_unauthorized_excess( + person, + ssn_codes, + humanitarian_marks, + weights, + noncitizens=noncitizens, + scope=is_student, + preserve_scope=is_worker, + retained_ead_cohort=retained_ead_cohort, + target=controls.undocumented.students, + seed=seed, + salt="immigration:ead_students", + draw_keys=draw_keys, + ) + _spill_pew_unauthorized_excess( + person, + ssn_codes, + humanitarian_marks, + weights, + noncitizens=noncitizens, + scope=is_worker, + preserve_scope=is_student, + retained_ead_cohort=retained_ead_cohort, + target=controls.undocumented.workers, + seed=seed, + salt="immigration:ead_workers", + draw_keys=draw_keys, + ) + return ssn_codes, humanitarian_marks def _derive_immigration_status( person: pd.DataFrame, ssn_codes: np.ndarray, + humanitarian_marks: np.ndarray, *, time_period: int, arrival_year: np.ndarray | None = None, @@ -719,14 +2227,10 @@ def _derive_immigration_status( ASEC bins are: no entry year means ``time_period``. """ - if arrival_year is None: - arrival_year = _asec_arrival_year(person, time_period=time_period) - else: - exact = np.asarray(arrival_year, dtype=np.float64) - arrival_year = np.where(np.isfinite(exact), exact, time_period).astype(np.int64) - years_in_us = time_period - arrival_year + entry_year, age_at_entry = _arrival_profile( + person, time_period=time_period, arrival_year=arrival_year + ) age = _integer_column(person, "A_AGE") - age_at_entry = np.maximum(0, age - years_in_us) birth_country = _integer_column(person, "PENATVTY") # Documented non-citizens default to LPR — the modal true status — and @@ -737,21 +2241,21 @@ def _derive_immigration_status( status[ssn_codes == 1] = "CITIZEN" status[ssn_codes == 0] = "UNDOCUMENTED" - documented_noncitizen = np.isin(ssn_codes, [2, 3]) - cuban_haitian = ( - documented_noncitizen - & np.isin(birth_country, _CUBAN_HAITIAN_BIRTH_CODES) - & (arrival_year >= _CUBAN_HAITIAN_ARRIVAL_CUTOFF) + cuban_haitian, daca = _special_status_masks( + ssn_codes=ssn_codes, + birth_country=birth_country, + arrival_year=entry_year, + age_at_entry=age_at_entry, + age=age, ) status[cuban_haitian] = "CUBAN_HAITIAN_ENTRANT" - - daca = ( - (ssn_codes == 2) - & (arrival_year <= _DACA_LATEST_ARRIVAL_YEAR) - & (age_at_entry < _DACA_MAX_AGE_AT_ENTRY) - & (age >= _DACA_MIN_CURRENT_AGE) - ) status[daca] = "DACA" + + # Humanitarian draws exclude Cuba/Haiti-born and the DACA cohort, so + # the marks are disjoint from both tags; every marked person carries a + # non-NONE SSN code, so the columns still agree about the undocumented. + marked = humanitarian_marks != "" + status[marked] = humanitarian_marks[marked] return status @@ -760,6 +2264,7 @@ def with_us_immigration_inputs( *, seed: int, time_period: int, + person_weight_scale: float = 1.0, ) -> Frame: """Run the ``immigration_status`` manifest stage over a US frame. @@ -771,6 +2276,11 @@ def with_us_immigration_inputs( CPS ASEC source columns. seed: Build-wide imputation seed. time_period: The dataset's time period (arrival-year arithmetic). + person_weight_scale: Stage-only multiplier for person design weights. + Pooled source projections use the inverse of their share of the + full pool's person mass so absolute national controls contribute + only that source's mass share. Standalone frames keep the default + multiplier of one. Returns: A new frame whose person table carries ``ssn_card_type`` and @@ -785,6 +2295,15 @@ def with_us_immigration_inputs( if frame.schema != US_SCHEMA: raise ValueError("US immigration inputs require the US schema.") + if ( + isinstance(person_weight_scale, bool) + or not np.isfinite(person_weight_scale) + or person_weight_scale <= 0 + ): + raise ValueError( + "US immigration person_weight_scale must be a finite positive " + f"number, got {person_weight_scale!r}." + ) person = frame.table("person") present = [ column for column in US_IMMIGRATION_OUTPUT_COLUMNS if column in person.columns @@ -799,7 +2318,9 @@ def with_us_immigration_inputs( ) stage_person = person.copy(deep=True) - stage_person[_PERSON_WEIGHT_COLUMN] = frame.resolve_weights("person").values + stage_person[_PERSON_WEIGHT_COLUMN] = np.asarray( + frame.resolve_weights("person").values, dtype=np.float64 + ) * float(person_weight_scale) output = run_source_stage( us_immigration_stage_spec(), tables={"person": stage_person}, @@ -829,6 +2350,22 @@ def with_us_immigration_inputs( ) +def _pew_unauthorized_rows(person: pd.DataFrame) -> np.ndarray: + """Rows inside Pew's broad unauthorized universe, from emitted labels. + + The engine statuses Pew retains (:data:`_PEW_UNAUTHORIZED_STATUS_VALUES`) + plus ``CUBAN_HAITIAN_ENTRANT`` rows holding an EAD, which only an EAD + spill from the residual pool produces. This is the universe the worker + and student controls bind and the population anchor grades. + """ + + status = person["immigration_status_str"].astype(str).to_numpy() + ssn = person["ssn_card_type"].astype(str).to_numpy() + return np.isin(status, _PEW_UNAUTHORIZED_STATUS_VALUES) | ( + (status == "CUBAN_HAITIAN_ENTRANT") & (ssn == "NON_CITIZEN_VALID_EAD") + ) + + def us_immigration_composition_summary(frame: Frame) -> dict[str, object]: """Weighted SSN-card-type and immigration-status composition of a frame.""" @@ -858,7 +2395,8 @@ def us_immigration_composition_summary(frame: Frame) -> dict[str, object]: def us_immigration_composition_gate( frame: Frame, *, - controls: UndocumentedControls | None = None, + time_period: int, + controls: ImmigrationControls | None = None, ) -> GateResult: """Release gate: the SSN/immigration surface exists and is plausible. @@ -866,12 +2404,20 @@ def us_immigration_composition_gate( mode: everyone a citizen with a valid SSN), when a value falls outside the engine enum domain, when the two columns disagree about citizenship or undocumented status, when the weighted non-citizen share leaves its - plausibility band, or when the emergent undocumented population strays - outside a coarse band around its cited published anchor. + plausibility band, when the emergent Pew-defined unauthorized population + strays outside a coarse band around its cited published anchor, or when a + humanitarian category's emitted mass leaves the coarse band around its + cited stock target (microcosm #767 — the H.R.1 §71109/§71301/§71302 and + SNAP §10108 eligibility channels all bind through these categories; an + explicit zero target must emit exactly zero). + + ``time_period`` must be the period the stage ran for: the source-aware + cohort evidence (the DACA arrival windows in particular) is re-derived at + that period, exactly as ``with_us_immigration_inputs`` derived it. """ if controls is None: - controls = _packaged_controls() + controls = us_immigration_controls() person = frame.table("person") weights = np.asarray(frame.resolve_weights("person").values, dtype=np.float64) @@ -880,13 +2426,18 @@ def us_immigration_composition_gate( details: dict[str, object] = { "summary": us_immigration_composition_summary(frame), "controls": { - "undocumented_workers": controls.workers, - "undocumented_students": controls.students, - "undocumented_population_anchor": controls.population_anchor, - "sources": dict(controls.sources), + "undocumented_workers": controls.undocumented.workers, + "undocumented_students": controls.undocumented.students, + "undocumented_population_anchor": (controls.undocumented.population_anchor), + "sources": dict(controls.undocumented.sources), + "humanitarian_status_stocks": { + draw.label: {"target": draw.target, "source": draw.source} + for draw in controls.humanitarian + }, }, "non_citizen_share_band": list(_NON_CITIZEN_SHARE_BAND), "undocumented_anchor_relative_band": list(_UNDOCUMENTED_ANCHOR_RELATIVE_BAND), + "humanitarian_target_relative_band": list(_HUMANITARIAN_TARGET_RELATIVE_BAND), } missing = [ @@ -908,6 +2459,11 @@ def us_immigration_composition_gate( ssn = person["ssn_card_type"].astype(str) status = person["immigration_status_str"].astype(str) + try: + evidence = _source_aware_immigration_profile(person, time_period=time_period) + except (SourceRuntimeError, ValueError) as exc: + evidence = None + failures.append(f"source-aware immigration evidence is invalid: {exc}") for column, values, domain in ( ("ssn_card_type", ssn, SSN_CARD_TYPE_VALUES), ("immigration_status_str", status, IMMIGRATION_STATUS_VALUES), @@ -939,6 +2495,33 @@ def us_immigration_composition_gate( f"{undocumented_disagreements} person(s) have ssn_card_type NONE " "and immigration_status_str UNDOCUMENTED disagreeing." ) + if evidence is not None: + evidence_citizen_disagreements = int((ssn_citizen != evidence.is_citizen).sum()) + if evidence_citizen_disagreements: + failures.append( + f"{evidence_citizen_disagreements} person(s) have emitted " + "citizenship disagreeing with their source PRCITSHP/CIT evidence." + ) + expected_cuban_haitian, expected_daca = _special_status_masks( + ssn_codes=_ssn_name_codes(person), + birth_country=evidence.birth_country, + arrival_year=evidence.arrival_year, + age_at_entry=evidence.age_at_entry, + age=evidence.age, + ) + special_status_invalid = ( + (status.to_numpy() == "CUBAN_HAITIAN_ENTRANT") != expected_cuban_haitian + ) | ((status.to_numpy() == "DACA") != expected_daca) + details["evidence_derived_status_compatibility"] = { + "cuban_haitian_entrant_rows": int(expected_cuban_haitian.sum()), + "daca_rows": int(expected_daca.sum()), + "invalid_rows": int(special_status_invalid.sum()), + } + if special_status_invalid.any(): + failures.append( + f"{int(special_status_invalid.sum())} person(s) violate the " + "source-aware Cuban/Haitian entrant or DACA cohort contract." + ) non_citizen_share = ( float(weights[~ssn_citizen].sum()) / total if total else float("nan") @@ -950,17 +2533,104 @@ def us_immigration_composition_gate( f"[{low}, {high}]." ) - undocumented = float(weights[ssn_none].sum()) - relative = undocumented / controls.population_anchor + status_values = status.to_numpy() + pew_unauthorized = float(weights[_pew_unauthorized_rows(person)].sum()) + details["pew_unauthorized_population"] = pew_unauthorized + details["pew_unauthorized_statuses"] = list(_PEW_UNAUTHORIZED_STATUS_VALUES) + details["pew_unauthorized_paired_status"] = { + "immigration_status_str": "CUBAN_HAITIAN_ENTRANT", + "ssn_card_type": "NON_CITIZEN_VALID_EAD", + } + relative = pew_unauthorized / controls.undocumented.population_anchor rel_low, rel_high = _UNDOCUMENTED_ANCHOR_RELATIVE_BAND if not (rel_low <= relative <= rel_high): failures.append( - f"emergent undocumented population {undocumented:,.0f} is " + f"emergent Pew-defined unauthorized population " + f"{pew_unauthorized:,.0f} is " f"{relative:.2f}x the published anchor " - f"{controls.population_anchor:,.0f} (band [{rel_low}, {rel_high}], " - f"{controls.sources[_ANCHOR_KEY]})." + f"{controls.undocumented.population_anchor:,.0f} " + f"(band [{rel_low}, {rel_high}], " + f"{controls.undocumented.sources[_ANCHOR_KEY]})." ) + hum_low, hum_high = _HUMANITARIAN_TARGET_RELATIVE_BAND + humanitarian_statuses = set(_HUMANITARIAN_STATUS_BY_CATEGORY.values()) + humanitarian_rows = np.isin(status_values, list(humanitarian_statuses)) + draw_achieved: dict[str, dict[str, object]] = {} + compatible_humanitarian = np.zeros(len(person), dtype=bool) + if evidence is not None: + for draw in controls.humanitarian: + emitted_mask = _humanitarian_emitted_mask( + person, + draw=draw, + profile=evidence, + ) + compatible_humanitarian |= emitted_mask + emitted = float(weights[emitted_mask].sum()) + relative = (emitted / draw.target) if draw.target > 0 else None + draw_achieved[draw.label] = { + "category": draw.category, + "origin": draw.origin, + "status": draw.status, + "target": draw.target, + "population": emitted, + "relative": relative, + "source": draw.source, + } + if draw.target <= 0: + if emitted > 0: + failures.append( + f"{draw.label}: {emitted:,.0f} compatible weighted " + "persons emitted against an explicit zero target." + ) + continue + if relative is None or not (hum_low <= relative <= hum_high): + failures.append( + f"{draw.label}: compatible emitted population {emitted:,.0f} " + f"is {relative:.2f}x the cited stock target {draw.target:,.0f} " + f"(band [{hum_low}, {hum_high}])." + ) + incompatible = humanitarian_rows & ~compatible_humanitarian + if incompatible.any(): + incompatible_population = float(weights[incompatible].sum()) + examples = sorted(set(status_values[incompatible].tolist())) + failures.append( + f"{int(incompatible.sum())} humanitarian row(s) " + f"({incompatible_population:,.0f} weighted persons) violate " + "their source-aware status/origin/arrival/SSN cohort contract; " + f"statuses={examples}." + ) + details["humanitarian_draw_achieved"] = draw_achieved + + achieved_by_category: dict[str, dict[str, object]] = {} + for category in HUMANITARIAN_STATUS_CATEGORIES: + status_name = _HUMANITARIAN_STATUS_BY_CATEGORY[category] + target = controls.humanitarian_target(category) + emitted = float(weights[status_values == status_name].sum()) + achieved_by_category[category] = { + "status": status_name, + "target": target, + "population": emitted, + "relative": (emitted / target) if target > 0 else None, + } + if target <= 0: + if emitted > 0: + failures.append( + f"{status_name}: {emitted:,.0f} weighted persons emitted " + "against an explicit zero target — the manifest documents " + "this category as not imputed." + ) + continue + category_relative = emitted / target + if not (hum_low <= category_relative <= hum_high): + failures.append( + f"{status_name}: emitted population {emitted:,.0f} is " + f"{category_relative:.2f}x the cited stock target " + f"{target:,.0f} (band [{hum_low}, {hum_high}]) — the H.R.1 " + "eligibility channels through this category are degenerate." + ) + details["humanitarian_achieved"] = achieved_by_category + return GateResult( name="immigration_composition", passed=not failures, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/multispine_pool.py b/packages/microcosm-build/src/microcosm/build/us_runtime/multispine_pool.py index 07e87eebf6..c174701854 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/multispine_pool.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/multispine_pool.py @@ -507,7 +507,7 @@ def simulation_ready(self) -> bool: type PoolOperator = Callable[[Frame], PoolStageOutput] type AgreementGate = Callable[[Frame], GateResult] -type SourceFrameOperator = Callable[[Frame], Frame | PoolStageOutput] +type SourceFrameOperator = Callable[..., Frame | PoolStageOutput] @dataclass(frozen=True) @@ -1973,10 +1973,13 @@ def _post_clone_source_operators() -> Mapping[str, SourceFrameOperator]: force_puf_imputation=True, ) ), - "with_us_immigration_inputs": lambda current: with_us_immigration_inputs( - current, - seed=POOL_RANDOM_SEED, - time_period=POOL_TIME_PERIOD, + "with_us_immigration_inputs": lambda current, *, person_weight_scale=1.0: ( + with_us_immigration_inputs( + current, + seed=POOL_RANDOM_SEED, + time_period=POOL_TIME_PERIOD, + person_weight_scale=person_weight_scale, + ) ), "with_us_education_inputs": lambda current: with_us_education_inputs( current, @@ -2182,8 +2185,42 @@ def _run_source_operator_chain( and contract.execution_scope == _CPS_SOURCE_EXECUTION_SCOPE ): declared_outputs = _persisted_source_outputs(declared_outputs) + person_design_weight_scaling: dict[str, float] | None = None if contract.execution_scope == _CPS_SOURCE_EXECUTION_SCOPE: available_mask = _cps_source_evidence_mask(current, phase=phase) + if operator_name == "with_us_immigration_inputs": + full_person_weights = np.asarray( + current.resolve_weights(current.schema.person_entity).values, + dtype=np.float64, + ) + cps_person_weights = full_person_weights[ + available_mask.to_numpy(dtype=bool) + ] + if ( + not np.isfinite(full_person_weights).all() + or (full_person_weights < 0).any() + or not np.isfinite(cps_person_weights).all() + or (cps_person_weights < 0).any() + ): + raise ValueError( + "Pooled immigration controls require finite non-negative " + "person design weights." + ) + full_person_mass = float(full_person_weights.sum()) + cps_person_mass = float(cps_person_weights.sum()) + if full_person_mass <= 0 or cps_person_mass <= 0: + raise ValueError( + "Pooled immigration controls require positive full-pool " + "and CPS-projection person design-weight mass; " + f"got full={full_person_mass!r}, cps={cps_person_mass!r}." + ) + person_weight_scale = full_person_mass / cps_person_mass + person_design_weight_scaling = { + "full_pool_person_design_weight_mass": full_person_mass, + "cps_projection_person_design_weight_mass": cps_person_mass, + "person_weight_scale": person_weight_scale, + "cps_person_mass_share": cps_person_mass / full_person_mass, + } available = _source_available_projection( current, available_mask, @@ -2202,13 +2239,29 @@ def _run_source_operator_chain( ) before_rows = _frame_row_counts(current) available_rows = _frame_row_counts(available) - kernel_outcome = operators[operator_name](available) + if person_design_weight_scaling is None: + kernel_outcome = operators[operator_name](available) + else: + kernel_outcome = operators[operator_name]( + available, + person_weight_scale=person_design_weight_scaling["person_weight_scale"], + ) kernel_receipt: Mapping[str, object] = {} if isinstance(kernel_outcome, PoolStageOutput): outcome = kernel_outcome.frame kernel_receipt = kernel_outcome.receipt else: outcome = kernel_outcome + if person_design_weight_scaling is not None: + if "person_design_weight_scaling" in kernel_receipt: + raise ValueError( + "Pooled immigration kernel receipt may not override the " + "orchestrator-owned person-design-weight scaling contract." + ) + kernel_receipt = { + **dict(kernel_receipt), + "person_design_weight_scaling": person_design_weight_scaling, + } if not isinstance(outcome, Frame): raise TypeError( f"Multispine source operator {operator_name!r} must return Frame, " diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/reform_coverage_smoke.py b/packages/microcosm-build/src/microcosm/build/us_runtime/reform_coverage_smoke.py index 8c8efce063..f377554f43 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/reform_coverage_smoke.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/reform_coverage_smoke.py @@ -132,6 +132,7 @@ def us_reform_coverage_smoke_gate( "effect": effect, "expected_sign": probe.expected_sign, "min_abs_effect": probe.min_abs_effect, + "measure_unit": probe.measure_unit, "binding_inputs": list(probe.binding_inputs), "issue": probe.issue, "passed": passed, @@ -142,11 +143,16 @@ def us_reform_coverage_smoke_gate( if probe.expected_sign == "either" else f"a {probe.expected_sign} effect" ) + if probe.measure_unit == "persons": + effect_text = f"{effect:+,.0f} persons" + floor_text = f"{probe.min_abs_effect:,.0f} persons" + else: + effect_text = f"{effect:+,.0f}" + floor_text = f"${probe.min_abs_effect:,.0f}" scored = ( - f"{probe.id}: '{probe.name}' scores {effect:+,.0f} on " + f"{probe.id}: '{probe.name}' scores {effect_text} on " f"{probe.budget_measure} for {probe_period}; expected " - f"{expectation} with magnitude at least " - f"${probe.min_abs_effect:,.0f}." + f"{expectation} with magnitude at least {floor_text}." ) # A wrong-signed effect that clears the floor proves the reform # binds, so the inputs are carried; what disagrees is the probe's diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/release_input_coverage.py b/packages/microcosm-build/src/microcosm/build/us_runtime/release_input_coverage.py index 00f88d9a7a..be1455a073 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/release_input_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/release_input_coverage.py @@ -339,9 +339,16 @@ def __post_init__(self) -> None: ) +#: What a reform-coverage probe's weighted budget-measure total counts. +#: ``usd`` measures carry PolicyEngine-US's ``currency-USD`` unit; +#: ``persons`` measures are person-level booleans, whose weighted sum +#: counts people. +REFORM_COVERAGE_MEASURE_UNITS = frozenset({"usd", "persons"}) + + @dataclass(frozen=True) class ReformCoverageProbe: - """A pinned reform whose $0 score on the export signals a coverage hole. + """A pinned reform whose zero score on the export signals a coverage hole. A reform that mechanically binds through named input leaves must move its budget measure. If it scores ~$0 on the release, the leaves it binds through @@ -357,6 +364,12 @@ class ReformCoverageProbe: neutralized_variable: Input leaf neutralized by a structural reform when no parameter change isolates the leaf. budget_measure: The variable whose weighted total change is scored. + measure_unit: What the budget measure's weighted total counts: + ``"usd"`` (dollars, the default) or ``"persons"`` (people, + for a person-level boolean measure). The smoke gate states + ``min_abs_effect`` in this unit, and the engine half of + :func:`assert_release_input_coverage_manifest_current` checks + it against the measure's PolicyEngine-US metadata. effect_direction: ``"reform_minus_baseline"`` or ``"baseline_minus_reform"``. period: Optional probe-specific scoring year. OBBBA probes use 2026 @@ -384,6 +397,7 @@ class ReformCoverageProbe: period: int | None = None expected_sign: str = "positive" neutralized_variable: str | None = None + measure_unit: str = "usd" def __post_init__(self) -> None: if not self.id: @@ -420,6 +434,11 @@ def __post_init__(self) -> None: raise ValueError( f"{self.id}: expected_sign must be 'positive', 'negative', or 'either'." ) + if self.measure_unit not in REFORM_COVERAGE_MEASURE_UNITS: + raise ValueError( + f"{self.id}: measure_unit must be one of " + f"{sorted(REFORM_COVERAGE_MEASURE_UNITS)}." + ) @dataclass(frozen=True) @@ -564,6 +583,7 @@ def load_release_input_coverage_manifest( if raw_probe.get("neutralized_variable") is None else str(raw_probe["neutralized_variable"]) ), + measure_unit=str(raw_probe.get("measure_unit", "usd")), ) ) @@ -718,6 +738,37 @@ def _coverage_engine() -> Any | None: return None +def _probe_measure_drift(engine: Any, probe: ReformCoverageProbe) -> list[str]: + """Why ``probe``'s budget measure disagrees with its ``measure_unit``. + + Empty when the measure is a live PolicyEngine-US variable of the declared + unit: a ``currency-USD`` variable for ``usd``, a person-level boolean for + ``persons``. + """ + try: + metadata = engine.variable_metadata(probe.budget_measure) + unit = engine.variable_unit(probe.budget_measure) + except ValueError: + return [ + f"probe {probe.id!r}: budget_measure {probe.budget_measure!r} is " + "not a PolicyEngine-US variable." + ] + if probe.measure_unit == "persons": + if (metadata.entity, metadata.dtype) != ("person", "bool"): + return [ + f"probe {probe.id!r}: measure_unit 'persons' counts people, so " + "its budget_measure must be a person-level boolean; " + f"{probe.budget_measure!r} is a {metadata.dtype} on " + f"{metadata.entity}." + ] + elif unit != "currency-USD": + return [ + f"probe {probe.id!r}: measure_unit 'usd' needs a dollar " + f"budget_measure; {probe.budget_measure!r} declares unit {unit!r}." + ] + return [] + + def _ecps_populated_layers() -> frozenset[str]: payload = _resource_payload(_ECPS_PARITY_REFERENCE_RESOURCE) shares = payload.get("nonzero_shares") @@ -803,9 +854,12 @@ def assert_release_input_coverage_manifest_current( given) under either its live or its historical spelling: the parity gate honours both, so a restored input filed as a gap would be exempted there even while this manifest still requires it. - - Every declared column must be a real PolicyEngine-US input leaf, and every - probe's ``binding_inputs`` / ``budget_measure`` must resolve on the live - engine, so the contract cannot guard names the engine no longer has. + - Every declared column must be a real PolicyEngine-US input leaf, every + probe's ``binding_inputs`` must be live input leaves, and every probe's + ``budget_measure`` must be a live variable in its ``measure_unit`` + (``currency-USD`` for ``usd``, a person-level boolean for + ``persons``), so the contract cannot guard names the engine no longer + has or state a floor in the wrong unit. A no-op for the engine-graph half when no engine is available (the workspace test environment); the checked-in-facts half always runs. @@ -921,6 +975,7 @@ def assert_release_input_coverage_manifest_current( f"probe {probe.id!r}: binding_inputs are not " f"PolicyEngine-US input leaves: {bad_inputs}." ) + failures.extend(_probe_measure_drift(engine, probe)) if failures: raise ValueError( diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py b/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py index 1943db7cb2..fe7968c3fb 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py @@ -100,6 +100,12 @@ TargetFamilies, transfer_acs_inputs, ) +from microcosm.build.us_runtime.immigration import ( + _HUMANITARIAN_TARGET_RELATIVE_BAND, + us_immigration_controls, + us_immigration_humanitarian_draw_mask, + us_immigration_humanitarian_transfer_selection_masks, +) from microcosm.build.us_runtime.late_producer_dag import ( ProducerContract, ProducerInput, @@ -217,6 +223,7 @@ US_LATE_SOURCE_FINALIZER_STAGE, US_LATE_TRANSFER_MODEL_CONFIG_INPUT, US_LATE_TRANSFER_TARGET_BANK_INPUT, + legacy_us_late_producer_contracts, legacy_us_late_producer_schedule_receipt, us_late_producer_schedule_receipt, ) @@ -4180,6 +4187,63 @@ def _validate_post_transfer_live_output_binding( "unmodeled_rows", "residual_null_rows", ) +_IMMIGRATION_TRANSFER_FAMILY = "source_operator_immigration" +_IMMIGRATION_TRANSFER_TARGETS = ("ssn_card_type", "immigration_status_str") +_IMMIGRATION_RECONCILIATION_KEY = "post_transfer_reconciliation" +_IMMIGRATION_RECONCILIATION_DRAW_FIELDS = frozenset( + { + "target", + "immutable_population", + "residual_target", + "eligible_recipient_population", + "selected_recipient_population", + "achieved_population", + "absolute_error", + "selection_threshold_tie_population", + "residual_selection_error", + "within_residual_discrete_weight_bound", + "immutable_overshoot", + "within_discrete_weight_bound", + "candidates_exhausted", + "candidate_shortfall_population", + } +) +_IMMIGRATION_RECONCILIATION_BOOLEAN_DRAW_FIELDS = frozenset( + { + "within_residual_discrete_weight_bound", + "within_discrete_weight_bound", + "candidates_exhausted", + } +) + + +def _is_immigration_transfer_target( + *, + entity: str, + family: str, + target: str, +) -> bool: + return ( + entity == "person" + and family.split("__batch_", 1)[0] == _IMMIGRATION_TRANSFER_FAMILY + and target in _IMMIGRATION_TRANSFER_TARGETS + ) + + +def _immigration_qrf_evidence_targets( + surface: TargetFamilies, +) -> tuple[tuple[str, str], ...]: + return tuple( + (entity, target) + for entity, families in surface.items() + for family, targets in families.items() + for target in targets + if _is_immigration_transfer_target( + entity=entity, + family=family, + target=target, + ) + ) def _validate_acs_transfer_row_counts( @@ -4215,6 +4279,297 @@ def _validate_acs_transfer_row_counts( return typed_counts +def _validate_immigration_post_transfer_reconciliation( + target_receipt: Mapping[str, object], + *, + boundary: str, + frame: Frame | None = None, +) -> Mapping[str, object]: + """Validate the persisted constrained-recipient reconciliation evidence.""" + + reconciliation = target_receipt.get(_IMMIGRATION_RECONCILIATION_KEY) + if not isinstance(reconciliation, Mapping): + raise ValueError( + f"{boundary}: immigration post-transfer reconciliation is absent." + ) + expected_keys = { + "kind", + "seed", + "time_period", + "mutable_rows", + "immutable_rows", + "citizenship_repairs", + "pair_repairs", + "special_status_assignments", + "floating_tolerance", + "selection_order", + "draws", + } + if set(reconciliation) != expected_keys or reconciliation.get("kind") != ( + "deterministic_humanitarian_residual_target" + ): + raise ValueError( + f"{boundary}: immigration post-transfer reconciliation schema is invalid." + ) + integer_fields = ( + "seed", + "time_period", + "mutable_rows", + "immutable_rows", + "citizenship_repairs", + "pair_repairs", + "special_status_assignments", + ) + if any( + not isinstance(reconciliation.get(field), int) + or isinstance(reconciliation[field], bool) + or reconciliation[field] < 0 + for field in integer_fields + ): + raise ValueError( + f"{boundary}: immigration reconciliation integer evidence is invalid." + ) + counts = _validate_acs_transfer_row_counts( + target_receipt, + boundary=boundary, + required=True, + ) + if ( + reconciliation["mutable_rows"] != counts["imputed_rows"] + or reconciliation["citizenship_repairs"] > reconciliation["mutable_rows"] + or reconciliation["pair_repairs"] > reconciliation["mutable_rows"] + or reconciliation["special_status_assignments"] > reconciliation["mutable_rows"] + or reconciliation["citizenship_repairs"] > reconciliation["pair_repairs"] + ): + raise ValueError( + f"{boundary}: immigration reconciliation row accounting is invalid." + ) + producer_rows = target_receipt.get("producer_rows") + if producer_rows is not None and ( + not isinstance(producer_rows, int) + or isinstance(producer_rows, bool) + or producer_rows < 0 + or reconciliation["immutable_rows"] != producer_rows + ): + raise ValueError( + f"{boundary}: immigration reconciliation immutable-row binding is invalid." + ) + controls = us_immigration_controls() + canonical_tolerance = max( + 1e-6, + max((draw.target for draw in controls.humanitarian), default=0.0) * 1e-12, + ) + tolerance = reconciliation.get("floating_tolerance") + if ( + not isinstance(tolerance, (int, float)) + or isinstance(tolerance, bool) + or not np.isfinite(float(tolerance)) + or tolerance <= 0 + or float(tolerance) != canonical_tolerance + ): + raise ValueError( + f"{boundary}: immigration reconciliation tolerance is non-canonical." + ) + tolerance = float(tolerance) + live_draw_populations: dict[str, tuple[float, float, float]] = {} + live_mutable_draw_masks: dict[str, np.ndarray] = {} + expected_mutable_draw_masks: dict[str, np.ndarray] = {} + if frame is not None: + person_entity = frame.schema.person_entity + person = frame.table(person_entity) + channel = person[support_channel_column(person_entity)].astype(str) + immutable_rows = channel.eq(BASE_ASEC_SUPPORT_CHANNEL).to_numpy(dtype=bool) + mutable_rows = channel.eq(ACS_STACKED_SUPPORT_CHANNEL).to_numpy(dtype=bool) + if ( + not np.logical_or(immutable_rows, mutable_rows).all() + or int(immutable_rows.sum()) != reconciliation["immutable_rows"] + or int(mutable_rows.sum()) != reconciliation["mutable_rows"] + ): + raise ValueError( + f"{boundary}: immigration reconciliation row counts differ " + "from the live ASEC/ACS frame." + ) + weights = np.asarray( + frame.resolve_weights(person_entity).values, + dtype=np.float64, + ) + if ( + weights.shape != (len(person),) + or not np.isfinite(weights).all() + or (weights < 0).any() + ): + raise ValueError( + f"{boundary}: immigration reconciliation live person weights " + "are invalid." + ) + for control in controls.humanitarian: + emitted = us_immigration_humanitarian_draw_mask( + frame, + control, + time_period=int(reconciliation["time_period"]), + ) + live_draw_populations[control.label] = ( + float(weights[emitted & immutable_rows].sum()), + float(weights[emitted & mutable_rows].sum()), + float(weights[emitted].sum()), + ) + live_mutable_draw_masks[control.label] = emitted & mutable_rows + expected_mutable_draw_masks = ( + us_immigration_humanitarian_transfer_selection_masks( + frame, + mutable_rows=mutable_rows, + seed=int(reconciliation["seed"]), + time_period=int(reconciliation["time_period"]), + controls=controls, + ) + ) + selection_order = reconciliation.get("selection_order") + draws = reconciliation.get("draws") + expected_order = [draw.label for draw in controls.humanitarian] + targets_by_label = { + draw.label: float(draw.target) for draw in controls.humanitarian + } + if ( + not isinstance(selection_order, list) + or selection_order != expected_order + or not isinstance(draws, Mapping) + or set(draws) != set(expected_order) + ): + raise ValueError( + f"{boundary}: immigration reconciliation draw order is non-canonical." + ) + numeric_fields = ( + _IMMIGRATION_RECONCILIATION_DRAW_FIELDS + - _IMMIGRATION_RECONCILIATION_BOOLEAN_DRAW_FIELDS + ) + band_floor = _HUMANITARIAN_TARGET_RELATIVE_BAND[0] + for label in expected_order: + draw = draws[label] + if not isinstance(draw, Mapping) or set(draw) != ( + _IMMIGRATION_RECONCILIATION_DRAW_FIELDS + ): + raise ValueError( + f"{boundary}: immigration reconciliation draw {label!r} schema " + "is invalid." + ) + # A draw meets both selection bounds exactly when its candidates hold + # its residual; an exhausted draw takes every candidate instead. + exhausted_flag = draw.get("candidates_exhausted") + if ( + any( + not isinstance(draw.get(field), (int, float)) + or isinstance(draw[field], bool) + or not np.isfinite(float(draw[field])) + or draw[field] < 0 + for field in numeric_fields + ) + or any( + not isinstance(draw.get(field), bool) + for field in _IMMIGRATION_RECONCILIATION_BOOLEAN_DRAW_FIELDS + ) + or any( + draw[field] is exhausted_flag + for field in ( + "within_residual_discrete_weight_bound", + "within_discrete_weight_bound", + ) + ) + ): + raise ValueError( + f"{boundary}: immigration reconciliation draw {label!r} values " + "are invalid." + ) + target = targets_by_label[label] + immutable = float(draw["immutable_population"]) + residual = float(draw["residual_target"]) + eligible = float(draw["eligible_recipient_population"]) + selected = float(draw["selected_recipient_population"]) + achieved = float(draw["achieved_population"]) + tie = float(draw["selection_threshold_tie_population"]) + expected_residual = max(0.0, target - immutable) + residual_error = abs(selected - residual) + absolute_error = abs(achieved - target) + immutable_overshoot = max(0.0, immutable - target) + if frame is not None: + live_immutable, live_selected, live_achieved = live_draw_populations[label] + if ( + abs(immutable - live_immutable) > tolerance + or abs(selected - live_selected) > tolerance + or abs(achieved - live_achieved) > tolerance + ): + raise ValueError( + f"{boundary}: immigration reconciliation draw {label!r} " + "differs from the live ASEC/ACS weighted population." + ) + if not np.array_equal( + live_mutable_draw_masks[label], + expected_mutable_draw_masks[label], + ): + raise ValueError( + f"{boundary}: immigration reconciliation draw {label!r} " + "selected mutable-row identities differ from the canonical " + "seeded selection." + ) + if target <= 0 and immutable > tolerance: + raise ValueError( + f"{boundary}: immigration reconciliation draw {label!r} has " + "immutable mass against an explicit-zero target." + ) + exhausted = eligible + tolerance < residual + if exhausted and not exhausted_flag: + raise ValueError( + f"{boundary}: immigration reconciliation draw {label!r} " + "conceals a recipient candidate shortfall." + ) + if exhausted_flag and not exhausted: + raise ValueError( + f"{boundary}: immigration reconciliation draw {label!r} " + "claims a candidate shortfall its eligible population covers." + ) + shortfall = residual - eligible if exhausted else 0.0 + if exhausted: + # Every candidate is taken; the draw keeps the gate's floor. + if abs(selected - eligible) > tolerance or tie > tolerance: + raise ValueError( + f"{boundary}: immigration reconciliation draw {label!r} " + "leaves candidates unselected after exhausting them." + ) + if achieved + tolerance < band_floor * target: + raise ValueError( + f"{boundary}: immigration reconciliation draw {label!r} " + "falls below the floor of the composition gate's band." + ) + elif selected + tolerance < residual: + raise ValueError( + f"{boundary}: immigration reconciliation draw {label!r} " + "underfills its residual target." + ) + if ( + abs(float(draw["target"]) - target) > tolerance + or abs(residual - expected_residual) > tolerance + or selected > eligible + tolerance + or tie > eligible + tolerance + or abs(achieved - (immutable + selected)) > tolerance + or abs(float(draw["residual_selection_error"]) - residual_error) > tolerance + or abs(float(draw["candidate_shortfall_population"]) - shortfall) + > tolerance + or abs(float(draw["absolute_error"]) - absolute_error) > tolerance + or abs(float(draw["immutable_overshoot"]) - immutable_overshoot) > tolerance + or ( + not exhausted + and ( + residual_error > tie + tolerance + or absolute_error > immutable_overshoot + tie + tolerance + ) + ) + ): + raise ValueError( + f"{boundary}: immigration reconciliation draw {label!r} " + "accounting is invalid." + ) + return reconciliation + + _PREGNANCY_STRUCTURAL_COUNT_FIELDS = ( "source_persons_checked", "physical_rows_checked", @@ -4354,6 +4709,11 @@ def _acs_pattern_predictor_authority( mandatory = tuple(housing["mandatory_features"]) required = (*required, *mandatory) optional = tuple(item for item in optional if item not in mandatory) + immigration_targets = set(contract["immigration_status_targets"]) + if immigration_targets.issubset(family_targets): + mandatory = tuple(contract["immigration_required_predictors"]) + required = (*required, *mandatory) + optional = tuple(item for item in optional if item not in mandatory) return tuple(required), tuple(optional) optional_names = acs_transfer_runtime._GROUP_OPTIONAL_NAMES return ( @@ -4900,6 +5260,7 @@ def validate_stacked_post_puf_transfer_receipt( validated_calibration_keys.update( spec.key for spec in expected_calibrations.values() ) + group_immigration_reconciliations: list[Mapping[str, object]] = [] for target_key, target_receipt in group_targets.items(): if not isinstance(target_receipt, Mapping): raise ValueError( @@ -4926,8 +5287,42 @@ def validate_stacked_post_puf_transfer_receipt( ) owner_receipt = target_receipt.get("post_transfer_calibration") spec = expected_calibrations.get(target_key) + target = target_key.rsplit("/", 1)[1] + # Attested schema-9 pools sealed their immigration targets as + # ordinary transfer targets: the paired reconciliation and QRF + # evidence did not exist, so requiring it would reject history. + is_immigration = ( + legacy_worker_authentication is None + and _is_immigration_transfer_target( + entity=group.entity, + family=group.family, + target=target, + ) + ) + if is_immigration: + group_immigration_reconciliations.append( + _validate_immigration_post_transfer_reconciliation( + target_receipt, + boundary=f"{boundary} target {target_key}", + frame=frame, + ) + ) + _validate_acs_imputed_pattern_evidence( + target_receipt, + expected_entity=group.entity, + expected_family=group.family, + expected_target=target, + expected_family_targets=group.targets, + expected_regime_targets=group.targets, + boundary=f"{boundary} target {target_key}", + ) + elif _IMMIGRATION_RECONCILIATION_KEY in target_receipt: + raise ValueError( + f"{boundary}: undeclared immigration reconciliation is " + f"attached to {target_key!r}." + ) if spec is None: - if "qrf_pattern_evidence" in target_receipt: + if "qrf_pattern_evidence" in target_receipt and not is_immigration: raise ValueError( f"{boundary}: undeclared ACS QRF pattern evidence is " f"attached to {target_key!r}." @@ -4947,7 +5342,7 @@ def validate_stacked_post_puf_transfer_receipt( target_receipt, expected_entity=group.entity, expected_family=group.family, - expected_target=target_key.rsplit("/", 1)[1], + expected_target=target, expected_family_targets=group.targets, expected_regime_targets=expected_regime_targets, boundary=f"{boundary} target {target_key}", @@ -5051,6 +5446,17 @@ def validate_stacked_post_puf_transfer_receipt( spec=spec, boundary=f"{boundary} target {target_key}", ) + if group_immigration_reconciliations: + if len(group_immigration_reconciliations) != len( + _IMMIGRATION_TRANSFER_TARGETS + ) or any( + _json_ready(item) != _json_ready(group_immigration_reconciliations[0]) + for item in group_immigration_reconciliations[1:] + ): + raise ValueError( + f"{boundary}: paired immigration reconciliation evidence " + f"is incomplete or inconsistent for group {name!r}." + ) if validated_calibration_keys != set(late_calibration_specs): raise ValueError( f"{boundary}: post-transfer calibration coverage does not match " @@ -8273,6 +8679,18 @@ def validate_stacked_late_producer_transition_authority( boundary=boundary, legacy_worker_authentication=legacy_worker_authentication, ) + transfer = receipt.get("post_puf_transfer") + assert isinstance(transfer, Mapping) + # Downstream fiscal and gate operators intentionally change the complete + # late-output digest, but they do not own source channels, resolved + # weights, or immigration outputs. Replay that reconstructible terminal + # evidence before authenticating the immutable transition carrier. + validate_stacked_post_puf_transfer_receipt( + transfer, + boundary=boundary, + frame=frame, + legacy_worker_authentication=legacy_worker_authentication, + ) _validate_late_transition_authority( frame, receipt, @@ -8342,7 +8760,17 @@ def validate_stacked_late_producer_receipt( receipt.get(digest_field), boundary=f"{boundary} {digest_field}" ) execution = receipt.get("execution") - expected_order = CANONICAL_US_LATE_PRODUCER_SCHEDULE.order + # An attested schema-9 pool sealed its rows against the schema-16 + # contracts. The legacy contracts restore microcosm #767's two immigration + # producers from the frozen schedule's inventories (each digest-verified); + # today's contracts for those two would reject the historical rows. + if legacy_worker_authentication is not None: + legacy_contracts = legacy_us_late_producer_contracts() + expected_registry = legacy_contracts.registry + expected_order = legacy_contracts.schedule.order + else: + expected_registry = CANONICAL_US_LATE_PRODUCER_REGISTRY + expected_order = CANONICAL_US_LATE_PRODUCER_SCHEDULE.order if not isinstance(execution, list) or len(execution) != len(expected_order): raise ValueError( f"{boundary}: stacked late-producer DAG must carry exactly " @@ -8358,7 +8786,7 @@ def validate_stacked_late_producer_receipt( raw_row = execution[index] previous_sha256 = _validate_late_execution_row( raw_row, - contract=CANONICAL_US_LATE_PRODUCER_REGISTRY[producer_name], + contract=expected_registry[producer_name], execution_index=index, expected_previous_sha256=previous_sha256, boundary=boundary, @@ -10481,10 +10909,17 @@ def _transfer_stacked_post_puf_inputs_evaluate( derive_schedule_d=derive_schedule_d, execution_contract=execution_contract, regime_evidence_targets=tuple( - (spec.entity, spec.target) - for spec in _stacked_post_transfer_calibration_specs( - surface, - stage="late_transfer", + dict.fromkeys( + ( + *( + (spec.entity, spec.target) + for spec in _stacked_post_transfer_calibration_specs( + surface, + stage="late_transfer", + ) + ), + *_immigration_qrf_evidence_targets(surface), + ) ) ), ) @@ -10729,7 +11164,8 @@ def _verify_post_puf_transfer_outcome( target_families, stage="late_transfer", ) - } + } | set(_immigration_qrf_evidence_targets(target_families)) + immigration_reconciliations: dict[str, Mapping[str, object]] = {} for entity, families in target_families.items(): table = frame.table(entity) for family, family_targets in families.items(): @@ -10809,11 +11245,43 @@ def _verify_post_puf_transfer_outcome( target_receipt["qrf_pattern_evidence"] = ( _acs_imputed_pattern_evidence(record) ) + if _is_immigration_transfer_target( + entity=entity, + family=family, + target=target, + ): + if record is None or not isinstance(record.reconciliation, Mapping): + failures.append( + f"{label}: constrained immigration reconciliation " + "evidence is absent." + ) + else: + evidence = _json_ready(record.reconciliation) + assert isinstance(evidence, Mapping) + target_receipt[_IMMIGRATION_RECONCILIATION_KEY] = evidence + immigration_reconciliations[target] = evidence if record is not None and record.structural_receipt is not None: target_receipt["structural_policy"] = dict( record.structural_receipt ) target_receipts[target_receipt_key] = target_receipt + if set(immigration_reconciliations) not in ( + set(), + set(_IMMIGRATION_TRANSFER_TARGETS), + ): + failures.append( + "post_puf_transfer/person/source_operator_immigration: paired " + "reconciliation evidence is incomplete." + ) + elif immigration_reconciliations and any( + _json_ready(evidence) + != _json_ready(immigration_reconciliations[_IMMIGRATION_TRANSFER_TARGETS[0]]) + for evidence in immigration_reconciliations.values() + ): + failures.append( + "post_puf_transfer/person/source_operator_immigration: paired " + "reconciliation evidence disagrees." + ) if failures: raise ValueError( "Stacked post-PUF transfer outcome verification failed:\n " diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_producer_registry.py b/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_producer_registry.py index 6f7acb1595..a3d0240eaf 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_producer_registry.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_producer_registry.py @@ -21,6 +21,8 @@ from collections import defaultdict from collections.abc import Mapping, Sequence from dataclasses import dataclass +from functools import cache +from importlib.resources import files from types import MappingProxyType from microcosm.build.us_runtime.acs_income_universe import ( @@ -37,6 +39,7 @@ ProducerInputColumn, ProducerOutput, ProducerSchedule, + _contract_payload, derive_producer_schedule, ) from microcosm.build.us_runtime.multispine_pool import ( @@ -95,16 +98,20 @@ "US_LATE_TRANSFER_TARGET_BANK_INPUT", "source_producer_name", "transfer_producer_name", + "LegacyLateProducerContracts", + "legacy_us_late_producer_contracts", "legacy_us_late_producer_schedule_receipt", "us_late_producer_schedule_payload", "us_late_producer_schedule_receipt", ] -# v16 declared person.s_corp_income as a whole-pool primary-PUF output while -# the donor carried Schedule E line 32 combined in partnership_income; since -# microcosm#1258 the S-corporation share is its own donor leaf, so it is an -# ordinary PUF-clone output that the late transfer carries to native rows -# (a content change under the same registry schema). v15 content-binds +# v17 binds the ASEC and ACS source-scoped evidence read by constrained +# immigration transfer and reconciliation. v16 declared person.s_corp_income +# as a whole-pool primary-PUF output while the donor carried Schedule E line +# 32 combined in partnership_income; since microcosm#1258 the S-corporation +# share is its own donor leaf, so it is an ordinary PUF-clone output that the +# late transfer carries to native rows (a content change under the same +# registry schema). v15 content-binds # the complete late dual-producer ownership matrix and # validates that it exhausts the primary/source/transfer intersection. v14 # scopes origin-exclusive raw requirements independently of their inventory @@ -126,7 +133,8 @@ # cardinalities across each execution row, binds source-receipt outputs to the # callback receipt, and requires the primary callback to report the exact # resources it consumed. Receipt v4 and registry v17 bind the portable worker -# identity; transition authority v2 encloses that receipt. Receipt v2 +# identity and persist the paired constrained-immigration reconciliation and +# QRF evidence; transition authority v2 encloses that receipt. Receipt v2 # introduced exact virtual-resource payloads. US_LATE_PRODUCER_REGISTRY_SCHEMA_VERSION = 17 US_LATE_PRODUCER_RECEIPT_SCHEMA_VERSION = 4 @@ -959,8 +967,7 @@ def _inventory( "A_MARITL", "A_SPOUSE", "A_HSCOL", - "WSAL_VAL", - "SEMP_VAL", + "A_LFSR", "MCARE", "CAID", "IHSFLG", @@ -982,6 +989,11 @@ def _inventory( _single("resolved_person_weight", "person", "@resolved_weight"), _requirement( "stable_source_identity", + ( + _column("person", "source_year"), + _column("person", "source_household_id"), + _column("person", "source_person_id"), + ), ( _column("person", "source_year"), _column("person", "source_person_id"), @@ -1200,6 +1212,69 @@ def _transfer_input_inventory(group: TransferProducerGroup) -> SourceInputInvent _ADULT_CARE_ROLE_INPUT, ) ) + if group.name == transfer_producer_name("person", "source_operator_immigration"): + post_transfer_structure.extend( + ( + _requirement( + "immigration_stable_person_lineage", + ( + _column("person", "source_year"), + _column("person", "source_household_id"), + _column("person", "source_person_id"), + ), + ( + _column("person", "source_year"), + _column("person", "source_person_id"), + ), + (_column("person", "person_id"),), + ), + _single( + "immigration_asec_citizenship", + "person", + "PRCITSHP", + value_kind="finite_numeric", + required_scope=_ASEC_SOURCE_SCOPE, + ), + _single( + "immigration_asec_origin", + "person", + "PENATVTY", + value_kind="finite_numeric", + required_scope=_ASEC_SOURCE_SCOPE, + ), + _single( + "immigration_asec_arrival", + "person", + "PEINUSYR", + value_kind="finite_numeric", + required_scope=_ASEC_SOURCE_SCOPE, + ), + _single( + "immigration_acs_citizenship", + "person", + "CIT", + value_kind="finite_numeric", + required_scope=_ACS_SOURCE_SCOPE, + ), + _single( + "immigration_acs_origin", + "person", + "POBP", + value_kind="finite_numeric", + required_scope=_ACS_SOURCE_SCOPE, + ), + _single( + "immigration_acs_arrival", + "person", + "YOEP", + # Native-born ACS people carry a structural blank. The + # source-aware runtime requires a finite year only for + # foreign-born CIT=4/5 rows. + value_kind="column_present", + required_scope=_ACS_SOURCE_SCOPE, + ), + ) + ) return _inventory( group.name, *structural, @@ -1612,7 +1687,18 @@ def _inventory_contract_inputs( return tuple(inputs) -def _build_registry() -> dict[str, ProducerContract]: +def _build_registry( + *, + source_inventories: Mapping[str, SourceInputInventory], + transfer_inventories: Mapping[str, SourceInputInventory], +) -> dict[str, ProducerContract]: + """Derive every late producer contract from its effective input inventories. + + The inventories are arguments so attested legacy scoring can derive + microcosm #767's two immigration producers from the schema-16 inventories + those pools sealed; everything else is derived exactly as for the live + registry. + """ late_surface = pool_post_puf_transfer_target_families() late_keys = _target_key_rows(late_surface) puf_keys = _target_key_rows(pool_post_puf_puf_producer_target_families()) @@ -1828,7 +1914,7 @@ def _build_registry() -> dict[str, ProducerContract]: direct_dependency_keys = { (item.entity, item.column) for item in direct_dependencies } - for requirement in US_LATE_SOURCE_INPUT_INVENTORIES[operator].requirements: + for requirement in source_inventories[operator].requirements: for alternative in requirement.alternatives: for item in alternative: key = (item.entity, item.column) @@ -1850,7 +1936,7 @@ def _build_registry() -> dict[str, ProducerContract]: *direct_dependencies, *_inventory_contract_inputs( name, - US_LATE_SOURCE_INPUT_INVENTORIES[operator], + source_inventories[operator], required_scope=_ASEC_SOURCE_SCOPE, ), } @@ -1900,7 +1986,7 @@ def _build_registry() -> dict[str, ProducerContract]: inputs: list[ProducerInput] = list( _inventory_contract_inputs( group.name, - US_LATE_TRANSFER_INPUT_INVENTORIES[group.name], + transfer_inventories[group.name], required_scope=_WHOLE_POOL_SCOPE, ) ) @@ -1943,7 +2029,7 @@ def _build_registry() -> dict[str, ProducerContract]: for item in inputs if not item.column.startswith("@effective:") } - for requirement in US_LATE_TRANSFER_INPUT_INVENTORIES[group.name].requirements: + for requirement in transfer_inventories[group.name].requirements: for alternative in requirement.alternatives: for item in alternative: key = (item.entity, item.column) @@ -2021,7 +2107,10 @@ def _build_registry() -> dict[str, ProducerContract]: CANONICAL_US_LATE_PRODUCER_REGISTRY: Mapping[str, ProducerContract] = MappingProxyType( - _build_registry() + _build_registry( + source_inventories=US_LATE_SOURCE_INPUT_INVENTORIES, + transfer_inventories=US_LATE_TRANSFER_INPUT_INVENTORIES, + ) ) CANONICAL_US_LATE_PRODUCER_SCHEDULE: ProducerSchedule = derive_producer_schedule( CANONICAL_US_LATE_PRODUCER_REGISTRY, @@ -2054,34 +2143,43 @@ def _inventory_payload(inventory: SourceInputInventory) -> dict[str, object]: } -def us_late_producer_schedule_payload() -> dict[str, object]: - """Return the complete JSON-safe declaration bound into checkpoint identity.""" +def _live_execution_receipt_contract() -> dict[str, object]: + return { + "version": US_LATE_PRODUCER_RECEIPT_SCHEMA_VERSION, + "row_binding": ( + "declared_globally_reconciled_input_and_scope_exact_output_" + "source_and_primary_callback_resource_receipt_and_previous_" + "execution_sha256" + ), + "virtual_resource_binding": ("exact_kind_specific_semantic_payload_and_sha256"), + "top_binding": ( + "entry_and_output_frame_sha256_execution_chain_source_" + "completion_nineteen_transfer_groups_and_constrained_" + "immigration_reconciliation" + ), + "transition_authority": { + "authority_id": US_LATE_PRODUCER_TRANSITION_AUTHORITY_ID, + "metadata_key": US_LATE_PRODUCER_TRANSITION_AUTHORITY_KEY, + "version": US_LATE_PRODUCER_TRANSITION_AUTHORITY_VERSION, + "independent_digest_required": True, + }, + } + - schedule = CANONICAL_US_LATE_PRODUCER_SCHEDULE +def _schedule_payload( + *, + schema_version: int, + execution_receipt_contract: Mapping[str, object], + schedule: ProducerSchedule, + source_inventories: Mapping[str, SourceInputInventory], + transfer_inventories: Mapping[str, SourceInputInventory], +) -> dict[str, object]: return { - "schema_version": US_LATE_PRODUCER_REGISTRY_SCHEMA_VERSION, + "schema_version": schema_version, "overlap_ownership": dict(us_late_overlap_ownership_receipt()), - "execution_receipt_contract": { - "version": US_LATE_PRODUCER_RECEIPT_SCHEMA_VERSION, - "row_binding": ( - "declared_globally_reconciled_input_and_scope_exact_output_" - "source_and_primary_callback_resource_receipt_and_previous_" - "execution_sha256" - ), - "virtual_resource_binding": ( - "exact_kind_specific_semantic_payload_and_sha256" - ), - "top_binding": ( - "entry_and_output_frame_sha256_execution_chain_source_" - "completion_and_nineteen_transfer_groups" - ), - "transition_authority": { - "authority_id": US_LATE_PRODUCER_TRANSITION_AUTHORITY_ID, - "metadata_key": US_LATE_PRODUCER_TRANSITION_AUTHORITY_KEY, - "version": US_LATE_PRODUCER_TRANSITION_AUTHORITY_VERSION, - "independent_digest_required": True, - }, - }, + "execution_receipt_contract": json.loads( + json.dumps(dict(execution_receipt_contract)) + ), "schedule_sha256": schedule.sha256, "external_stages": list(US_LATE_EXTERNAL_STAGES), "order": list(schedule.order), @@ -2097,8 +2195,8 @@ def us_late_producer_schedule_payload() -> dict[str, object]: for group in CANONICAL_US_LATE_TRANSFER_GROUPS ], "source_input_inventories": [ - _inventory_payload(US_LATE_SOURCE_INPUT_INVENTORIES[operator]) - for operator in sorted(US_LATE_SOURCE_INPUT_INVENTORIES) + _inventory_payload(source_inventories[operator]) + for operator in sorted(source_inventories) ], "primary_puf_input_inventory": _inventory_payload( US_LATE_PRIMARY_PUF_INPUT_INVENTORY @@ -2107,12 +2205,24 @@ def us_late_producer_schedule_payload() -> dict[str, object]: US_LATE_ACS_EARNINGS_UNIVERSE_INPUT_INVENTORY ), "transfer_input_inventories": [ - _inventory_payload(US_LATE_TRANSFER_INPUT_INVENTORIES[name]) - for name in sorted(US_LATE_TRANSFER_INPUT_INVENTORIES) + _inventory_payload(transfer_inventories[name]) + for name in sorted(transfer_inventories) ], } +def us_late_producer_schedule_payload() -> dict[str, object]: + """Return the complete JSON-safe declaration bound into checkpoint identity.""" + + return _schedule_payload( + schema_version=US_LATE_PRODUCER_REGISTRY_SCHEMA_VERSION, + execution_receipt_contract=_live_execution_receipt_contract(), + schedule=CANONICAL_US_LATE_PRODUCER_SCHEDULE, + source_inventories=US_LATE_SOURCE_INPUT_INVENTORIES, + transfer_inventories=US_LATE_TRANSFER_INPUT_INVENTORIES, + ) + + def us_late_producer_schedule_receipt() -> Mapping[str, object]: """Return the byte-stable production schedule identity and useful counts.""" @@ -2138,26 +2248,61 @@ def us_late_producer_schedule_receipt() -> Mapping[str, object]: ) -def legacy_us_late_producer_schedule_receipt() -> Mapping[str, object]: - """Reconstruct the exact schema-16 schedule for attested legacy scoring.""" +#: Content hash of the schema-16 schedule that attested schema-9 pools sealed. +#: The schedule is a frozen resource checked against this digest. Attested +#: legacy scoring takes two things from it: the schema-16 execution receipt +#: contract, and the input inventories of the two producers microcosm #767 +#: changed (the immigration source stage and its transfer group). Every other +#: legacy contract follows today's registry, as on main, where microcosm #1239 +#: and #1258 changed the primary-PUF and PUF-itemization contracts after +#: schema 16; restoring those to sealed history is separate work. +LEGACY_SCHEMA16_LATE_PRODUCER_SCHEDULE_PAYLOAD_SHA256 = ( + "02e618cc656eb39990ed99dca2b30a52794e01e2b06a3c2df87ca4a7d85ab086" +) +#: Source operators and transfer groups whose schema-16 inventories the legacy +#: contracts take from the frozen schedule (microcosm #767). +LEGACY_SCHEMA16_SEALED_SOURCE_INVENTORIES = ("with_us_immigration_inputs",) +LEGACY_SCHEMA16_SEALED_TRANSFER_INVENTORIES = ( + "transfer:person/source_operator_immigration", +) +#: Canonical-JSON SHA-256 of each of those producers' contracts as the +#: schema-16 registry derived them (measured at main 7c4a5ac08, whose registry +#: reproduces the frozen schedule's ``schedule_sha256``). +_LEGACY_SCHEMA16_SEALED_CONTRACT_SHA256 = MappingProxyType( + { + "source:with_us_immigration_inputs": ( + "8ffdcf5128e54956ea63669033cddcb894af93333fdf61e2262d9f54105ec5af" + ), + "transfer:person/source_operator_immigration": ( + "81c5bdc4567090d8c68aac8101214b9f2db3a16485cc14521b81e530a0860f96" + ), + } +) +_LEGACY_SCHEDULE_DERIVED_KEYS = frozenset( + { + "payload_sha256", + "producer_count", + "source_producer_count", + "transfer_group_count", + "transfer_target_count", + "status", + } +) - receipt = json.loads(json.dumps(dict(us_late_producer_schedule_receipt()))) - receipt["schema_version"] = 16 - contract = receipt["execution_receipt_contract"] - contract["version"] = 3 - contract["transition_authority"]["version"] = 1 + +@cache +def _verified_legacy_schedule_json() -> str: + """Return the frozen schema-16 schedule text once its hash is verified.""" + + resource = files("microcosm.build.us").joinpath( + "legacy_schema16_late_producer_schedule.json" + ) + text = resource.read_text(encoding="utf-8") + receipt = json.loads(text) payload = { key: value for key, value in receipt.items() - if key - not in { - "payload_sha256", - "producer_count", - "source_producer_count", - "transfer_group_count", - "transfer_target_count", - "status", - } + if key not in _LEGACY_SCHEDULE_DERIVED_KEYS } canonical = json.dumps( payload, @@ -2165,5 +2310,166 @@ def legacy_us_late_producer_schedule_receipt() -> Mapping[str, object]: separators=(",", ":"), allow_nan=False, ).encode("utf-8") - receipt["payload_sha256"] = hashlib.sha256(canonical).hexdigest() - return MappingProxyType(receipt) + observed = hashlib.sha256(canonical).hexdigest() + if ( + observed != LEGACY_SCHEMA16_LATE_PRODUCER_SCHEDULE_PAYLOAD_SHA256 + or receipt.get("payload_sha256") != observed + ): + raise ValueError( + "frozen schema-16 late-producer schedule does not match its " + "sealed content hash." + ) + return text + + +def _inventory_from_payload(payload: Mapping[str, object]) -> SourceInputInventory: + """Invert :func:`_inventory_payload` exactly.""" + + requirements = payload["requirements"] + assert isinstance(requirements, Sequence) + return SourceInputInventory( + operator=str(payload["operator"]), + requirements=tuple( + EffectiveInputRequirement( + label=row["label"], + optional=row["optional"], + required_scope=row["required_scope"], + alternatives=tuple( + tuple( + ScopedInput(item["entity"], item["column"], item["value_kind"]) + for item in alternative + ) + for alternative in row["alternatives"] + ), + ) + for row in requirements + ), + ) + + +def _canonical_sha256(payload: object) -> str: + return hashlib.sha256( + json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + ).hexdigest() + + +@dataclass(frozen=True) +class LegacyLateProducerContracts: + """The late producer contracts attested schema-9 pools are scored against.""" + + registry: Mapping[str, ProducerContract] + schedule: ProducerSchedule + source_inventories: Mapping[str, SourceInputInventory] + transfer_inventories: Mapping[str, SourceInputInventory] + + +def _sealed_inventories( + live: Mapping[str, SourceInputInventory], + sealed_rows: Sequence[Mapping[str, object]], + names: Sequence[str], +) -> dict[str, SourceInputInventory]: + sealed = {str(row["operator"]): row for row in sealed_rows} + inventories = dict(live) + for name in names: + row = sealed.get(name) + if name not in inventories or row is None: + raise ValueError( + f"frozen schema-16 schedule has no {name!r} inventory to restore." + ) + inventory = _inventory_from_payload(row) + if _inventory_payload(inventory) != row: + raise ValueError( + f"frozen schema-16 {name!r} inventory does not round-trip." + ) + inventories[name] = inventory + return inventories + + +@cache +def legacy_us_late_producer_contracts() -> LegacyLateProducerContracts: + """Derive the legacy contracts, with #767's two producers at schema 16. + + Today's derivation runs over today's inventories, except the immigration + source stage and its transfer group, whose inventories come from the frozen + schema-16 schedule. Each of those two contracts must hash to the digest the + schema-16 registry derived, so any drift in their derivation refuses here + instead of scoring a historical pool's execution rows against it. + """ + + frozen = json.loads(_verified_legacy_schedule_json()) + source_inventories = _sealed_inventories( + US_LATE_SOURCE_INPUT_INVENTORIES, + frozen["source_input_inventories"], + LEGACY_SCHEMA16_SEALED_SOURCE_INVENTORIES, + ) + transfer_inventories = _sealed_inventories( + US_LATE_TRANSFER_INPUT_INVENTORIES, + frozen["transfer_input_inventories"], + LEGACY_SCHEMA16_SEALED_TRANSFER_INVENTORIES, + ) + registry = _build_registry( + source_inventories=source_inventories, + transfer_inventories=transfer_inventories, + ) + for name, expected in _LEGACY_SCHEMA16_SEALED_CONTRACT_SHA256.items(): + contract = registry.get(name) + if contract is None or _canonical_sha256(_contract_payload(contract)) != ( + expected + ): + raise ValueError( + f"legacy late-producer contract {name!r} no longer matches the " + "schema-16 contract attested schema-9 pools sealed." + ) + return LegacyLateProducerContracts( + registry=MappingProxyType(registry), + schedule=derive_producer_schedule( + registry, + external_stages=US_LATE_EXTERNAL_STAGES, + ), + source_inventories=MappingProxyType(source_inventories), + transfer_inventories=MappingProxyType(transfer_inventories), + ) + + +@cache +def _legacy_schedule_receipt_json() -> str: + frozen = json.loads(_verified_legacy_schedule_json()) + legacy = legacy_us_late_producer_contracts() + payload = _schedule_payload( + schema_version=frozen["schema_version"], + execution_receipt_contract=frozen["execution_receipt_contract"], + schedule=legacy.schedule, + source_inventories=legacy.source_inventories, + transfer_inventories=legacy.transfer_inventories, + ) + return json.dumps( + { + **payload, + "payload_sha256": _canonical_sha256(payload), + "producer_count": len(legacy.registry), + "source_producer_count": len(POOL_POST_CLONE_SOURCE_OPERATOR_ORDER), + "transfer_group_count": len(CANONICAL_US_LATE_TRANSFER_GROUPS), + "transfer_target_count": sum( + len(group.targets) for group in CANONICAL_US_LATE_TRANSFER_GROUPS + ), + "status": "derived_and_import_validated", + }, + allow_nan=False, + ) + + +def legacy_us_late_producer_schedule_receipt() -> Mapping[str, object]: + """Return the schema-16 schedule receipt attested legacy scoring checks. + + It carries the frozen schedule's schema version and execution receipt + contract over the legacy contracts. Only the text is cached; every caller + gets its own parse, so no caller can alter the nested content another one + validates against. + """ + + return MappingProxyType(json.loads(_legacy_schedule_receipt_json())) diff --git a/packages/microcosm-build/tests/engine_contract/us/test_release_input_coverage.py b/packages/microcosm-build/tests/engine_contract/us/test_release_input_coverage.py index 29591bbc3a..e1431a05e5 100644 --- a/packages/microcosm-build/tests/engine_contract/us/test_release_input_coverage.py +++ b/packages/microcosm-build/tests/engine_contract/us/test_release_input_coverage.py @@ -1,6 +1,8 @@ """Tests split from packages/microcosm-build/tests/test_release_input_coverage.py.""" # ruff: noqa: F403, F405 +import dataclasses + from test_support.microcosm_build.release_input_coverage import * @@ -440,3 +442,70 @@ def test_real_engine_treats_receipt_inputs_as_false_default_input_leaves( ) assert not failing.passed assert failing.details["degenerate_required"] == sorted(names) + + +_HR1_MEDICAID_PROBE = "hr1_medicaid_humanitarian_eligibility_restoration" + + +@pytest.fixture(scope="module") +def live_engine(): + from microcosm.frame.adapters.policyengine_us import PolicyEngineUSEngine + + return PolicyEngineUSEngine() + + +def _with_probe_change(probe_id: str, **changes) -> ReleaseInputCoverageManifest: + manifest = load_release_input_coverage_manifest() + return ReleaseInputCoverageManifest( + reference=manifest.reference, + columns=manifest.columns, + probes=tuple( + dataclasses.replace(probe, **changes) if probe.id == probe_id else probe + for probe in manifest.probes + ), + schema_version=manifest.schema_version, + ) + + +class TestProbeMeasureUnits: + """The engine half checks each probe's measure against its declared unit.""" + + def test_shipped_probes_match_their_measures(self, live_engine) -> None: + assert_release_input_coverage_manifest_current(engine=live_engine) + + def test_people_count_declared_as_dollars_is_refused(self, live_engine) -> None: + with pytest.raises( + ValueError, + match=( + "measure_unit 'usd' needs a dollar budget_measure; " + "'medicaid_enrolled' declares unit None" + ), + ): + assert_release_input_coverage_manifest_current( + engine=live_engine, + manifest=_with_probe_change(_HR1_MEDICAID_PROBE, measure_unit="usd"), + ) + + def test_dollars_declared_as_people_are_refused(self, live_engine) -> None: + with pytest.raises( + ValueError, + match="must be a person-level boolean; 'medicaid' is a float on person", + ): + assert_release_input_coverage_manifest_current( + engine=live_engine, + manifest=_with_probe_change( + _HR1_MEDICAID_PROBE, budget_measure="medicaid" + ), + ) + + def test_unknown_measure_is_refused(self, live_engine) -> None: + with pytest.raises( + ValueError, + match="budget_measure 'not_a_variable' is not a PolicyEngine-US variable", + ): + assert_release_input_coverage_manifest_current( + engine=live_engine, + manifest=_with_probe_change( + _HR1_MEDICAID_PROBE, budget_measure="not_a_variable" + ), + ) diff --git a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py index 519651a171..08ac2d7e0a 100644 --- a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py +++ b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py @@ -82,22 +82,22 @@ def test_us_coverage_is_exact_complete_and_honest( assert_coverage_complete(coverage_report) assert coverage_report["status"] == "pass" fields = coverage_report["field_usage"] - assert fields["configuration_field_count"] == 42394 - assert fields["authored_normative_field_count"] == 32609 - assert fields["resolved_binding_field_count"] == 9_785 - assert fields["consumed_field_count"] == 42394 + assert fields["configuration_field_count"] == 42_781 + assert fields["authored_normative_field_count"] == 32_749 + assert fields["resolved_binding_field_count"] == 10_032 + assert fields["consumed_field_count"] == 42_781 assert fields["unused_field_count"] == 0 assert fields["multiple_primary_use_field_count"] == 0 assert fields["claim_count"] == 48 assert fields["mode_counts"] == { - "legacy_behavior": 14_269, - "compiler_semantic": 27673, - "front_end_validation": 349, + "legacy_behavior": 14_301, + "compiler_semantic": 28_025, + "front_end_validation": 352, "identity_only": 103, } assert fields["generation0_effect_counts"] == { - "legacy_behavior": 38_780, - "no_generation0_effect": 3614, + "legacy_behavior": 38_878, + "no_generation0_effect": 3_903, } inventory = coverage_report["inventory_coverage"] @@ -105,10 +105,10 @@ def test_us_coverage_is_exact_complete_and_honest( assert inventory["covered_item_count"] == 41 assert inventory["missing_item_count"] == 0 assert inventory["missing_items"] == [] - assert inventory["counts"]["producer_inputs"] == 2_747 + assert inventory["counts"]["producer_inputs"] == 2_753 assert inventory["counts"]["ownership_rows"] == 18 assert inventory["counts"]["tail_control_fields"] == 942 - assert inventory["counts"]["seed_owner_bindings"] == 112 + assert inventory["counts"]["seed_owner_bindings"] == 125 @pytest.mark.parametrize( diff --git a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_stacked_authority_semantics.py b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_stacked_authority_semantics.py index 31a263a057..afff3a7396 100644 --- a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_stacked_authority_semantics.py +++ b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_stacked_authority_semantics.py @@ -70,7 +70,7 @@ def test_authority_projection_is_field_and_byte_identical_to_live_generation_zer assert projected == live assert stacked_identity_bytes(projected) == _canonical_bytes(live) assert projected["sha256"] == ( - "bf681aedc859fd6398ef72da68f584069b7d25b3085c549bf99f82e8324c4eef" + "7cdedd5bf056f1407910eddcd40ad0660bd7ef2d6defe224612c02b3476cbf15" ) assert { name: component["sha256"] for name, component in projected["components"].items() @@ -85,7 +85,7 @@ def test_authority_projection_is_field_and_byte_identical_to_live_generation_zer "cacc6c11e114dbae3aaa2761cc6b3fcb1191cd9b689b1c2bd096614c51ebff8b" ), "late_producer_schedule": ( - "80321bdcd6b2278da993ac6156b97b5b7532ed671e19caddc6cea19f1ace8816" + "bdc3ab40ac748822cea2712a517e6ec60b22674337b10bc0176a81809db07466" ), "metric_registry": ( "832245589a8ec5a33fc151e60662a245d411a636ef62debb884d65abcad3024a" @@ -188,7 +188,7 @@ def test_checkpoint_projection_is_field_and_byte_identical_to_live_oracle( ] assert ( projected["pool_code"]["late_producer_schedule"]["schedule_sha256"] - == "f1fb754e6ade602be394837bc0d7d091f2fba4b4b83fb00f59a0258fe56eb5da" + == "46cea046d2fb8223d32772bd6a7807c52791640943949154982dcc52d93fa9eb" ) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_release_input_coverage.py b/packages/microcosm-build/tests/engine_free/shared/test_release_input_coverage.py index 7e5838e1ab..9e205bc218 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_release_input_coverage.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_release_input_coverage.py @@ -184,6 +184,72 @@ def test_either_sign_probe_passes_both_directions_and_keeps_floor_teeth( assert not structural_zero.passed assert "an effect in either direction" in structural_zero.failures[0] + def test_dollar_probe_states_its_floor_in_dollars(self, monkeypatch) -> None: + monkeypatch.setattr(smoke_module, "_build_reform", lambda changes: "REFORM") + result = us_reform_coverage_smoke_gate( + simulate=lambda reform: _Sim(10.0e9), probes=[_probe()] + ) + assert not result.passed + assert ( + "scores +0 on ssi for 2024; expected a positive effect with " + "magnitude at least $1,000,000,000." in result.failures[0] + ) + assert result.details["results"]["ssi_probe"]["measure_unit"] == "usd" + + def test_people_count_probe_states_effect_and_floor_in_people( + self, monkeypatch + ) -> None: + # A person-level boolean measure (medicaid_enrolled) sums weighted + # people, so the gate must state the effect and floor as people, not + # print a dollar floor (microcosm#767). + monkeypatch.setattr(smoke_module, "_build_reform", lambda changes: "REFORM") + probe = ReformCoverageProbe( + id="enrollment_probe", + name="Status-list restoration", + parameter_changes={ + "gov.example.statuses": {"2027-01-01.2100-12-31": ["REFUGEE"]} + }, + budget_measure="medicaid_enrolled", + binding_inputs=("immigration_status_str",), + min_abs_effect=10_000.0, + reason="The restoration binds through the status leaf.", + issue="PolicyEngine/microcosm#767", + period=2027, + measure_unit="persons", + ) + + undersized = us_reform_coverage_smoke_gate( + simulate=lambda reform: _Sim(2_000.0 if reform else 0.0), + probes=[probe], + ) + assert not undersized.passed + failure = undersized.failures[0] + assert "scores +2,000 persons on medicaid_enrolled for 2027" in failure + assert "magnitude at least 10,000 persons." in failure + assert "$" not in failure + result = undersized.details["results"]["enrollment_probe"] + assert result["measure_unit"] == "persons" + + passing = us_reform_coverage_smoke_gate( + simulate=lambda reform: _Sim(250_000.0 if reform else 0.0), + probes=[probe], + ) + assert passing.passed + + def test_probe_rejects_unknown_measure_unit(self) -> None: + with pytest.raises(ValueError, match="measure_unit must be one of"): + ReformCoverageProbe( + id="bad_unit", + name="Bad unit", + parameter_changes={"gov.example.switch": {"2026": 0}}, + budget_measure="medicaid_enrolled", + binding_inputs=("x",), + min_abs_effect=1.0, + reason="r.", + issue="PolicyEngine/microcosm#1", + measure_unit="households", + ) + def test_probe_rejects_unknown_expected_sign(self) -> None: with pytest.raises(ValueError, match="expected_sign must be"): ReformCoverageProbe( @@ -546,6 +612,23 @@ def test_form_4952_election_is_promoted_with_unique_probe(self) -> None: assert probe.neutralized_variable == output assert probe.binding_inputs == (output,) + def test_only_the_hr1_medicaid_probe_counts_people(self) -> None: + # The household-batched post-export scorer refuses Medicaid dollars, + # which PolicyEngine-US prices through a file-wide state denominator + # (microcosm#956), so the H.R.1 Medicaid probe counts enrollees. Every + # other probe sums dollars, the loader's default unit. + counted = { + probe.id: (probe.budget_measure, probe.min_abs_effect) + for probe in us_release_reform_coverage_probes() + if probe.measure_unit != "usd" + } + assert counted == { + "hr1_medicaid_humanitarian_eligibility_restoration": ( + "medicaid_enrolled", + 10_000.0, + ) + } + def test_shipped_ssi_probe_binds_through_the_assets(self) -> None: probes = us_release_reform_coverage_probes() assert probes, "the shipped manifest must pin at least one reform probe" diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_compiler_ir.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_compiler_ir.py index a59775325c..57f66c2a4b 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_compiler_ir.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_compiler_ir.py @@ -22,7 +22,7 @@ ) from microcosm.build.spec_engine.resolver import F0_CONTRACT_ONLY_KERNEL_IDS -US_SCHEDULE_SHA256 = "f1fb754e6ade602be394837bc0d7d091f2fba4b4b83fb00f59a0258fe56eb5da" +US_SCHEDULE_SHA256 = "46cea046d2fb8223d32772bd6a7807c52791640943949154982dcc52d93fa9eb" @pytest.fixture(scope="module") @@ -107,9 +107,9 @@ def test_us_seed_stream_map_is_complete_and_owner_typed( ) -> None: seed_map = compiled_us.seed_stream_map assert seed_map.protocol_id == "legacy-v1" - assert len(seed_map.sites) == 53 + assert len(seed_map.sites) == 66 assert len(seed_map.owners) == 54 - assert sum(len(site.owners) for site in seed_map.sites) == 112 + assert sum(len(site.owners) for site in seed_map.sites) == 125 assert {site.id for site in seed_map.sites} == { site.id for site in resolved_us.seed_protocol.sites } @@ -244,5 +244,5 @@ def test_bundle_without_imputation_compiles_empty_graph( assert compiled.producer_graph.nodes == () assert compiled.stage_dag.nodes == () assert compiled.nodes == () - assert len(compiled.seed_stream_map.sites) == 53 + assert len(compiled.seed_stream_map.sites) == 66 assert compiled.seed_stream_map.owners == () diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py index 63120bd15d..0f39f8c2a4 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py @@ -95,22 +95,22 @@ def _mutated_bundle( def test_exact_complete_ledger_has_one_primary_mode_per_pointer(field_ledger) -> None: - assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42394 + assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_781 assert field_ledger.source_counts == { - "authored": 32609, - "resolved_bindings": 9_785, + "authored": 32_749, + "resolved_bindings": 10_032, } assert field_ledger.mode_counts == { - "legacy_behavior": 14_269, - "compiler_semantic": 27673, - "front_end_validation": 349, + "legacy_behavior": 14_301, + "compiler_semantic": 28_025, + "front_end_validation": 352, "identity_only": 103, } assert field_ledger.generation0_effect_counts == { - "legacy_behavior": 38_780, - "no_generation0_effect": 3614, + "legacy_behavior": 38_878, + "no_generation0_effect": 3_903, } - assert len({field.pointer for field in field_ledger.fields}) == 42394 + assert len({field.pointer for field in field_ledger.fields}) == 42_781 def test_eligibility_concepts_are_validation_not_generation0_behavior( diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_identity_contracts.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_identity_contracts.py index fce631f3a5..8d1f13e22a 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_identity_contracts.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_identity_contracts.py @@ -118,13 +118,13 @@ def test_identity_contract_objects_are_closed_world( load_schema_registry().validate(mutated, "spine.schema.json") -def test_all_53_seed_sites_resolve_to_typed_real_owners( +def test_all_seed_sites_resolve_to_typed_real_owners( identity_documents: tuple[dict[str, object], list[dict[str, object]]], ) -> None: spine, source_stages = identity_documents bindings = _resolve(spine, source_stages) - assert len(bindings) == len(LEGACY_V1_PROTOCOL.sites) == 53 + assert len(bindings) == len(LEGACY_V1_PROTOCOL.sites) == 66 assert [binding.site for binding in bindings] == [ site.id for site in LEGACY_V1_PROTOCOL.sites ] diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_imputation_semantics.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_imputation_semantics.py index bf0f3b19e2..9a6eefdb37 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_imputation_semantics.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_imputation_semantics.py @@ -98,7 +98,7 @@ def test_imputation_projector_matches_live_plans_and_graph_receipts( for key, expected in live.items(): assert canonical_json_bytes(projected[key]) == canonical_json_bytes(expected) assert projected["late_producer_schedule_receipt"]["schedule_sha256"] == ( - "f1fb754e6ade602be394837bc0d7d091f2fba4b4b83fb00f59a0258fe56eb5da" + "46cea046d2fb8223d32772bd6a7807c52791640943949154982dcc52d93fa9eb" ) assert projected["overlap_ownership"]["sha256"] == ( "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356" diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_legacy_adapter.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_legacy_adapter.py index b2cd0ebc76..0aff5d49c9 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_legacy_adapter.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_legacy_adapter.py @@ -197,10 +197,10 @@ def test_adapter_preserves_generation_zero_identity_components( imputation = legacy_payload["imputation"] assert isinstance(imputation, dict) assert legacy_payload["stacked_authority_receipt"]["sha256"] == ( - "bf681aedc859fd6398ef72da68f584069b7d25b3085c549bf99f82e8324c4eef" + "7cdedd5bf056f1407910eddcd40ad0660bd7ef2d6defe224612c02b3476cbf15" ) assert imputation["late_producer_schedule_receipt"]["schedule_sha256"] == ( - "f1fb754e6ade602be394837bc0d7d091f2fba4b4b83fb00f59a0258fe56eb5da" + "46cea046d2fb8223d32772bd6a7807c52791640943949154982dcc52d93fa9eb" ) assert imputation["overlap_ownership"]["sha256"] == ( "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356" diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py index 26910035b3..7092899401 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py @@ -25,7 +25,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # ssi_disability_criteria's attested bytes. # The opt-in cross-fitted gate calibration moves the fit kernels' bytes. assert first.spec_sha256 == ( - "5b39d83cafb6f312ce73e5550ab36da6676ce265779d615f49c7987f5cc5d36e" + "97d33501596875471497481f9ca40b84b9b4c3aff2362c9625ea336f1273c613" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_plan_lock.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_plan_lock.py index 78df801e0d..04025e2dc0 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_plan_lock.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_plan_lock.py @@ -42,7 +42,7 @@ def test_plan_lock_is_complete_and_closed_world(compiled_us: CompiledSpecIR) -> assert payload["spec_binding"]["attestation"] == "mirror-attested" assert len(payload["producer_graph"]["nodes"]) == 38 assert len(payload["producer_graph"]["authored"]["ownership_matrix"]) == 18 - assert len(payload["seed_stream_map"]["sites"]) == 53 + assert len(payload["seed_stream_map"]["sites"]) == 66 assert len(payload["nodes"]) == 38 mutated = copy.deepcopy(payload) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_seeds.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_seeds.py index bba3a43b43..1cb2653cc2 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_seeds.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_seeds.py @@ -45,6 +45,19 @@ "housing_inputs_training_cap", "immigration_ead_students_assignment", "immigration_ead_workers_assignment", + "immigration_humanitarian_asylee_assignment", + "immigration_humanitarian_deportation_withheld_assignment", + "immigration_humanitarian_paroled_one_year_afghanistan_assignment", + "immigration_humanitarian_paroled_one_year_nicaragua_assignment", + "immigration_humanitarian_paroled_one_year_ukraine_assignment", + "immigration_humanitarian_paroled_one_year_venezuela_assignment", + "immigration_humanitarian_refugee_assignment", + "immigration_humanitarian_tps_el_salvador_assignment", + "immigration_humanitarian_tps_honduras_assignment", + "immigration_humanitarian_tps_nepal_assignment", + "immigration_humanitarian_tps_nicaragua_assignment", + "immigration_humanitarian_tps_other_designated_assignment", + "immigration_humanitarian_tps_venezuela_assignment", "legacy_congressional_district_assignment", "legacy_geography_ladder", "legacy_puma_ladder", @@ -142,6 +155,19 @@ ( "immigration_ead_workers_assignment", "immigration_ead_students_assignment", + "immigration_humanitarian_paroled_one_year_afghanistan_assignment", + "immigration_humanitarian_paroled_one_year_ukraine_assignment", + "immigration_humanitarian_paroled_one_year_nicaragua_assignment", + "immigration_humanitarian_paroled_one_year_venezuela_assignment", + "immigration_humanitarian_refugee_assignment", + "immigration_humanitarian_asylee_assignment", + "immigration_humanitarian_deportation_withheld_assignment", + "immigration_humanitarian_tps_venezuela_assignment", + "immigration_humanitarian_tps_el_salvador_assignment", + "immigration_humanitarian_tps_honduras_assignment", + "immigration_humanitarian_tps_nicaragua_assignment", + "immigration_humanitarian_tps_nepal_assignment", + "immigration_humanitarian_tps_other_designated_assignment", ), "packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py", ), diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_evidence_kernels.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_evidence_kernels.py index 77c631bcdc..f9191fd13b 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_evidence_kernels.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_evidence_kernels.py @@ -168,6 +168,9 @@ "microcosm.data.source_enrichment", "microcosm.data.stored_inputs", "microcosm.data.us_critical_targets", + # microcosm#767: us_runtime.acs_transfer reads its immigration + # evidence at us_runtime.acs_pums's ACS_2024_1YR_VINTAGE. + "microcosm.build.us_runtime.acs_pums", ] ) BATTERY = "toy.gates.terminal" diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_immigration.py b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_immigration.py index 2eb95df032..3fc18a3ab4 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_immigration.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_immigration.py @@ -2,9 +2,14 @@ from __future__ import annotations +import dataclasses +import functools + import numpy as np import pandas as pd import pytest +from hypothesis import given, settings +from hypothesis import strategies as st from microcosm.build.us_runtime import acs_local_immigration from microcosm.build.us_runtime.acs_local_immigration import ( @@ -17,6 +22,9 @@ YEARS_SINCE_US_ENTRY_COLUMN, _acs_cps_view, _acs_draw_keys, + _cps_labor_force_status, + _humanitarian_reconciliation, + _status_reconciliation, acs_local_immigration_receipt_failures, acs_local_immigration_signal_gate, require_acs_local_immigration_donor, @@ -26,11 +34,15 @@ from microcosm.build.us_runtime.acs_transfer import ASEC_PUF_DONOR_SPINE from microcosm.build.us_runtime.base_pool import spine_column from microcosm.build.us_runtime.immigration import ( + _CPS_LABOR_FORCE_STATUS_CODES, US_IMMIGRATION_REQUIRED_SOURCE_COLUMNS, + HumanitarianDraw, + ImmigrationControls, UndocumentedControls, _assign_ssn_card_codes, _derive_immigration_status, - _packaged_controls, + _pew_unauthorized_rows, + us_immigration_controls, ) from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights @@ -66,6 +78,9 @@ _ASEC_BASE: dict[str, object] = { "PRCITSHP": 1, "PEINUSYR": 0, + # PENATVTY 57: born in the United States. The humanitarian cohorts and + # the whole-file gate read the birth country on every row (microcosm#767). + "PENATVTY": 57, "WSAL_VAL": 0.0, "SEMP_VAL": 0.0, "A_HSCOL": 0, @@ -124,24 +139,28 @@ def _asec_population() -> list[dict[str, object]]: "ssn_card_type": "OTHER_NON_CITIZEN", "immigration_status_str": "LEGAL_PERMANENT_RESIDENT", } + mexico = {"PENATVTY": 303} return ( [{} for _ in range(88)] - + [{"PRCITSHP": 4, "PEINUSYR": 20} for _ in range(5)] - + [{"PRCITSHP": 5, "PEINUSYR": 24, **lpr} for _ in range(3)] + + [{"PRCITSHP": 4, "PEINUSYR": 20, **mexico} for _ in range(5)] + + [{"PRCITSHP": 5, "PEINUSYR": 24, **mexico, **lpr} for _ in range(3)] + [ { + # Code 28 is the 2024 ASEC's last bin (2022-2024). "PRCITSHP": 5, - "PEINUSYR": 29, + "PEINUSYR": 28, + **mexico, "ssn_card_type": "NON_CITIZEN_VALID_EAD", "immigration_status_str": "LEGAL_PERMANENT_RESIDENT", } ] + [ # Undocumented workers: the donor already delivers 3.3M of the - # 8.3M worker control on the pooled weights. + # packaged worker control on the pooled weights. { "PRCITSHP": 5, "PEINUSYR": 24, + **mexico, "WSAL_VAL": 20_000.0, "ssn_card_type": "NONE", "immigration_status_str": "UNDOCUMENTED", @@ -260,6 +279,42 @@ def _person_weights(frame: Frame) -> np.ndarray: return np.asarray(frame.resolve_weights("person").values, dtype=np.float64) +@functools.cache +def _packaged() -> ImmigrationControls: + return us_immigration_controls() + + +def _controls( + undocumented: UndocumentedControls | None = None, + humanitarian: tuple[HumanitarianDraw, ...] = (), +) -> ImmigrationControls: + return ImmigrationControls( + undocumented=_packaged().undocumented if undocumented is None else undocumented, + humanitarian=humanitarian, + ) + + +@pytest.fixture(autouse=True) +def _undocumented_controls_only(monkeypatch) -> None: + """Run the stage and its gate on the packaged undocumented controls alone. + + The #1020/#1052 cases predate the microcosm#767 humanitarian draws, and + every foreign-born person in their populations was born in Mexico or + Cuba/Haiti, outside every draw's origins. The packaged draws would select + no one and the whole-file stock bands would fail, so these cases run with + no draws; the humanitarian cases install their own with + :func:`_install_controls`. + """ + + _install_controls(monkeypatch, _controls()) + + +def _install_controls(monkeypatch, controls: ImmigrationControls) -> None: + monkeypatch.setattr( + acs_local_immigration, "us_immigration_controls", lambda: controls + ) + + def _filled(frame: Frame | None = None, *, seed: int = 0) -> Frame: return with_acs_local_immigration_inputs( _frame() if frame is None else frame, seed=seed, time_period=PERIOD @@ -350,7 +405,6 @@ def test_acs_legal_status_indicators_mark_other_non_citizen(indicator) -> None: {"COW": 1}, {"COW": 2}, {"COW": 6}, - {"ESR": 1}, {"ESR": np.nan}, {"SCHG": 14}, {"YOEP": 1982}, @@ -365,6 +419,26 @@ def test_non_indicators_leave_a_non_worker_undocumented(non_indicator) -> None: assert person["immigration_status_str"].iloc[-2] == "UNDOCUMENTED" +@pytest.mark.parametrize("esr", [1, 2, 3]) +def test_a_civilian_labor_force_esr_is_a_worker_not_an_indicator(esr) -> None: + """ESR 1-3 is not the Armed Forces indicator (4/5): the person stays in + the residual pool, where the kernel counts them as a worker.""" + + _, before = with_acs_local_immigration_inputs(_frame(), seed=0, time_period=PERIOD) + result, after = with_acs_local_immigration_inputs( + _with_extra_acs_person(ESR=esr), seed=0, time_period=PERIOD + ) + assert result.table("person")["ssn_card_type"].iloc[-2] in { + "NONE", + "NON_CITIZEN_VALID_EAD", + } + capacity = [ + receipt["controls"]["statuses"]["undocumented_workers"]["acs_capacity"] + for receipt in (before, after) + ] + assert capacity[1] - capacity[0] == pytest.approx(0.55e6) + + def test_cuban_haitian_class_uses_pobp_and_the_exact_entry_year() -> None: cuban = _filled(_with_extra_acs_person(POBP=327, HINS4=1)).table("person") assert cuban["immigration_status_str"].iloc[-2] == "CUBAN_HAITIAN_ENTRANT" @@ -396,12 +470,34 @@ def test_cps_view_covers_the_kernel_inputs_and_zeros_unmapped_indicators() -> No blank = view.iloc[-2] assert (blank["WSAL_VAL"], blank["SEMP_VAL"]) == (0.0, 0.0) assert (blank["PEIO1COW"], blank["A_HSCOL"], blank["A_MJOCC"]) == (0, 0, 0) + assert blank["A_LFSR"] == 0 assert blank["PEAFEVER"] == 2 # US-born rows have no entry year. assert np.isnan(entry_year[person["CIT"].to_numpy() == 1]).all() assert entry_year[-2] == 2015 +@pytest.mark.parametrize( + ("esr", "a_lfsr"), + [(1, 1), (2, 2), (3, 3), (4, 0), (5, 0), (6, 7), (np.nan, 0)], +) +def test_esr_is_the_cps_labor_force_status(esr, a_lfsr) -> None: + """The kernel's worker test reads A_LFSR; ACS ESR carries the same status.""" + + frame = _with_extra_acs_person(ESR=esr, AGEP=35) + person = frame.table("person").loc[_acs(frame)] + view, _ = _acs_cps_view(person, time_period=PERIOD) + assert view["A_LFSR"].iloc[-2] == a_lfsr + assert "A_LFSR" in acs_local_immigration.ACS_TO_CPS_INDICATOR_MAPPING + + +def test_an_esr_outside_the_codebook_is_refused() -> None: + with pytest.raises(ValueError, match="ACS ESR"): + with_acs_local_immigration_inputs( + _with_extra_acs_person(ESR=9), seed=0, time_period=PERIOD + ) + + def _cps_row(**overrides: object) -> dict[str, object]: row = { "PRCITSHP": 5, @@ -411,6 +507,8 @@ def _cps_row(**overrides: object) -> dict[str, object]: "A_MARITL": 7, "A_SPOUSE": 0, "A_HSCOL": 0, + # 7: not in the labor force. Workers set 1 (employed, at work). + "A_LFSR": 7, "WSAL_VAL": 0.0, "SEMP_VAL": 0.0, "MCARE": 2, @@ -439,8 +537,10 @@ def test_exact_arrival_years_reproduce_the_peinusyr_bins() -> None: rows = [ (_cps_row(PEINUSYR=5), 1972.0), # IRCA cohort: documented (_cps_row(PEINUSYR=24), 2015.0), # residual - (_cps_row(PEINUSYR=19, A_AGE=29, WSAL_VAL=20_000.0), 2005.0), # DACA - (_cps_row(PEINUSYR=24, A_AGE=40, WSAL_VAL=20_000.0), 2015.0), # EAD LPR + # DACA + (_cps_row(PEINUSYR=19, A_AGE=29, WSAL_VAL=20_000.0, A_LFSR=1), 2005.0), + # EAD LPR + (_cps_row(PEINUSYR=24, A_AGE=40, WSAL_VAL=20_000.0, A_LFSR=1), 2015.0), (_cps_row(PENATVTY=327, CAID=1), 2015.0), # Cuban entrant (_cps_row(PENATVTY=332, PEINUSYR=5, MCARE=1), 1972.0), # pre-1980 Haiti (_cps_row(PRCITSHP=4, PEINUSYR=20), 2007.0), # naturalized @@ -449,19 +549,23 @@ def test_exact_arrival_years_reproduce_the_peinusyr_bins() -> None: person = pd.DataFrame([row for row, _ in rows]) person.insert(0, "person_id", np.arange(1, len(person) + 1)) arrival = np.asarray([year for _, year in rows]) - controls = UndocumentedControls( - workers=0.001, - students=0.001, - population_anchor=1.0, - sources={ - "undocumented_workers": "https://example.com/w", - "undocumented_students": "https://example.com/s", - "undocumented_population_anchor": "https://example.com/p", - }, + controls = _controls( + UndocumentedControls( + workers=0.001, + students=0.001, + population_anchor=1.0, + sources={ + "undocumented_workers": "https://example.com/w", + "undocumented_students": "https://example.com/s", + "undocumented_population_anchor": "https://example.com/p", + }, + ) ) weights = np.ones(len(person)) - binned = _assign_ssn_card_codes(person, weights, seed=0, controls=controls) - exact = _assign_ssn_card_codes( + binned, binned_marks = _assign_ssn_card_codes( + person, weights, seed=0, controls=controls, time_period=PERIOD + ) + exact, exact_marks = _assign_ssn_card_codes( person.drop(columns=["PEINUSYR"]), weights, seed=0, @@ -470,11 +574,15 @@ def test_exact_arrival_years_reproduce_the_peinusyr_bins() -> None: time_period=PERIOD, ) assert binned.tolist() == exact.tolist() == [3, 0, 2, 2, 3, 3, 1, 1] + assert binned_marks.tolist() == exact_marks.tolist() == [""] * len(rows) assert ( - _derive_immigration_status(person, binned, time_period=PERIOD).tolist() + _derive_immigration_status( + person, binned, binned_marks, time_period=PERIOD + ).tolist() == _derive_immigration_status( person.drop(columns=["PEINUSYR"]), exact, + exact_marks, time_period=PERIOD, arrival_year=arrival, ).tolist() @@ -515,7 +623,7 @@ def test_acs_rows_target_the_donor_residual() -> None: ) weights = _person_weights(frame) acs = _acs(frame) - national = _packaged_controls() + national = _packaged().undocumented controls = receipt["controls"] assert controls["method"] == ACS_LOCAL_IMMIGRATION_CONTROLS_METHOD assert controls["acs_person_weight_share"] == pytest.approx(2 / 3) @@ -529,8 +637,9 @@ def test_acs_rows_target_the_donor_residual() -> None: assert workers["residual"] == pytest.approx(national.workers - 3 * 1.1e6) assert workers["acs_capacity"] == pytest.approx(14 * 0.55e6) person = result.table("person") - worker = (person["WAGP"].fillna(0) > 0).to_numpy() - remaining = weights[acs & worker & person["ssn_card_type"].eq("NONE").to_numpy()] + # The kernel's worker test on ACS rows: ESR employed or unemployed, 16+. + worker = (person["ESR"].isin([1, 2, 3]) & (person["AGEP"] >= 16)).to_numpy() + remaining = weights[acs & worker & _pew_unauthorized_rows(person)] assert workers["acs_assigned"] == pytest.approx(remaining.sum()) # The spill stops at the first record that reaches the residual. residual = workers["residual"] @@ -540,19 +649,20 @@ def test_acs_rows_target_the_donor_residual() -> None: students = controls["statuses"]["undocumented_students"] assert students["donor_delivered"] == 0.0 assert students["residual"] == national.students + assert controls["humanitarian"] == {} assert acs_local_immigration_receipt_failures(receipt) == [] assert receipt["acs_composition"]["non_citizen_share"] == pytest.approx(0.07) def _custom_controls(monkeypatch, *, workers: float, students: float): - national = _packaged_controls() + national = _packaged().undocumented controls = UndocumentedControls( workers=workers, students=students, population_anchor=national.population_anchor, sources=national.sources, ) - monkeypatch.setattr(acs_local_immigration, "_packaged_controls", lambda: controls) + _install_controls(monkeypatch, _controls(controls)) return controls @@ -585,11 +695,15 @@ def test_pooled_counts_meet_the_controls_not_f_plus_a_times_them(monkeypatch): frame, seed=3, time_period=PERIOD ) person = result.table("person") - none = person["ssn_card_type"].eq("NONE").to_numpy() - worker = (person["WAGP"].fillna(0) > 0) | (person["WSAL_VAL"].fillna(0) > 0) + unauthorized = _pew_unauthorized_rows(person) + # ACS rows: ESR employed or unemployed at 16+; donor rows carry no A_LFSR, + # so the earnings proxy. + worker = (person["ESR"].isin([1, 2, 3]) & (person["AGEP"] >= 16)) | ( + person["WSAL_VAL"].fillna(0) > 0 + ) student = person["SCHG"].isin([15, 16]) | person["A_HSCOL"].eq(2) - pooled_workers = weights[none & worker.to_numpy()].sum() - pooled_students = weights[none & student.to_numpy()].sum() + pooled_workers = weights[unauthorized & worker.to_numpy()].sum() + pooled_students = weights[unauthorized & student.to_numpy()].sum() assert pooled_workers == controls.workers assert pooled_students == controls.students statuses = receipt["controls"]["statuses"] @@ -612,21 +726,24 @@ def test_pooled_counts_meet_the_controls_not_f_plus_a_times_them(monkeypatch): # The pre-review rule targeted the ACS rows at A x control. acs_person = frame.table("person").loc[acs] view, entry_year = _acs_cps_view(acs_person, time_period=PERIOD) - old = _assign_ssn_card_codes( + old, _ = _assign_ssn_card_codes( view, weights[acs], seed=3, - controls=UndocumentedControls( - workers=share * controls.workers, - students=share * controls.students, - population_anchor=controls.population_anchor, - sources=controls.sources, + controls=_controls( + UndocumentedControls( + workers=share * controls.workers, + students=share * controls.students, + population_anchor=controls.population_anchor, + sources=controls.sources, + ) ), arrival_year=entry_year, time_period=PERIOD, draw_keys=_acs_draw_keys(frame.table("household"), acs_person), ) - old_acs = weights[acs][(old == 0) & (view["WSAL_VAL"].to_numpy() > 0)].sum() + old_workers = np.isin(view["A_LFSR"].to_numpy(), (1, 2, 3, 4)) + old_acs = weights[acs][(old == 0) & old_workers].sum() old_pooled = 6 * u + old_acs assert abs(old_pooled - (0.5 + share) * controls.workers) <= u assert old_pooled > controls.workers + u @@ -644,8 +761,8 @@ def test_a_donor_over_the_control_leaves_no_acs_residual(monkeypatch) -> None: assert workers["acs_assigned"] == 0.0 person = result.table("person") acs = _acs(result) - acs_workers = acs & (person["WAGP"].fillna(0) > 0).to_numpy() - assert not person.loc[acs_workers, "ssn_card_type"].eq("NONE").any() + acs_workers = acs & person["ESR"].isin([1, 2, 3]).to_numpy() + assert not _pew_unauthorized_rows(person)[acs_workers].any() assert acs_local_immigration_receipt_failures(receipt) == [] @@ -687,6 +804,32 @@ def test_donor_status_columns_fall_back_to_harmonized_inputs() -> None: ] == pytest.approx(3 * 1.1e6) +def test_a_donor_carrying_a_lfsr_is_measured_with_the_kernels_worker_test(): + """A donor with ``A_LFSR`` is measured as the kernel labels: in the labor + force at 16+, not positive earnings last year.""" + + frame = _frame() + person = frame.table("person").copy() + asec = ~_acs(frame) + person["A_LFSR"] = np.where(asec, 7.0, np.nan) + workers = np.flatnonzero(asec & person["WSAL_VAL"].gt(0).to_numpy()) + assert len(workers) == 3 + # Two employed workers, and one unemployed with no earnings last year: + # the kernel counts all three; the earnings proxy would count two. + person.loc[person.index[workers[:2]], "A_LFSR"] = 1.0 + person.loc[person.index[workers[2]], ["A_LFSR", "WSAL_VAL"]] = [3.0, 0.0] + _, receipt = with_acs_local_immigration_inputs( + _with_person(frame, person), seed=5, time_period=PERIOD + ) + controls = receipt["controls"] + assert controls["donor_status_columns"]["worker"] == ( + "A_LFSR in [1, 2, 3, 4] and age >= 16" + ) + assert controls["statuses"]["undocumented_workers"][ + "donor_delivered" + ] == pytest.approx(3 * 1.1e6) + + def test_a_donor_without_worker_or_student_columns_is_refused() -> None: frame = _frame() person = frame.table("person").drop(columns=["WSAL_VAL"]) @@ -770,6 +913,263 @@ def test_receipt_failures_name_a_target_that_is_not_the_residual(controls, match assert any(match in failure for failure in failures), failures +def _draw_entry(target: float, donor: float, assigned: float) -> dict[str, float]: + return { + "target": target, + "donor_delivered": donor, + "residual": max(0.0, target - donor), + "acs_assigned": assigned, + "pooled_total": donor + assigned, + } + + +def _receipt_controls(humanitarian: object) -> dict[str, object]: + return { + "method": ACS_LOCAL_IMMIGRATION_CONTROLS_METHOD, + "statuses": { + "undocumented_workers": _status(9.7e6, 3e6, 6.7e6), + "undocumented_students": _status(4.08e5, 0.0, 4.08e5), + }, + "humanitarian": humanitarian, + } + + +@pytest.mark.parametrize( + ("humanitarian", "match"), + [ + (None, "humanitarian is missing"), + ({"refugee": _draw_entry(2e5, 1e5, 1e5)}, "missing ['tps:venezuela']"), + ( + { + "refugee": _draw_entry(2e5, 1e5, 1e5), + "tps:venezuela": _draw_entry(6e5, 2e5, 4e5), + "asylee": _draw_entry(1e5, 0.0, 1e5), + }, + "unexpected ['asylee']", + ), + ( + { + "refugee": {**_draw_entry(2e5, 1e5, 1e5), "residual": 2e5}, + "tps:venezuela": _draw_entry(6e5, 2e5, 4e5), + }, + "refugee: residual", + ), + ( + { + "refugee": _draw_entry(2e5, 1e5, 1e5), + "tps:venezuela": {**_draw_entry(6e5, 2e5, 4e5), "pooled_total": 4e5}, + }, + "tps:venezuela: pooled total", + ), + ( + { + "refugee": {"target": 2e5}, + "tps:venezuela": _draw_entry(6e5, 2e5, 4e5), + }, + "humanitarian.refugee lacks", + ), + ], + ids=[ + "no-humanitarian", + "missing-draw", + "unexpected-draw", + "residual-not-residual", + "pooled-not-sum", + "non-numeric", + ], +) +def test_receipt_failures_name_a_humanitarian_target_that_is_not_the_residual( + monkeypatch, humanitarian, match +): + _install_controls(monkeypatch, _controls(humanitarian=_DRAWS)) + controls = _receipt_controls(humanitarian) + if humanitarian is None: + del controls["humanitarian"] + failures = acs_local_immigration_receipt_failures({"controls": controls}) + assert failures + assert any(match in failure for failure in failures), failures + valid = _receipt_controls( + { + "refugee": _draw_entry(2e5, 1e5, 1e5), + "tps:venezuela": _draw_entry(6e5, 2e5, 4e5), + } + ) + assert acs_local_immigration_receipt_failures({"controls": valid}) == [] + + +# -------------------------------------------------------------------------- +# Humanitarian stocks (microcosm#767) +# -------------------------------------------------------------------------- + +#: One ACS person's weight; a donor person weighs 2U. +U = 0.55e6 +_DRAWS = ( + HumanitarianDraw( + category="refugee", + origin=None, + status="REFUGEE", + target=2.5 * U, + source="https://example.com/refugee", + ), + HumanitarianDraw( + category="tps", + origin="venezuela", + status="TPS", + target=5 * U, + source="https://example.com/tps-venezuela", + ), +) + + +def _humanitarian_frame() -> Frame: + """The default frame plus humanitarian origins on both spines. + + The donor already carries one Venezuelan TPS holder and one Congolese + refugee (2U each). The ACS rows add four Venezuelan-born 2022 arrivals + with no legal-status indicator (the residual pool, so TPS candidates) + and two Congolese-born 2023 arrivals on Medicaid (documented, so refugee + candidates), each with a US-born partner. + """ + + asec_rows = _asec_population() + [ + { + "PRCITSHP": 5, + "PEINUSYR": 28, + "PENATVTY": 373, + "ssn_card_type": "NON_CITIZEN_VALID_EAD", + "immigration_status_str": "TPS", + }, + { + "PRCITSHP": 5, + "PEINUSYR": 28, + "PENATVTY": 412, + "ssn_card_type": "OTHER_NON_CITIZEN", + "immigration_status_str": "REFUGEE", + }, + ] + acs_rows = _acs_population() + for _ in range(4): + acs_rows += [_noncitizen(POBP=373, YOEP=2022), {}] + for _ in range(2): + acs_rows += [_noncitizen(POBP=412, YOEP=2023, HINS4=1), {}] + return _frame(asec_rows=asec_rows, acs_rows=acs_rows) + + +def _added_acs_rows(person: pd.DataFrame, birth: int) -> pd.DataFrame: + acs = person[TAG].eq(ACS_2024_1YR_SPINE) + return person.loc[acs & person["POBP"].eq(birth)] + + +def test_humanitarian_draws_target_the_donor_residual_per_draw(monkeypatch): + """Each cited stock: donor delivered plus ACS assigned pools to it. + + TPS (Venezuela): target 5U, donor 2U, so the ACS rows draw 3U of their + four candidates. Refugee: target 2.5U, donor 2U, so the residual 0.5U + draws one ACS candidate (U), the first record to reach it. + """ + + _install_controls(monkeypatch, _controls(humanitarian=_DRAWS)) + result, receipt = with_acs_local_immigration_inputs( + _humanitarian_frame(), seed=7, time_period=PERIOD + ) + humanitarian = receipt["controls"]["humanitarian"] + assert set(humanitarian) == {"refugee", "tps:venezuela"} + assert humanitarian["tps:venezuela"] == { + "category": "tps", + "origin": "venezuela", + "status": "TPS", + "target": 5 * U, + "donor_delivered": 2 * U, + "residual": 3 * U, + "acs_assigned": 3 * U, + "pooled_total": 5 * U, + "pooled_relative": 1.0, + "donor_excess": 0.0, + "source": "https://example.com/tps-venezuela", + } + refugee = humanitarian["refugee"] + assert refugee["residual"] == pytest.approx(0.5 * U) + assert refugee["acs_assigned"] == pytest.approx(U) + assert refugee["pooled_total"] == pytest.approx(3 * U) + assert acs_local_immigration_receipt_failures(receipt) == [] + + person = result.table("person") + venezuelan = _added_acs_rows(person, 373) + tps = venezuelan["immigration_status_str"].eq("TPS") + assert int(tps.sum()) == 3 + # Residual-pool selections hold an EAD; the one left is undocumented. + assert venezuelan.loc[tps, "ssn_card_type"].eq("NON_CITIZEN_VALID_EAD").all() + assert venezuelan.loc[~tps, list(LABELS)].values.tolist() == [ + ["NONE", "UNDOCUMENTED"] + ] + congolese = _added_acs_rows(person, 412) + assert sorted(congolese["immigration_status_str"]) == [ + "LEGAL_PERMANENT_RESIDENT", + "REFUGEE", + ] + assert congolese["ssn_card_type"].eq("OTHER_NON_CITIZEN").all() + # TPS holders stay inside Pew's universe; refugees do not. + unauthorized = _pew_unauthorized_rows(person) + assert unauthorized[person.index.get_indexer(venezuelan.index)].all() + assert not unauthorized[person.index.get_indexer(congolese.index)].any() + + gate = acs_local_immigration_signal_gate(result, time_period=PERIOD) + assert gate.passed, gate.failures + achieved = gate.details["file"]["details"]["humanitarian_draw_achieved"] + assert achieved["tps:venezuela"]["population"] == pytest.approx(5 * U) + assert achieved["refugee"]["population"] == pytest.approx(3 * U) + + +def test_a_donor_over_a_humanitarian_stock_leaves_no_acs_residual(monkeypatch): + """The ACS rows only add: a donor at 2x the stock fails the whole-file + band, and the receipt records the excess.""" + + over = ( + HumanitarianDraw( + category="refugee", + origin=None, + status="REFUGEE", + target=U, + source="https://example.com/refugee", + ), + _DRAWS[1], + ) + _install_controls(monkeypatch, _controls(humanitarian=over)) + result, receipt = with_acs_local_immigration_inputs( + _humanitarian_frame(), seed=7, time_period=PERIOD + ) + refugee = receipt["controls"]["humanitarian"]["refugee"] + assert refugee["residual"] == 0.0 + assert refugee["acs_assigned"] == 0.0 + assert refugee["donor_excess"] == pytest.approx(U) + assert refugee["pooled_relative"] == pytest.approx(2.0) + assert acs_local_immigration_receipt_failures(receipt) == [] + person = result.table("person") + assert ( + not _added_acs_rows(person, 412)["immigration_status_str"].eq("REFUGEE").any() + ) + gate = acs_local_immigration_signal_gate(result, time_period=PERIOD) + assert not gate.passed + assert any( + failure.startswith("file: refugee: compatible emitted population") + for failure in gate.failures + ), gate.failures + + +def test_the_gate_grades_the_controls_it_is_given(monkeypatch) -> None: + _install_controls(monkeypatch, _controls(humanitarian=_DRAWS)) + result, _ = with_acs_local_immigration_inputs( + _humanitarian_frame(), seed=7, time_period=PERIOD + ) + assert acs_local_immigration_signal_gate(result, time_period=PERIOD).passed + # Without the draws, every emitted humanitarian status is unexpected. + bare = acs_local_immigration_signal_gate( + result, time_period=PERIOD, controls=_controls() + ) + assert not bare.passed + assert any("explicit zero target" in failure for failure in bare.failures) + + def test_entry_clock_is_an_arrival_proxy_with_a_lag_sensitivity() -> None: """#1052 review: LPRs at 5, 7 and 3 years since arrival flip at lags 1+, 3+ and never; the receipt also counts every other bar-status person.""" @@ -914,11 +1314,20 @@ def test_years_since_us_entry_on_both_spines() -> None: asec_rows = _asec_population() + [ { "PRCITSHP": 5, - "PEINUSYR": 29, + "PEINUSYR": 28, + "PENATVTY": 303, "age": 30.0, "ssn_card_type": "OTHER_NON_CITIZEN", "immigration_status_str": "LEGAL_PERMANENT_RESIDENT", }, + { + "PRCITSHP": 5, + "PEINUSYR": 29, + "PENATVTY": 303, + "age": 1.0, + "ssn_card_type": "OTHER_NON_CITIZEN", + "immigration_status_str": "LEGAL_PERMANENT_RESIDENT", + }, {"PRCITSHP": 2, "PEINUSYR": 0, "age": 61.0}, ] acs_rows = _acs_population() + [ @@ -940,7 +1349,11 @@ def test_years_since_us_entry_on_both_spines() -> None: assert (asec.loc[native, YEARS] == asec.loc[native, "age"]).all() assert (asec.loc[asec["PEINUSYR"].eq(24), YEARS] == PERIOD - 2015).all() assert (asec.loc[asec["PEINUSYR"].eq(20), YEARS] == PERIOD - 2007).all() - assert asec[YEARS].iloc[-2] == 0.0 # code 29: 2024 arrivals + # Code 28, the 2024 ASEC's last bin (2022-2024), reads as its 2023 + # midpoint. Code 29, which only the 2026 ASEC uses (2024-2026), reads as + # the kernel's 2024: a pooled default-pool donor is never refused for it. + assert asec[YEARS].iloc[-3] == 1.0 + assert asec[YEARS].iloc[-2] == 0.0 assert asec[YEARS].iloc[-1] == 61.0 # born in Puerto Rico: age acs_native = acs["CIT"].isin([1, 2, 3]) assert (acs.loc[acs_native, YEARS] == acs.loc[acs_native, "AGEP"]).all() @@ -1074,6 +1487,7 @@ def test_donor_preflight_accepts_a_complete_donor() -> None: ), "PEINUSYR", ), + (lambda person: person.drop(columns="PENATVTY"), "PENATVTY"), ], ) def test_donor_preflight_refuses_an_unusable_donor(change, match) -> None: @@ -1091,7 +1505,7 @@ def test_donor_preflight_refuses_an_unusable_donor(change, match) -> None: def test_gate_passes_on_the_filled_surface() -> None: - gate = acs_local_immigration_signal_gate(_filled()) + gate = acs_local_immigration_signal_gate(_filled(), time_period=PERIOD) assert gate.passed, gate.failures assert gate.name == ACS_LOCAL_IMMIGRATION_GATE_NAME acs = gate.details["per_spine"][ACS_2024_1YR_SPINE] @@ -1112,7 +1526,9 @@ def test_gate_fails_on_the_default_filled_release_signature() -> None: for column in LABELS: person[column] = person[column].where(~acs, "CITIZEN") person[YEARS] = ENGINE_DEFAULT_YEARS_SINCE_US_ENTRY - gate = acs_local_immigration_signal_gate(_with_person(frame, person)) + gate = acs_local_immigration_signal_gate( + _with_person(frame, person), time_period=PERIOD + ) assert not gate.passed failures = " ".join(gate.failures) for column in LABELS: @@ -1124,7 +1540,7 @@ def test_gate_fails_on_the_default_filled_release_signature() -> None: def test_gate_fails_on_missing_cells_and_columns() -> None: - gate = acs_local_immigration_signal_gate(_frame()) + gate = acs_local_immigration_signal_gate(_frame(), time_period=PERIOD) assert not gate.passed failures = " ".join(gate.failures) assert f"{ACS_2024_1YR_SPINE}: ssn_card_type has 400 missing row(s)" in failures @@ -1132,7 +1548,9 @@ def test_gate_fails_on_missing_cells_and_columns() -> None: assert f"{spine}: missing {YEARS}" in failures person = _filled().table("person").copy() person.loc[person.index[0], YEARS] = np.nan - missing = acs_local_immigration_signal_gate(_with_person(_filled(), person)) + missing = acs_local_immigration_signal_gate( + _with_person(_filled(), person), time_period=PERIOD + ) assert missing.failures == ( f"{ASEC_PUF_DONOR_SPINE}: {YEARS} has 1 missing row(s).", ) @@ -1146,10 +1564,14 @@ def test_gate_fails_when_labels_contradict_cit_or_leave_the_band() -> None: "OTHER_NON_CITIZEN", "LEGAL_PERMANENT_RESIDENT", ] - contradicted = acs_local_immigration_signal_gate(_with_person(frame, person)) + contradicted = acs_local_immigration_signal_gate( + _with_person(frame, person), time_period=PERIOD + ) assert contradicted.failures == ( f"{ACS_2024_1YR_SPINE}: 1 person(s) are labelled against their measured " "CIT citizenship.", + "file: 1 person(s) have emitted citizenship disagreeing with their source " + "PRCITSHP/CIT evidence.", ) person = frame.table("person").copy() person["CIT"] = person["CIT"].where(~_acs(frame), 5) @@ -1157,7 +1579,9 @@ def test_gate_fails_when_labels_contradict_cit_or_leave_the_band() -> None: "OTHER_NON_CITIZEN", "LEGAL_PERMANENT_RESIDENT", ] - out_of_band = acs_local_immigration_signal_gate(_with_person(frame, person)) + out_of_band = acs_local_immigration_signal_gate( + _with_person(frame, person), time_period=PERIOD + ) assert any( failure.startswith(f"{ACS_2024_1YR_SPINE}: non-citizen weighted share 1.0000") for failure in out_of_band.failures @@ -1174,7 +1598,9 @@ def test_gate_reports_but_does_not_grade_donor_composition() -> None: "OTHER_NON_CITIZEN", "LEGAL_PERMANENT_RESIDENT", ] - gate = acs_local_immigration_signal_gate(_with_person(frame, person)) + gate = acs_local_immigration_signal_gate( + _with_person(frame, person), time_period=PERIOD + ) donor = gate.details["per_spine"][ASEC_PUF_DONOR_SPINE] assert donor["composition"]["non_citizen_share"] == pytest.approx(0.47) assert "non_citizen_share" not in donor @@ -1194,7 +1620,7 @@ def test_gate_checks_the_whole_file_anchor() -> None: ) }, ) - gate = acs_local_immigration_signal_gate(tenth) + gate = acs_local_immigration_signal_gate(tenth, time_period=PERIOD) assert not gate.passed assert all(failure.startswith("file: ") for failure in gate.failures) assert "published anchor" in " ".join(gate.failures) @@ -1204,12 +1630,202 @@ def test_gate_fails_on_missing_tags_or_unknown_spines() -> None: frame = _filled() person = frame.table("person") untagged = acs_local_immigration_signal_gate( - _with_person(frame, person.assign(**{TAG: person[TAG].where(person.index > 0)})) + _with_person( + frame, person.assign(**{TAG: person[TAG].where(person.index > 0)}) + ), + time_period=PERIOD, ) assert untagged.failures == (f"Missing person origin tags: {TAG}.",) unknown = acs_local_immigration_signal_gate( _with_person( frame, person.assign(**{TAG: person[TAG].where(person.index > 0, "other")}) - ) + ), + time_period=PERIOD, ) assert "unsupported spine" in " ".join(unknown.failures) + + +# -------------------------------------------------------------------------- +# Properties (the microcosm#767 port of #1052) +# -------------------------------------------------------------------------- +# +# Each property restates a mechanism the module states; the *deduced* one +# combines two of them. +# +# - ``_cps_labor_force_status``: every ACS ``ESR`` code maps onto the CPS +# ``A_LFSR`` codebook, and a blank is code 0 ("a blank (under 16) is code +# 0"); any other value, an infinity included, is refused. Differential: +# the mapped codes pass the kernel's worker test +# (``_CPS_LABOR_FORCE_STATUS_CODES``) exactly for ESR 1-3, the civilian +# labor force. +# - ``_status_reconciliation`` and ``_humanitarian_reconciliation`` against +# ``acs_local_immigration_receipt_failures`` (differential: producer +# versus checker): every receipt the producers can write passes the +# checker, and each humanitarian entry splits ``target - donor`` into a +# residual and a donor excess, at most one of them nonzero. +# - The stage, for each humanitarian draw (the ACS rows draw toward +# ``max(0, target - donor-emitted mass)``): the receipt records exactly +# that residual; the donor's delivered mass does not depend on the +# target; the ACS rows emitted with the draw's status weigh exactly +# ``acs_assigned``; a covered stock assigns nothing; a feasible residual +# is reached and overshot by less than one ACS row (every ACS row weighs +# U and draw keys are unique, so each tie group is one row); an +# infeasible one takes every candidate; and (deduced: threshold +# selection) a larger stock selects a superset of a smaller stock's rows. +# +# Independence limits: the ESR table below transcribes the ACS and CPS +# codebooks, written by the author of the mapping; the stage properties take +# each draw's candidate mass from the stage itself (an exhausting stock), +# and only the fixture test pins that mass to the rows the fixture adds. + +_UNIT = settings(max_examples=100, deadline=None, database=None) +_STAGE = settings(max_examples=20, deadline=None, database=None) + +#: ACS ``ESR`` to CPS ``A_LFSR``, from the two codebooks: civilian employed +#: at work and with a job keep 1 and 2; unemployed is 3 (looking); the Armed +#: Forces fall outside the CPS labor-force universe (0); not in the labor +#: force is 7. +_CODEBOOK_ESR_TO_A_LFSR = {1: 1, 2: 2, 3: 3, 4: 0, 5: 0, 6: 7} +_CIVILIAN_LABOR_FORCE_ESR = (1, 2, 3) +_MASS = st.integers(0, 10**8).map(float) +_CONTROL = st.integers(1, 10**8).map(float) +#: A humanitarian stock in quarter-person steps of U, so every sum is exact. +_STOCK = st.integers(0, 40).map(lambda quarters: quarters * U / 4) + + +@_UNIT +@given( + st.lists(st.sampled_from((*_CODEBOOK_ESR_TO_A_LFSR, None)), min_size=1, max_size=40) +) +def test_every_esr_maps_onto_the_cps_labor_force_codebook(codes) -> None: + esr = pd.Series([np.nan if code is None else float(code) for code in codes]) + status = _cps_labor_force_status(esr) + assert status.tolist() == [ + 0 if code is None else _CODEBOOK_ESR_TO_A_LFSR[code] for code in codes + ] + worker = np.isin(status, _CPS_LABOR_FORCE_STATUS_CODES) + assert worker.tolist() == [code in _CIVILIAN_LABOR_FORCE_ESR for code in codes] + + +@_UNIT +@given( + st.one_of( + st.integers(-10_000, 10_000).map(float), + st.floats(allow_nan=False), + ).filter(lambda value: value not in _CODEBOOK_ESR_TO_A_LFSR) +) +def test_any_other_esr_is_refused(value) -> None: + with pytest.raises(ValueError, match="ACS ESR must be one of"): + _cps_labor_force_status(pd.Series([6.0, value])) + + +@_UNIT +@given( + workers=st.tuples(_CONTROL, _MASS, _MASS, _MASS), + students=st.tuples(_CONTROL, _MASS, _MASS, _MASS), + draws=st.tuples(*(st.tuples(_MASS, _MASS, _MASS) for _ in _DRAWS)), +) +def test_every_receipt_the_producers_write_passes_the_checker( + workers, students, draws +) -> None: + humanitarian = {} + for draw, (target, donor, assigned) in zip(_DRAWS, draws, strict=True): + entry = _humanitarian_reconciliation( + dataclasses.replace(draw, target=target), donor, assigned + ) + assert entry["residual"] - entry["donor_excess"] == target - donor + assert min(entry["residual"], entry["donor_excess"]) == 0.0 + humanitarian[draw.label] = entry + receipt = { + "controls": { + "method": ACS_LOCAL_IMMIGRATION_CONTROLS_METHOD, + "statuses": { + "undocumented_workers": _status_reconciliation(*workers), + "undocumented_students": _status_reconciliation(*students), + }, + "humanitarian": humanitarian, + } + } + with pytest.MonkeyPatch.context() as patch: + _install_controls(patch, _controls(humanitarian=_DRAWS)) + assert acs_local_immigration_receipt_failures(receipt) == [] + + +def _stage_draws(refugee: float, tps: float) -> tuple[HumanitarianDraw, ...]: + return ( + dataclasses.replace(_DRAWS[0], target=refugee), + dataclasses.replace(_DRAWS[1], target=tps), + ) + + +def _run_humanitarian( + draws: tuple[HumanitarianDraw, ...], *, seed: int +) -> tuple[Frame, dict, list[str]]: + with pytest.MonkeyPatch.context() as patch: + _install_controls(patch, _controls(humanitarian=draws)) + result, receipt = with_acs_local_immigration_inputs( + _humanitarian_frame(), seed=seed, time_period=PERIOD + ) + failures = acs_local_immigration_receipt_failures(receipt) + return result, receipt, failures + + +@functools.cache +def _humanitarian_capacity() -> dict[str, float]: + """Each draw's ACS candidate mass: what an exhausting stock assigns.""" + + _, receipt, _ = _run_humanitarian(_stage_draws(100 * U, 100 * U), seed=0) + return { + label: entry["acs_assigned"] + for label, entry in receipt["controls"]["humanitarian"].items() + } + + +def test_an_exhausting_stock_takes_every_added_acs_candidate() -> None: + """The four Venezuelan and two Congolese ACS rows are the only candidates.""" + + assert _humanitarian_capacity() == { + "refugee": pytest.approx(2 * U), + "tps:venezuela": pytest.approx(4 * U), + } + + +@_STAGE +@given(refugee=_STOCK, tps=_STOCK, seed=st.integers(0, 2**16)) +def test_each_humanitarian_draw_assigns_its_donor_residual(refugee, tps, seed) -> None: + draws = _stage_draws(refugee, tps) + result, receipt, failures = _run_humanitarian(draws, seed=seed) + assert failures == [] + capacity = _humanitarian_capacity() + weights = _person_weights(result) + acs = _acs(result) + status = result.table("person")["immigration_status_str"].to_numpy() + for draw in draws: + entry = receipt["controls"]["humanitarian"][draw.label] + assert entry["target"] == draw.target + assert entry["donor_delivered"] == pytest.approx(2 * U) + residual = max(0.0, draw.target - entry["donor_delivered"]) + assert entry["residual"] == residual + assigned = entry["acs_assigned"] + assert entry["pooled_total"] == entry["donor_delivered"] + assigned + emitted = float(weights[acs & (status == draw.status)].sum()) + assert assigned == pytest.approx(emitted) + if residual == 0.0: + assert assigned == 0.0 + elif residual <= capacity[draw.label]: + assert residual <= assigned < residual + U + else: + assert assigned == pytest.approx(capacity[draw.label]) + + +@_STAGE +@given(low=_STOCK, high=_STOCK, seed=st.integers(0, 2**16)) +def test_a_larger_stock_selects_a_superset_of_acs_rows(low, high, seed) -> None: + low, high = sorted((low, high)) + small, _, _ = _run_humanitarian(_stage_draws(low, low), seed=seed) + large, _, _ = _run_humanitarian(_stage_draws(high, high), seed=seed) + acs = _acs(small) + for draw in _DRAWS: + before = small.table("person")["immigration_status_str"].eq(draw.status) + after = large.table("person")["immigration_status_str"].eq(draw.status) + assert not (acs & before.to_numpy() & ~after.to_numpy()).any() diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_release_tool.py b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_release_tool.py index 90468a5c40..2ecda1b0af 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_release_tool.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_release_tool.py @@ -1301,7 +1301,13 @@ def test_reviewed_null_fill_refuses_to_default_fill_immigration(tmp_path) -> Non def _residual_controls(**worker_overrides) -> dict: - """A donor-residual controls block (microcosm#1052 review).""" + """A donor-residual controls block (microcosm#1052 review, microcosm#767). + + The humanitarian block carries one entry per packaged stock draw, each + targeted at its donor residual. + """ + + from microcosm.build.us_runtime.immigration import us_immigration_controls def status(control: float, donor: float, assigned: float) -> dict: return { @@ -1312,15 +1318,21 @@ def status(control: float, donor: float, assigned: float) -> dict: "pooled_total": donor + assigned, } + humanitarian = {} + for draw in us_immigration_controls().humanitarian: + entry = status(draw.target, 0.4 * draw.target, 0.6 * draw.target) + entry["target"] = entry.pop("control") + humanitarian[draw.label] = entry return { - "method": "national_control_minus_donor_pooled_delivered", + "method": "national_control_minus_donor_pooled_delivered_v2", "statuses": { "undocumented_workers": { - **status(8.3e6, 3.0e6, 5.3e6), + **status(9.7e6, 3.0e6, 6.7e6), **worker_overrides, }, "undocumented_students": status(4.08e5, 1.0e5, 3.08e5), }, + "humanitarian": humanitarian, } @@ -1368,7 +1380,28 @@ def _staging_immigration_summary(**overrides) -> dict: acs_local_immigration={ "issue": "microcosm#1020", "assigned_sha256": "b" * 64, - "controls": _residual_controls(residual=8.3e6 * 2 / 3), + "controls": _residual_controls(residual=9.7e6 * 2 / 3), + } + ), + _staging_immigration_summary( + acs_local_immigration={ + "issue": "microcosm#1020", + "assigned_sha256": "b" * 64, + "controls": { + **_residual_controls(), + "method": "national_control_minus_donor_pooled_delivered", + }, + } + ), + _staging_immigration_summary( + acs_local_immigration={ + "issue": "microcosm#1020", + "assigned_sha256": "b" * 64, + "controls": { + key: value + for key, value in _residual_controls().items() + if key != "humanitarian" + }, } ), ], @@ -1382,6 +1415,8 @@ def _staging_immigration_summary(**overrides) -> dict: "no-digest", "pre-1052-review-share-scaled", "targets-not-the-donor-residual", + "pre-767-no-humanitarian-residuals", + "no-humanitarian-block", ], ) def test_immigration_consumers_refuse_a_staging_run_without_the_stage(summary): @@ -3479,18 +3514,19 @@ def _stub_local_immigration_gate(module, monkeypatch, *, passed=True) -> None: from microcosm.build.gates import GateResult - monkeypatch.setattr( - module, - "acs_local_immigration_signal_gate", - lambda frame: GateResult( + def gate(frame, *, time_period): + # The release grades the cohort evidence at its own period. + assert time_period == module.PERIOD + return GateResult( name="acs_local_immigration_signal", passed=passed, failures=() if passed else ("acs_2024_1yr: ssn_card_type is constant 'CITIZEN'",), details={}, - ), - ) + ) + + monkeypatch.setattr(module, "acs_local_immigration_signal_gate", gate) def _stub_local_work_disability_gate( diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_acs_multispine_legacy_builder.py b/packages/microcosm-build/tests/engine_free/us/test_us_acs_multispine_legacy_builder.py index df013fcca2..643662df46 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_acs_multispine_legacy_builder.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_acs_multispine_legacy_builder.py @@ -370,8 +370,10 @@ def fake_immigration(frame, *, seed, time_period): order.append("immigration") return labelled, immigration_receipt - def fake_immigration_gate(frame): + def fake_immigration_gate(frame, *, time_period): assert frame is labelled + # The gate re-derives the cohort evidence at the stage's own period. + assert time_period == captured["immigration"][1] return GateResult( name="acs_local_immigration_signal", passed=True, @@ -775,7 +777,7 @@ def refusing_stage(frame, *, seed, time_period): monkeypatch.setattr( builder, "acs_local_immigration_signal_gate", - lambda frame: GateResult( + lambda frame, *, time_period: GateResult( name="acs_local_immigration_signal", passed=False, failures=("acs_2024_1yr: ssn_card_type is constant 'CITIZEN'",), @@ -789,7 +791,7 @@ def refusing_stage(frame, *, seed, time_period): monkeypatch.setattr( builder, "acs_local_immigration_signal_gate", - lambda frame: GateResult( + lambda frame, *, time_period: GateResult( name="acs_local_immigration_signal", passed=True, failures=() ), ) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_acs_pums.py b/packages/microcosm-build/tests/engine_free/us/test_us_acs_pums.py index 5c7072fe53..d315dd0fb5 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_acs_pums.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_acs_pums.py @@ -8,6 +8,7 @@ import pytest from microcosm.build.serialization_dtypes import CANONICAL_STRING_DTYPE +from microcosm.build.source_runtime import SourceRuntimeError from microcosm.build.us_runtime.acs_inputs import map_acs_native_inputs from microcosm.build.us_runtime.acs_pums import ( ACS_2024_1YR_SPINE, @@ -15,6 +16,10 @@ build_acs_pums_unit_frame, load_acs_pums_tables, ) +from microcosm.build.us_runtime.immigration import ( + _source_aware_immigration_profile, + us_immigration_evidence_features, +) from microcosm.build.us_runtime.operator_boundary import ( assert_operator_free_source_frame, ) @@ -69,6 +74,9 @@ def _person( "SSIP": 0, "RETP": 0, "INTP": 0, + "CIT": 1, + "POBP": 6, + "YOEP": None, "PWGTP": 10, } row.update(overrides) @@ -83,7 +91,16 @@ def _person( return row -def _source(tmp_path: Path) -> AcsPumsSource: +_ACS_IMMIGRATION_EVIDENCE = ("CIT", "POBP", "YOEP") + + +def _source(tmp_path: Path, *, immigration_evidence: bool = True) -> AcsPumsSource: + """The shared fixture; ``immigration_evidence=False`` drops CIT/POBP/YOEP. + + The loader keeps those three optional (microcosm#1020): an absent column + stays absent, and the immigration consumers refuse rows without them. + """ + household_zip = tmp_path / "csv_hus.zip" person_zip = tmp_path / "csv_pus.zip" _write_csv_zip( @@ -102,39 +119,54 @@ def _source(tmp_path: Path) -> AcsPumsSource: "psam_husb.csv": [_household("2024HU0000001", NP=3)], }, ) - _write_csv_zip( - person_zip, - { - "psam_pusa.csv": [ - _person("2024HU0000002", 1, 20, MAR=4, WAGP=30_000), - _person( - "2024HU0000001", - 1, - 20, - AGEP=42, - WAGP=60_000, - ), - ], - "psam_pusb.csv": [ - _person( - "2024HU0000001", - 2, - 21, - AGEP=40, - SEX=2, - WAGP=45_000, - ), - _person( - "2024HU0000001", - 3, - 25, - AGEP=10, - MAR=5, - WAGP=None, - ), - ], - }, - ) + members = { + "psam_pusa.csv": [ + _person( + "2024HU0000002", + 1, + 20, + MAR=4, + WAGP=30_000, + CIT=5, + POBP=164, + YOEP=2021, + ), + _person( + "2024HU0000001", + 1, + 20, + AGEP=42, + WAGP=60_000, + ), + ], + "psam_pusb.csv": [ + _person( + "2024HU0000001", + 2, + 21, + AGEP=40, + SEX=2, + WAGP=45_000, + ), + _person( + "2024HU0000001", + 3, + 25, + AGEP=10, + MAR=5, + WAGP=None, + ), + ], + } + if not immigration_evidence: + members = { + name: [ + {k: v for k, v in row.items() if k not in _ACS_IMMIGRATION_EVIDENCE} + for row in rows + ] + for name, rows in members.items() + } + _write_csv_zip(person_zip, members) return AcsPumsSource(household_zip=household_zip, person_zip=person_zip) @@ -229,7 +261,7 @@ def test_acs_loader_keeps_immigration_sources_and_renames_military_service( def test_acs_loader_leaves_absent_immigration_sources_absent(tmp_path: Path) -> None: - tables, _ = load_acs_pums_tables(_source(tmp_path)) + tables, _ = load_acs_pums_tables(_source(tmp_path, immigration_evidence=False)) # ESR is also a required work-experience column, so it is always loaded. optional = [column for column in _IMMIGRATION_SOURCES if column != "ESR"] assert "ESR" in tables["person"] @@ -237,6 +269,61 @@ def test_acs_loader_leaves_absent_immigration_sources_absent(tmp_path: Path) -> assert column not in tables["person"] +def test_acs_rows_without_immigration_evidence_cannot_feed_the_kernel( + tmp_path: Path, +) -> None: + """The loader keeps CIT/POBP/YOEP optional (microcosm#1020), so the + refusal sits with the consumers: the kernel's evidence profile, which the + ACS transfer reads (microcosm#767), refuses a row with no citizenship + source instead of reading the blank as a code.""" + + def person_view(source: AcsPumsSource) -> pd.DataFrame: + # The unit frame derives the kernel's A_AGE from AGEP + # (_with_structural_columns); the loader's raw table carries only AGEP. + person = load_acs_pums_tables(source)[0]["person"] + return person.assign(A_AGE=person["AGEP"]) + + without = person_view(_source(tmp_path, immigration_evidence=False)) + with pytest.raises(SourceRuntimeError, match="exactly one row-level citizenship"): + _source_aware_immigration_profile(without, time_period=2024) + # The same archive with the three columns feeds it: one row-level source + # per person, read from CIT (5 = not a citizen). + person = person_view(_source(tmp_path)) + profile = _source_aware_immigration_profile(person, time_period=2024) + keys = list( + zip( + person["SERIALNO"].astype(str), + person["SPORDER"].astype(int), + strict=True, + ) + ) + assert dict(zip(keys, profile.is_citizen.tolist(), strict=True)) == { + ("2024HU0000001", 1): True, + ("2024HU0000001", 2): True, + ("2024HU0000001", 3): True, + ("2024HU0000002", 1): False, + } + + +def test_acs_frame_without_immigration_evidence_refuses_transfer_predictors( + tmp_path: Path, +) -> None: + """The ACS transfer reads its immigration predictors through + us_immigration_evidence_features on the built unit frame (microcosm#767); + a frame from an archive without CIT/POBP/YOEP refuses there.""" + + pytest.importorskip("microunit") # sanctioned tax-unit constructor (us extra) + frame, _metadata = build_acs_pums_unit_frame( + _source(tmp_path, immigration_evidence=False) + ) + with pytest.raises(SourceRuntimeError, match="exactly one row-level citizenship"): + us_immigration_evidence_features(frame, time_period=2024) + frame, _metadata = build_acs_pums_unit_frame(_source(tmp_path)) + features = us_immigration_evidence_features(frame, time_period=2024) + assert np.isfinite(features.to_numpy()).all() + assert sorted(features["is_us_citizen"].tolist()) == [0.0, 1.0, 1.0, 1.0] + + _WORK_DISABILITY_SOURCES = { "WKWN": 26, "DDRS": 2, @@ -589,6 +676,14 @@ def test_load_acs_pums_tables_streams_all_csv_members_and_keeps_native_blanks( "2024HU0000002", ] assert pd.isna(tables["person"].loc[2, "WAGP"]) + foreign_born = ( + tables["person"].loc[tables["person"]["SERIALNO"].eq("2024HU0000002")].iloc[0] + ) + assert (foreign_born["CIT"], foreign_born["POBP"], foreign_born["YOEP"]) == ( + 5, + 164, + 2021, + ) assert metadata["vacant_household_rows_dropped"] == 1 assert metadata["household_csv_members"] == ["psam_husa.csv", "psam_husb.csv"] assert metadata["person_csv_members"] == ["psam_pusa.csv", "psam_pusb.csv"] @@ -622,6 +717,9 @@ def test_build_acs_pums_unit_frame_preserves_lineage_geography_and_weights( assert person["source_household_id"].dtype == np.dtype(np.int64) assert pd.api.types.is_string_dtype(person["source_person_id"].dtype) assert person["source_row_id"].dtype == np.dtype(np.int64) + assert person.loc[person["source_household_id"].eq(2), "CIT"].item() == 5 + assert person.loc[person["source_household_id"].eq(2), "POBP"].item() == 164 + assert person.loc[person["source_household_id"].eq(2), "YOEP"].item() == 2021 assert metadata["weighted_household_population"] == pytest.approx(30.0) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_acs_transfer.py b/packages/microcosm-build/tests/engine_free/us/test_us_acs_transfer.py index 9deedd116d..f1588835f2 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_acs_transfer.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_acs_transfer.py @@ -11,6 +11,7 @@ import microcosm.build.us_runtime.acs_transfer as acs_transfer_module import microcosm.build.us_runtime.acs_transfer_bank as acs_transfer_bank_module +import microcosm.build.us_runtime.immigration as immigration_module from microcosm.build.frame_checkpoint import write_frame_checkpoint from microcosm.build.serialization_dtypes import CANONICAL_STRING_DTYPE from microcosm.build.us_runtime.acs_transfer import ( @@ -22,6 +23,7 @@ ACS_PERSON_TRANSFER_PREDICTORS, AcsTransferResult, acs_adult_care_qualifying_rows, + acs_transfer_donor_requirements, declared_acs_transfer_target_families, default_acs_transfer_target_families, transfer_acs_inputs, @@ -29,6 +31,11 @@ from microcosm.build.us_runtime.acs_transfer_bank import ( AcsTransferTargetBankStore, ) +from microcosm.build.us_runtime.immigration import ( + HumanitarianDraw, + ImmigrationControls, + UndocumentedControls, +) from microcosm.build.us_runtime.puf_support import clone_us_frame_for_puf_support from microcosm.build.us_runtime.spine_assembly import assemble_spines from microcosm.fit import Regime @@ -84,6 +91,9 @@ def _donor_frame() -> Frame: ), "age": [45.0, 43.0, 30.0, 8.0, 68.0, 66.0, 39.0, 17.0], "is_female": [False, True, True, False, False, True, True, False], + "PRCITSHP": [1, 1, 1, 1, 1, 1, 5, 5], + "PENATVTY": [57, 57, 57, 57, 57, 57, 164, 373], + "PEINUSYR": [0, 0, 0, 0, 0, 0, 28, 24], "is_household_head": [True, False, True, False, True, False, True, False], "employment_income_before_lsr": [ 80_000.0, @@ -208,6 +218,9 @@ def _recipient_frame() -> Frame: ), "age": [41.0, 40.0, 27.0, 72.0, 69.0, 15.0], "is_female": [False, True, True, False, True, False], + "CIT": [1, 1, 5, 1, 1, 5], + "POBP": [6, 6, 164, 36, 36, 373], + "YOEP": [np.nan, np.nan, 2023, np.nan, np.nan, 2015], "is_household_head": [True, False, True, True, False, False], "employment_income_before_lsr": [ 65_000.0, @@ -263,6 +276,90 @@ def _recipient_frame() -> Frame: ) +def _mixed_immigration_recipient() -> Frame: + """One immutable ASEC row followed by ACS rows with paired null targets.""" + + base = _recipient_frame() + tables = {entity: base.table(entity).copy() for entity in base.entities} + person = tables["person"] + for column in ("PRCITSHP", "PENATVTY", "PEINUSYR"): + person[column] = np.nan + immutable_index = person.index[0] + person.loc[immutable_index, ["CIT", "POBP", "YOEP"]] = np.nan + person.loc[immutable_index, ["PRCITSHP", "PENATVTY", "PEINUSYR"]] = [ + 5, + 200, + 27, + ] + person["ssn_card_type"] = pd.Series( + ["OTHER_NON_CITIZEN", *([np.nan] * (len(person) - 1))], + index=person.index, + dtype=object, + ) + person["immigration_status_str"] = pd.Series( + ["PAROLED_ONE_YEAR", *([np.nan] * (len(person) - 1))], + index=person.index, + dtype=object, + ) + return Frame( + tables, + base.schema, + {entity: base.weights_for(entity) for entity in base.weighted_entities}, + base.strata, + mass_log=base.mass_log, + ) + + +def _small_immigration_controls( + *, + ukraine_target: float = 60.0, +) -> ImmigrationControls: + source = "https://example.com/immigration-control" + return ImmigrationControls( + undocumented=UndocumentedControls( + workers=1.0, + students=1.0, + population_anchor=1.0, + sources={ + "undocumented_workers": source, + "undocumented_students": source, + "undocumented_population_anchor": source, + }, + ), + humanitarian=( + HumanitarianDraw( + category="paroled_one_year", + origin="afghanistan", + status="PAROLED_ONE_YEAR", + target=50.0, + source=source, + ), + HumanitarianDraw( + category="paroled_one_year", + origin="ukraine", + status="PAROLED_ONE_YEAR", + target=ukraine_target, + source=source, + ), + HumanitarianDraw( + category="tps", + origin="venezuela", + status="TPS", + target=70.0, + source=source, + ), + ), + ) + + +def _no_humanitarian_controls() -> ImmigrationControls: + controls = _small_immigration_controls() + return ImmigrationControls( + undocumented=controls.undocumented, + humanitarian=(), + ) + + def _replace_column( frame: Frame, entity: str, @@ -677,7 +774,6 @@ def test_explicit_declared_plan_preserves_deferred_geography( lambda: plan, ) monkeypatch.setattr(acs_transfer_module, "QRF", _MeanQRF) - result = transfer_acs_inputs( _recipient_frame(), _donor_frame(), @@ -745,6 +841,59 @@ def test_declared_families_are_independent_of_release_coverage_surface() -> None assert "receives_wic" in production_declared["person"]["model_required_boolean"] +def test_donor_requirements_include_immigration_source_evidence_triplet() -> None: + donor = _with_columns( + _donor_frame(), + "person", + { + "ssn_card_type": ["CITIZEN"] * 6 + ["NONE", "NONE"], + "immigration_status_str": ["CITIZEN"] * 6 + + ["UNDOCUMENTED", "UNDOCUMENTED"], + }, + ) + plan = { + "person": { + "source_operator_immigration": ( + "ssn_card_type", + "immigration_status_str", + ) + } + } + + requirements = acs_transfer_donor_requirements(donor, plan) + + assert {"PRCITSHP", "PENATVTY", "PEINUSYR"}.issubset(requirements["person"]) + + acs_tables = {entity: donor.table(entity).copy() for entity in donor.entities} + acs_person = acs_tables["person"].rename( + columns={"PRCITSHP": "CIT", "PENATVTY": "POBP"} + ) + acs_person = acs_person.drop(columns=["PEINUSYR"]) + acs_person["YOEP"] = [np.nan] * 6 + [2023, 2015] + acs_tables["person"] = acs_person + acs_donor = Frame( + acs_tables, + donor.schema, + {entity: donor.weights_for(entity) for entity in donor.weighted_entities}, + donor.strata, + mass_log=donor.mass_log, + ) + acs_requirements = acs_transfer_donor_requirements(acs_donor, plan) + assert {"CIT", "POBP", "YOEP"}.issubset(acs_requirements["person"]) + + tables = {entity: donor.table(entity).copy() for entity in donor.entities} + tables["person"] = tables["person"].drop(columns=["PEINUSYR"]) + incomplete = Frame( + tables, + donor.schema, + {entity: donor.weights_for(entity) for entity in donor.weighted_entities}, + donor.strata, + mass_log=donor.mass_log, + ) + with pytest.raises(ValueError, match="incomplete donor evidence triplet"): + acs_transfer_donor_requirements(incomplete, plan) + + def test_declared_plan_carries_the_23_stage_base_surface() -> None: """The M/N/O donor repairs must reach the ACS spine through the plan.""" @@ -876,6 +1025,301 @@ def test_explicit_transfer_adds_requested_model_inputs( and "immigration_status_str" not in call["targets"] for call in joint_calls ) + assert { + "__acs_transfer_is_us_citizen", + "__acs_transfer_birth_country_code", + "__acs_transfer_arrival_year", + }.issubset(joint_calls[0]["predictors"]) + + +def test_joint_immigration_codec_never_emits_constrained_baseline( + monkeypatch: pytest.MonkeyPatch, +) -> None: + constrained = [ + "PAROLED_ONE_YEAR", + "REFUGEE", + "ASYLEE", + "DEPORTATION_WITHHELD", + "TPS", + "CUBAN_HAITIAN_ENTRANT", + "DACA", + "REFUGEE", + ] + donor = _with_columns( + _donor_frame(), + "person", + { + "ssn_card_type": ["OTHER_NON_CITIZEN"] * 8, + "immigration_status_str": constrained, + }, + ) + monkeypatch.setattr(acs_transfer_module, "QRF", _MeanQRF) + monkeypatch.setattr( + immigration_module, + "us_immigration_controls", + _no_humanitarian_controls, + ) + + result = transfer_acs_inputs( + _recipient_frame(), + donor, + target_families={ + "person": { + "source_operator_immigration": ( + "ssn_card_type", + "immigration_status_str", + ) + } + }, + donor_channel=None, + n_estimators=1, + ) + + assert set(result.frame.person["immigration_status_str"]) <= { + "CITIZEN", + "LEGAL_PERMANENT_RESIDENT", + } + encoding = acs_transfer_module._target_encodings( + donor.person, + targets=("ssn_card_type", "immigration_status_str"), + )["immigration_status_str"] + assert not {pair[1] for pair in encoding.categories}.intersection(constrained) + + +def test_transfer_rederives_entrant_and_daca_only_on_compatible_acs_cohorts( + monkeypatch: pytest.MonkeyPatch, +) -> None: + donor = _with_columns( + _donor_frame(), + "person", + { + "ssn_card_type": ["NON_CITIZEN_VALID_EAD"] * 8, + "immigration_status_str": [ + "CUBAN_HAITIAN_ENTRANT", + "DACA", + ] + * 4, + }, + ) + recipient = _with_columns( + _recipient_frame(), + "person", + { + "age": [50.0, 50.0, 29.0, 50.0, 40.0, 70.0], + "CIT": [5, 5, 5, 5, 1, 5], + "POBP": [327, 164, 312, 312, 6, 332], + "YOEP": [2000, 2020, 2005, 2020, np.nan, 1970], + }, + ) + monkeypatch.setattr(acs_transfer_module, "QRF", _MeanQRF) + monkeypatch.setattr( + immigration_module, + "us_immigration_controls", + _no_humanitarian_controls, + ) + + result = transfer_acs_inputs( + recipient, + donor, + target_families={ + "person": { + "source_operator_immigration": ( + "ssn_card_type", + "immigration_status_str", + ) + } + }, + donor_channel=None, + n_estimators=1, + ) + + assert result.frame.person["immigration_status_str"].tolist() == [ + "CUBAN_HAITIAN_ENTRANT", + "LEGAL_PERMANENT_RESIDENT", + "DACA", + "LEGAL_PERMANENT_RESIDENT", + "CITIZEN", + "LEGAL_PERMANENT_RESIDENT", + ] + receipt = next( + item.reconciliation + for item in result.imputed_inputs + if item.column == "immigration_status_str" + ) + assert receipt is not None + assert receipt["special_status_assignments"] == 2 + + +def test_humanitarian_reconciliation_uses_residual_targets_and_records_receipt( + monkeypatch: pytest.MonkeyPatch, +) -> None: + donor = _with_columns( + _donor_frame(), + "person", + { + "ssn_card_type": ["OTHER_NON_CITIZEN"] * 8, + "immigration_status_str": ["LEGAL_PERMANENT_RESIDENT"] * 8, + }, + ) + monkeypatch.setattr(acs_transfer_module, "QRF", _MeanQRF) + monkeypatch.setattr( + immigration_module, + "us_immigration_controls", + _small_immigration_controls, + ) + + result = transfer_acs_inputs( + _mixed_immigration_recipient(), + donor, + target_families={ + "person": { + "source_operator_immigration": ( + "ssn_card_type", + "immigration_status_str", + ) + } + }, + donor_channel=None, + seed=19, + n_estimators=1, + ) + + person = result.frame.person + assert person["immigration_status_str"].tolist() == [ + "PAROLED_ONE_YEAR", # immutable Afghanistan contribution + "CITIZEN", # source CIT repairs the imputed baseline pair + "PAROLED_ONE_YEAR", # compatible Ukrainian residual + "CITIZEN", + "CITIZEN", + "TPS", # compatible Venezuelan residual + ] + assert person.loc[person.index[0], "ssn_card_type"] == "OTHER_NON_CITIZEN" + reconciled = [ + item + for item in result.imputed_inputs + if item.column in {"ssn_card_type", "immigration_status_str"} + ] + assert len(reconciled) == 2 + assert reconciled[0].reconciliation == reconciled[1].reconciliation + receipt = reconciled[0].reconciliation + assert receipt is not None + assert receipt["mutable_rows"] == 5 + assert receipt["immutable_rows"] == 1 + draws = receipt["draws"] + assert draws["paroled_one_year:afghanistan"] == { + "target": 50.0, + "immutable_population": 50.0, + "residual_target": 0.0, + "eligible_recipient_population": 0.0, + "selected_recipient_population": 0.0, + "achieved_population": 50.0, + "absolute_error": 0.0, + "selection_threshold_tie_population": 0.0, + "residual_selection_error": 0.0, + "within_residual_discrete_weight_bound": True, + "immutable_overshoot": 0.0, + "within_discrete_weight_bound": True, + "candidates_exhausted": False, + "candidate_shortfall_population": 0.0, + } + assert draws["paroled_one_year:ukraine"]["residual_target"] == 60.0 + assert draws["paroled_one_year:ukraine"]["achieved_population"] == 60.0 + assert draws["tps:venezuela"]["achieved_population"] == 70.0 + assert all(draw["within_discrete_weight_bound"] for draw in draws.values()) + + +def _transfer_mixed_immigration(ukraine_target: float, monkeypatch) -> object: + donor = _with_columns( + _donor_frame(), + "person", + { + "ssn_card_type": ["OTHER_NON_CITIZEN"] * 8, + "immigration_status_str": ["LEGAL_PERMANENT_RESIDENT"] * 8, + }, + ) + monkeypatch.setattr(acs_transfer_module, "QRF", _MeanQRF) + monkeypatch.setattr( + immigration_module, + "us_immigration_controls", + lambda: _small_immigration_controls(ukraine_target=ukraine_target), + ) + return transfer_acs_inputs( + _mixed_immigration_recipient(), + donor, + target_families={ + "person": { + "source_operator_immigration": ( + "ssn_card_type", + "immigration_status_str", + ) + } + }, + donor_channel=None, + seed=19, + n_estimators=1, + ) + + +def test_humanitarian_reconciliation_takes_every_candidate_when_they_fall_short( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # The Ukrainian candidates hold 60 weighted persons against a residual of + # 61, as a stacked pool's ACS arm can hold fewer recent arrivals than a + # draw's residual: the draw takes them all and records the gap. + result = _transfer_mixed_immigration(61.0, monkeypatch) + + person = result.frame.person + assert person["immigration_status_str"].tolist() == [ + "PAROLED_ONE_YEAR", + "CITIZEN", + "PAROLED_ONE_YEAR", + "CITIZEN", + "CITIZEN", + "TPS", + ] + receipt = next( + item.reconciliation + for item in result.imputed_inputs + if item.column == "immigration_status_str" + ) + assert receipt is not None + ukraine = receipt["draws"]["paroled_one_year:ukraine"] + assert ukraine == { + "target": 61.0, + "immutable_population": 0.0, + "residual_target": 61.0, + "eligible_recipient_population": 60.0, + "selected_recipient_population": 60.0, + "achieved_population": 60.0, + "absolute_error": 1.0, + "selection_threshold_tie_population": 0.0, + "residual_selection_error": 1.0, + "within_residual_discrete_weight_bound": False, + "immutable_overshoot": 0.0, + "within_discrete_weight_bound": False, + "candidates_exhausted": True, + "candidate_shortfall_population": 1.0, + } + others = [ + draw + for label, draw in receipt["draws"].items() + if label != "paroled_one_year:ukraine" + ] + assert all(not draw["candidates_exhausted"] for draw in others) + assert all(draw["within_discrete_weight_bound"] for draw in others) + + +def test_humanitarian_reconciliation_refuses_a_shortfall_below_the_band_floor( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # 60 candidates reach 0.30 of a 200 target, under the composition gate's + # floor of 0.35. + with pytest.raises( + ValueError, + match=r"candidate shortfall for 'paroled_one_year:ukraine' leaves it " + r"below 0\.35 of its target", + ): + _transfer_mixed_immigration(200.0, monkeypatch) def test_large_target_family_is_split_to_bound_retained_qrf_forests( @@ -1163,6 +1607,11 @@ def test_target_bank_resumes_joint_immigration_codec_as_one_model_target( tmp_path: Path, ) -> None: _lock_bank_fixture_threads(monkeypatch) + monkeypatch.setattr( + immigration_module, + "us_immigration_controls", + _no_humanitarian_controls, + ) donor = _with_columns( _donor_frame(), "person", @@ -2991,17 +3440,17 @@ def test_adult_care_qualifying_rows_fails_closed_on_missing_structure( # caller opts in; the ACS local income pass is the only caller that does. # --------------------------------------------------------------------------- -#: The execution contract's SHA-256 before person predictor extensions -#: existed. Every caller that passes none (the shared plan, the pool lane, -#: banked checkpoints and the spec projection) must keep these. +#: The execution contract's SHA-256 with no person predictor extensions +#: (schema 3, #767). Every caller that passes none (the shared plan, the +#: pool lane, banked checkpoints and the spec projection) must keep these. _DEFAULT_CONTRACT_SHA256 = ( - "f6294b47f18f223e757baafa545075c4b9fcea7d168fc59ac38fc7e9352cbb80" + "30eb194ad1ce1f2f76ed06c299195e6e2d556fca3ec1fe95b8b6cd70f752fe6f" ) _DECLARED_PLAN_CONTRACT_SHA256 = ( - "e90a808a1218681a7084d16807f926d5522717556f2b97708e416082a876e0ca" + "cce290dc5454df6818aa25eb087642049b4574446e1c9a61a8aef74aa8df1d69" ) _PREGNANCY_CONTRACT_SHA256 = ( - "cda751e047f1772b20fb019fab8dc72ea77730e13f76bb671744c51c9dc27fa5" + "e5b4d7c3187e640e8a30d2f938e4d0e7fc179664b801b1f618d8e882bb052e0c" ) _EXTENSION_FEATURE = "__acs_transfer_fixture_interest_total" _EXTENSION_FAMILIES = { diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_asec_checkpoint.py b/packages/microcosm-build/tests/engine_free/us/test_us_asec_checkpoint.py index c011233609..68e64f802c 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_asec_checkpoint.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_asec_checkpoint.py @@ -135,7 +135,7 @@ def _raw_binding(frame: Frame) -> dict[str, object]: "operation": "exact_source_join", "source_pins": [pin], } - for column in ("ED_VAL", "LKWEEKS", "PAW_TYP") + for column in ("A_LFSR", "ED_VAL", "LKWEEKS", "PAW_TYP") }, "schema_version": ASEC_RAW_STAGE_SCHEMA_VERSION, "source_construction_identity": frame_identity(frame).to_payload(), @@ -167,6 +167,7 @@ def _raw_us_frame(*, id_offset: int = 0) -> Frame: tables["person"]["ED_VAL"] = [0.0, 500.0] tables["person"]["LKWEEKS"] = [-1, 12] tables["person"]["PAW_TYP"] = np.asarray([0, 1], dtype=np.int64) + tables["person"]["A_LFSR"] = np.asarray([1, 7], dtype=np.int64) return Frame( tables, source.schema, @@ -249,7 +250,7 @@ def test_loads_operator_untouched_raw_stage_checkpoint(tmp_path: Path) -> None: @pytest.mark.parametrize( "column", - ("ED_VAL", "LKWEEKS", "PAW_TYP", "PERIDNUM", "source_year"), + ("A_LFSR", "ED_VAL", "LKWEEKS", "PAW_TYP", "PERIDNUM", "source_year"), ) def test_raw_loader_rejects_missing_input_complete_source_column( tmp_path: Path, @@ -279,6 +280,7 @@ def test_raw_loader_rejects_missing_input_complete_source_column( ("ED_VAL", [0.0, np.nan], "ED_VAL must be complete"), ("LKWEEKS", [-1, 53], "LKWEEKS must be complete"), ("PAW_TYP", [0, 4], "PAW_TYP must be complete integers"), + ("A_LFSR", [1, 6], "A_LFSR must be complete integers"), ), ) def test_raw_loader_rejects_invalid_input_complete_source_values( diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py index 9d9cdaab60..eb1206df7b 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py @@ -25,7 +25,7 @@ def test_source_surface_classification_is_complete() -> None: } assert normative["stage_asset"] == { "id": "source_stages", - "sha256": "959214c437bfa59d43e32279b82f29a0987649cd7579ff87567c70f084eae66c", + "sha256": "67587983a455bfffde688b024e7d4bd03a9d1d1f3b79291881558e60bb20311a", } assert operational["stage_asset"] == { "path": "microcosm.build.us/source_stages.json" diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_education_assistance_source.py b/packages/microcosm-build/tests/engine_free/us/test_us_education_assistance_source.py index e1e4cd0821..09d564f9dd 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_education_assistance_source.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_education_assistance_source.py @@ -1,4 +1,4 @@ -"""The pinned ASEC education-assistance sidecar: load, verify, and fill. +"""The pinned ASEC ED_VAL/A_LFSR sidecar: load, verify, and fill. The frozen census_cps inputs never carried raw ASEC ``ED_VAL`` (microcosm#417's sibling gap), so the education-inputs stage restores it from @@ -21,8 +21,11 @@ from microcosm.build.us_runtime.education_assistance_source import ( ASEC_EDUCATION_ASSISTANCE_ARCHIVES, ASEC_EDUCATION_ASSISTANCE_INCOME_YEARS, + ASEC_LABOR_FORCE_STATUS_COLUMN, + ASEC_LABOR_FORCE_STATUS_VALID_CODES, AsecEducationArchive, fill_asec_education_assistance_source, + fill_asec_labor_force_status_source, load_asec_education_assistance_sources, ) @@ -41,6 +44,7 @@ def _source_frame(rows: int = 6) -> pd.DataFrame: "A_LINENO": np.ones(rows, dtype=np.int64), "PERIDNUM": [_peridnum(index) for index in range(rows)], "ED_VAL": [0.0, 0.0, 2_500.0, 0.0, 12_000.0, 0.0][:rows], + "A_LFSR": [0, 1, 2, 3, 4, 7][:rows], "A_FNLWGT": np.full(rows, 100.0), } ) @@ -112,9 +116,29 @@ def test_loader_reads_pinned_zip_and_audits(pinned_archive) -> None: ) assert list(source["source_year"].unique()) == [_YEAR] assert len(source) == len(frame) + assert source[ASEC_LABOR_FORCE_STATUS_COLUMN].tolist() == [0, 1, 2, 3, 4, 7] + assert source[ASEC_LABOR_FORCE_STATUS_COLUMN].dtype == np.dtype("int64") assert source.attrs["source_audit"][_YEAR]["positive_rows"] == 2 +def test_loader_rejects_invalid_labor_force_status( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + frame = _source_frame() + frame.loc[0, ASEC_LABOR_FORCE_STATUS_COLUMN] = 6 + member = f"pppub{str(_YEAR + 1)[2:]}.csv" + path = tmp_path / "invalid-a-lfsr.zip" + _write_archive(path, frame, member) + monkeypatch.setitem( + ASEC_EDUCATION_ASSISTANCE_ARCHIVES, + _YEAR, + _pins_for(path, frame, member), + ) + + with pytest.raises(ValueError, match="A_LFSR must be a complete integer"): + load_asec_education_assistance_sources({_YEAR: path}, income_years=(_YEAR,)) + + def test_loader_rejects_wrong_zip_bytes(pinned_archive, tmp_path) -> None: path, frame = pinned_archive tampered = tmp_path / "tampered.zip" @@ -155,6 +179,21 @@ def test_fill_joins_ed_val_by_identity(pinned_archive) -> None: assert row["ED_VAL"] == expected[row["PERIDNUM"]] +def test_fill_joins_labor_force_status_by_identity(pinned_archive) -> None: + path, frame = pinned_archive + source = load_asec_education_assistance_sources( + {_YEAR: path}, income_years=(_YEAR,) + ) + filled = fill_asec_labor_force_status_source(_person_frame(), source) + expected = frame.set_index("PERIDNUM")[ASEC_LABOR_FORCE_STATUS_COLUMN] + for _, row in filled.iterrows(): + assert row[ASEC_LABOR_FORCE_STATUS_COLUMN] == expected[row["PERIDNUM"]] + assert filled[ASEC_LABOR_FORCE_STATUS_COLUMN].dtype == np.dtype("int64") + assert set(filled[ASEC_LABOR_FORCE_STATUS_COLUMN]) <= ( + ASEC_LABOR_FORCE_STATUS_VALID_CODES + ) + + def test_fill_fails_closed_on_uncovered_key(pinned_archive) -> None: path, _ = pinned_archive source = load_asec_education_assistance_sources( @@ -199,6 +238,19 @@ def test_fill_refuses_to_overwrite_measured_values(pinned_archive) -> None: fill_asec_education_assistance_source(person, source) +def test_labor_force_fill_refuses_to_overwrite_measured_values( + pinned_archive, +) -> None: + path, _ = pinned_archive + source = load_asec_education_assistance_sources( + {_YEAR: path}, income_years=(_YEAR,) + ) + person = _person_frame() + person[ASEC_LABOR_FORCE_STATUS_COLUMN] = 7 + with pytest.raises(ValueError, match="refusing to overwrite"): + fill_asec_labor_force_status_source(person, source) + + def test_pins_reject_audit_drift(pinned_archive, monkeypatch) -> None: path, frame = pinned_archive pins = ASEC_EDUCATION_ASSISTANCE_ARCHIVES[_YEAR] diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py index f630a50fb5..d28500728b 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py @@ -609,7 +609,8 @@ def test__given_target_frame_checkpoint__then_builder_round_trips_frame( staged_frame_sha256="staged-frame-sha", ) # 12 = the identity carries the staged-frame digest (microcosm#956); 13 = - # the batched base pass. Schema 2 distinguishes the values+mask codec from + # the batched base pass; 14 = source-aware humanitarian-stock columns + # (microcosm#767). Schema 2 distinguishes the values+mask codec from # schema-1 checkpoints. assert identity["schema_version"] == 2 assert ( @@ -833,7 +834,8 @@ def test__given_stale_materializer_version_checkpoint__then_builder_rejects_it( """A checkpoint stored under a superseded materializer version must not load. Version 12 adds the staged-frame digest to the identity (microcosm#956) - and 13 the batched base pass. The version constant participates in the + and 13 the batched base pass; 14 adds the source-aware humanitarian-stock + target columns (microcosm#767). The version constant participates in the identity comparison; this pins rejection of the immediately preceding version and the one before it, derived from the current constant so every bump moves the test with it. @@ -872,7 +874,7 @@ def test__given_stale_materializer_version_checkpoint__then_builder_rejects_it( staged_frame_sha256="staged-frame-sha", ) current = builder.TARGET_FRAME_CHECKPOINT_MATERIALIZER_VERSION - assert current >= 13 # microcosm#1018 uses version 12. + assert current >= 14 # #1018 uses 12, #956 13, #767 14. assert identity["materializer_version"] == current # The two preceding versions must both miss against the current version. stale_identity = {**dict(identity), "materializer_version": current - 1} @@ -6886,6 +6888,7 @@ def fake_smoke(*, simulate, period): "crash", "telemetry", "puf_tail", + "immigration_drift", "spm_missing_pool", "qrf_tail_register", "qrf_tail_register_clean", @@ -6926,6 +6929,10 @@ def test_main_writes_diagnostics_before_post_calibration_gate_failure( ``puf_tail``: exact-k selection loses the original PUF capital-gains tail; the failure is batched while diagnostics and final-weight evidence remain, every later terminal group runs, and release artifacts stay suppressed. + ``immigration_drift``: the source-frame immigration composition passes, then + calibrated weights make the same selected status mix fail on the final + export-frame recheck; that final verdict reaches diagnostics and aborts + publication. ``spm_missing_pool``: a prepared pool without the source role fails at the real SPM signal gate before calibration or terminal coverage checks. ``qrf_tail_register``: the per-run tail register carries a stale and an @@ -7064,11 +7071,13 @@ def test_main_writes_diagnostics_before_post_calibration_gate_failure( ) class FakeFrame: - # Household-only, shaped like the real ``Frame`` contract: ``schema`` - # is always present, and ``table`` raises ``ValueError`` for an entity - # the schema does not declare (``Frame.table``). That is what lets the - # pre-calibration SPM composition advisory degrade to a notice here - # instead of aborting the run before the gate under test. + # Household and person only, shaped like the real ``Frame`` contract: + # ``schema`` is always present, and ``table`` raises ``ValueError`` for + # an entity the schema does not declare (``Frame.table``). That is what + # lets the pre-calibration SPM composition advisory degrade to a notice + # here instead of aborting the run before the gate under test; the + # person table carries the immigration statuses the composition gate + # reads. schema = EntitySchema(group_entities=("household",)) def n(self, entity): @@ -7077,16 +7086,42 @@ def n(self, entity): def table(self, entity): if terminal_mode == "spm_missing_pool" and entity == "person": + # A prepared pool carries its immigration statuses; only the + # SPM source role is missing. return pd.DataFrame( - {"person_id": [1, 2, 3, 4], "age": [30, 40, 16, 17]} + { + "person_id": [1, 2, 3, 4], + "age": [30, 40, 16, 17], + "immigration_status_str": [ + "REFUGEE", + "LEGAL_PERMANENT_RESIDENT", + "ASYLEE", + "LEGAL_PERMANENT_RESIDENT", + ], + } ) - if entity != "household": + if entity not in self.schema.entities: raise ValueError( f"Unknown entity {entity!r}; schema declares " f"{list(self.schema.entities)}." ) size = self.n("household") - return pd.DataFrame({"household_id": np.arange(1, size + 1, dtype="int64")}) + if entity == "household": + return pd.DataFrame( + {"household_id": np.arange(1, size + 1, dtype="int64")} + ) + assert entity == "person" + return pd.DataFrame( + { + "household_id": np.arange(1, size + 1, dtype="int64"), + "immigration_status_str": [ + "REFUGEE", + "LEGAL_PERMANENT_RESIDENT", + "ASYLEE", + "LEGAL_PERMANENT_RESIDENT", + ][:size], + } + ) def weights_for(self, entity): assert entity == "household" @@ -7096,8 +7131,8 @@ def weights_for(self, entity): ) class FakeExportFrame: - # Household-only, shaped like the real ``Frame`` contract for the same - # reason ``FakeFrame`` above is: ``schema`` is always present and + # Household and person only, shaped like the real ``Frame`` contract for + # the same reason ``FakeFrame`` above is: ``schema`` is always present and # ``table`` raises ``ValueError`` for an entity the schema does not # carry. That is what lets the batched SPM composition gate contribute # one "cannot be classified" failure line to this run's report — the @@ -7117,12 +7152,25 @@ def weights_for(self, entity): ) def table(self, entity): - if entity != "household": + if entity not in self.schema.entities: raise ValueError( f"Unknown entity {entity!r}; schema declares " f"{list(self.schema.entities)}." ) - return pd.DataFrame({"household_id": np.asarray([10, 20], dtype="int64")}) + if entity == "household": + return pd.DataFrame( + {"household_id": np.asarray([10, 20], dtype="int64")} + ) + assert entity == "person" + return pd.DataFrame( + { + "household_id": np.asarray([10, 20], dtype="int64"), + "immigration_status_str": [ + "REFUGEE", + "LEGAL_PERMANENT_RESIDENT", + ], + } + ) if prepared_pool: loss_basis = builder._fiscal_target_loss_basis(registry, np.ones(1)) @@ -7318,6 +7366,7 @@ def complete(self): "retirement", "telemetry", "puf_tail", + "immigration_drift", "spm_missing_pool", "qrf_tail_register", "qrf_tail_register_clean", @@ -7893,14 +7942,51 @@ def fake_energy_subsidy_signal_gate(frame): "with_us_immigration_inputs", lambda frame, *, seed, time_period: frame, ) + + def fake_immigration_composition_gate(frame, *, time_period): + assert time_period == builder.PERIOD + statuses = frame.table("person")["immigration_status_str"].astype(str) + household_weights = frame.weights_for("household") + weights = np.asarray(household_weights.values, dtype=np.float64) + assert len(statuses) == len(weights) + humanitarian = statuses.isin( + {"REFUGEE", "ASYLEE", "DEPORTATION_WITHHELD", "CUBAN_HAITIAN_ENTRANT"} + ).to_numpy(dtype=bool) + humanitarian_share = float(np.average(humanitarian, weights=weights)) + calls = captured.setdefault("immigration_gate_calls", []) + is_final_drift = ( + terminal_mode == "immigration_drift" + and household_weights.kind == WeightKind.CALIBRATED + ) + calls.append( + { + "weight_kind": household_weights.kind.value, + "weights": weights.tolist(), + "statuses": statuses.tolist(), + "humanitarian_share": humanitarian_share, + "passed": not is_final_drift, + } + ) + failure = ( + "weighted humanitarian status share drifted from 0.500000 to " + f"{humanitarian_share:.6f} after calibration " + "[final-immigration-sentinel]" + ) + return builder.GateResult( + name="immigration_composition", + passed=not is_final_drift, + failures=(failure,) if is_final_drift else (), + details={ + "checked": True, + "weight_kind": household_weights.kind.value, + "humanitarian_share": humanitarian_share, + }, + ) + monkeypatch.setattr( builder, "us_immigration_composition_gate", - lambda frame: builder.GateResult( - name="immigration_composition", - passed=True, - details={"checked": True}, - ), + fake_immigration_composition_gate, ) monkeypatch.setattr( builder, @@ -8684,6 +8770,12 @@ def fake_other_health_insurance_signal_gate(frame): passed=True, details={"checked": True}, ) + if terminal_mode == "immigration_drift": + return builder.GateResult( + name="other_health_insurance_premiums_signal", + passed=True, + details={"checked": True}, + ) # Export-frame call fails deliberately: the microcosm#547 cofailure # regression proves a failing post-solve signal gate batches # alongside the SSI delivery failure instead of masking it with an @@ -8977,6 +9069,7 @@ def fake_ssi_delivery_gate(diagnostics, *, targets, enforcement_fences=None): "integrity", "retirement", "puf_tail", + "immigration_drift", "qrf_tail_register_clean", "qrf_tail_register_green", "qrf_tail_register_green_skipped_smoke", @@ -9021,6 +9114,14 @@ def fake_release_gate_failures(*args, **kwargs): return [] if terminal_mode == "crash": raise RuntimeError("release-gate evaluation exploded [crash-sentinel]") + if terminal_mode == "immigration_drift": + final_immigration_gate = args[6] + assert final_immigration_gate.name == "immigration_composition" + assert not final_immigration_gate.passed + return [ + f"Immigration composition failed: {failure}" + for failure in final_immigration_gate.failures + ] if terminal_mode == "retirement": # The degraded pre-solve contract (PR #557 round 3): the failing # degenerate gate is NOT raised early — the gate object itself @@ -9151,6 +9252,19 @@ def fake_release_gate_failures(*args, **kwargs): "Bernoulli-law violation [final-integrity-sentinel]" ) assert "SSI take-up delivery failed:" not in message + elif terminal_mode == "immigration_drift": + # The drift leads; the only other line is the SPM composition + # gate's degraded-mode refusal on the household/person-only fake + # export frame, which this cofailure contract expects. + assert message == ( + "Release gates failed: Immigration composition failed: weighted " + "humanitarian status share drifted from 0.500000 to 0.255319 " + "after calibration [final-immigration-sentinel]; SPM measurement " + "composition failed (export frame): the export frame cannot be " + "classified, so the rule the engine applies to it cannot be " + "checked: Unknown entity 'spm_unit'; schema declares " + "['person', 'household']." + ) elif clean_run: # The register mismatch is the run's only terminal failure, yet # it reaches the batched raise (the old register raise escaped @@ -9168,14 +9282,14 @@ def fake_release_gate_failures(*args, **kwargs): "Release gates failed: SSI take-up delivery failed: " "18_64 delivered over envelope [cofailure-sentinel]" ) - if not clean_run: + if terminal_mode != "immigration_drift" and not clean_run: assert ( "Other health insurance signal failed on the export frame: " "premiums signal flattened [cofailure-sentinel]" in message ) if clean_run: pass - elif terminal_mode != "crash": + elif terminal_mode not in {"crash", "immigration_drift"}: assert "ctc failed" in message if terminal_mode == "telemetry": assert ( @@ -9183,7 +9297,7 @@ def fake_release_gate_failures(*args, **kwargs): "attach_artifact('calibration_diagnostics') crashed" in message ) assert "telemetry-crash-sentinel" in message - else: + elif terminal_mode == "crash": assert "health-input exploded [crash-sentinel]" in message assert "release-gate evaluation exploded [crash-sentinel]" in message assert "ctc failed" not in message @@ -9203,6 +9317,26 @@ def fake_release_gate_failures(*args, **kwargs): else: # pragma: no cover - defensive assertion raise AssertionError("Expected post-calibration gate failure.") + if terminal_mode == "immigration_drift": + immigration_calls = captured["immigration_gate_calls"] + assert len(immigration_calls) == 2 + source_call, final_call = immigration_calls + assert source_call["weight_kind"] == "importance" + assert source_call["weights"] == [1.0, 1.0, 1.0, 1.0] + assert source_call["statuses"] == [ + "REFUGEE", + "LEGAL_PERMANENT_RESIDENT", + "ASYLEE", + "LEGAL_PERMANENT_RESIDENT", + ] + assert source_call["humanitarian_share"] == pytest.approx(0.5) + assert source_call["passed"] is True + assert final_call["weight_kind"] == "calibrated" + assert final_call["weights"] == [12.0, 35.0] + assert final_call["statuses"] == ["REFUGEE", "LEGAL_PERMANENT_RESIDENT"] + assert final_call["humanitarian_share"] == pytest.approx(12.0 / 47.0) + assert final_call["passed"] is False + release_dir = out / "releases" / release_id written_diagnostics = json.loads( (release_dir / "calibration_diagnostics.json").read_text() @@ -9232,6 +9366,12 @@ def fake_release_gate_failures(*args, **kwargs): "Bernoulli-law violation [final-integrity-sentinel]" in written_diagnostics["build"]["release_gates"]["failures"] ) + elif terminal_mode == "immigration_drift": + assert written_diagnostics["build"]["release_gates"]["failures"] == [ + "Immigration composition failed: weighted humanitarian status share " + "drifted from 0.500000 to 0.255319 after calibration " + "[final-immigration-sentinel]" + ] # The SSI retry-basis artifact is written even though the run fails # terminally — it IS the remedy input for the next attempt. assert (release_dir / "us_ssi_take_up.json").exists() @@ -9256,6 +9396,15 @@ def fake_release_gate_failures(*args, **kwargs): cache_context = captured["materialize_kwargs"][ "target_materialization_cache_context" ] + active_specs = captured["ssi_band_targets_specs"] + assert any( + spec.metadata.get("target_role") == "humanitarian_immigration_stock" + for spec in active_specs + ) + assert ( + cache_context["target_registry_version"] + == TargetRegistry(active_specs, country="us").version + ) expected_evidence_identity = builder._target_frame_checkpoint_identity( base_dataset_sha256=cache_context["base_dataset_sha256"], policyengine_us_version=cache_context["policyengine_us_version"], @@ -9338,6 +9487,7 @@ def fake_release_gate_failures(*args, **kwargs): "retirement", "telemetry", "puf_tail", + "immigration_drift", "qrf_tail_register", "qrf_tail_register_clean", }: @@ -9381,6 +9531,12 @@ def fake_release_gate_failures(*args, **kwargs): "premiums signal flattened [cofailure-sentinel]", "ctc failed", ] + elif terminal_mode == "immigration_drift": + expected_gate_failures = [ + "Immigration composition failed: weighted humanitarian status share " + "drifted from 0.500000 to 0.255319 after calibration " + "[final-immigration-sentinel]" + ] elif clean_run: # Nothing failed before the terminal tail gate. expected_gate_failures = [] @@ -9678,7 +9834,12 @@ def fake_release_gate_failures(*args, **kwargs): "integrity_gate", # persisted-flag recheck on the export frame "delivery_gate", # enforced-band delivery, after the artifact exists ] - if terminal_mode not in {"integrity", "retirement", "qrf_tail_register_clean"}: + if terminal_mode not in { + "integrity", + "retirement", + "immigration_drift", + "qrf_tail_register_clean", + }: # A delivery miss rewrites the final measurement as the retry basis. expected_ssi_event_order.append("write:us_ssi_take_up.json") assert captured["ssi_event_order"] == expected_ssi_event_order @@ -12199,6 +12360,240 @@ def _invalidate_all_caches(self): assert compilation["dropped_target_names"] == [] +def test_humanitarian_stock_specs_follow_positive_manifest_draws( + monkeypatch, +) -> None: + builder = _load_builder_module() + draws = ( + SimpleNamespace( + label="paroled_one_year:afghanistan", + category="paroled_one_year", + origin="afghanistan", + status="PAROLED_ONE_YEAR", + target=75_000.0, + source="https://example.test/parole", + ), + SimpleNamespace( + label="refugee", + category="refugee", + origin=None, + status="REFUGEE", + target=160_000.0, + source="https://example.test/refugee", + ), + SimpleNamespace( + label="deportation_withheld", + category="deportation_withheld", + origin=None, + status="DEPORTATION_WITHHELD", + target=0.0, + source="https://example.test/withheld", + ), + SimpleNamespace( + label="tps:venezuela", + category="tps", + origin="venezuela", + status="TPS", + target=344_335.0, + source="https://example.test/tps", + ), + ) + monkeypatch.setattr( + builder, + "us_immigration_controls", + lambda: SimpleNamespace(humanitarian=draws), + ) + + specs = builder._humanitarian_immigration_stock_specs(time_period=2024) + + assert [spec.name for spec in specs] == [ + "humanitarian_immigration_stock.paroled_one_year.afghanistan", + "humanitarian_immigration_stock.refugee", + "humanitarian_immigration_stock.tps.venezuela", + ] + assert [spec.value for spec in specs] == [75_000.0, 160_000.0, 344_335.0] + assert [spec.source for spec in specs] == [ + "https://example.test/parole", + "https://example.test/refugee", + "https://example.test/tps", + ] + assert all(spec.period == 2024 for spec in specs) + assert all( + spec.metadata["materializer"] == "humanitarian_immigration_stock" + for spec in specs + ) + assert specs[0].metadata["humanitarian_origin"] == "afghanistan" + assert "humanitarian_origin" not in specs[1].metadata + # The active-registry content hash therefore binds the manifest controls; + # removing even one draw produces a different checkpoint registry id. + assert ( + TargetRegistry(specs, country="us").version + != TargetRegistry(specs[:-1], country="us").version + ) + + +def test_humanitarian_stock_materializer_collapses_source_aware_person_masks( + monkeypatch, + small_frame, +) -> None: + from microcosm.calibrate import build_constraint_matrix + + builder = _load_builder_module() + draws = ( + SimpleNamespace( + label="paroled_one_year:afghanistan", + category="paroled_one_year", + origin="afghanistan", + status="PAROLED_ONE_YEAR", + target=1.0, + source="fixture", + ), + SimpleNamespace( + label="refugee", + category="refugee", + origin=None, + status="REFUGEE", + target=2.0, + source="fixture", + ), + ) + controls = SimpleNamespace(humanitarian=draws) + monkeypatch.setattr(builder, "us_immigration_controls", lambda: controls) + observed: list[tuple[str, int]] = [] + + def draw_mask(frame, draw, *, time_period): + assert frame is small_frame + observed.append((draw.label, time_period)) + return { + "paroled_one_year:afghanistan": np.asarray([True, False, False, False]), + "refugee": np.asarray([False, True, True, False]), + }[draw.label] + + monkeypatch.setattr(builder, "us_immigration_humanitarian_draw_mask", draw_mask) + specs = builder._humanitarian_immigration_stock_specs(time_period=2024) + household = small_frame.table("household").copy() + + builder._materialize_humanitarian_immigration_stock_targets( + frame=small_frame, + household=household, + target_specs=specs, + time_period=2024, + ) + + np.testing.assert_array_equal( + household[ + "humanitarian_immigration_stock.paroled_one_year.afghanistan" + ].to_numpy(), + np.asarray([1.0, 0.0]), + ) + np.testing.assert_array_equal( + household["humanitarian_immigration_stock.refugee"].to_numpy(), + np.asarray([1.0, 1.0]), + ) + assert observed == [ + ("paroled_one_year:afghanistan", 2024), + ("refugee", 2024), + ] + + target_frame = Frame( + { + "person": small_frame.table("person"), + "household": household, + }, + small_frame.schema, + {"household": small_frame.weights_for("household")}, + small_frame.strata, + ) + registry, compilation = builder._compile_materialized_target_registry( + target_frame, + specs, + ) + target_set = registry.to_target_set() + + assert compilation["dropped_target_names"] == [] + assert tuple(target.row_name for target in target_set) == ( + "humanitarian_immigration_stock.paroled_one_year.afghanistan@2024", + "humanitarian_immigration_stock.refugee@2024", + ) + problem = build_constraint_matrix(target_frame, target_set) + assert problem.names == tuple(target.row_name for target in target_set) + assert problem.skipped == () + np.testing.assert_array_equal( + problem.matrix.toarray(), + np.asarray([[1.0, 0.0], [1.0, 1.0]]), + ) + np.testing.assert_array_equal( + problem.target_vector, + np.asarray([1.0, 2.0]), + ) + + +def test_dense_and_l0_solvers_consume_compiled_registry_target_set() -> None: + import ast + + builder = _load_builder_module() + tree = ast.parse(Path(builder.__file__).read_text()) + main_fn = next( + node + for node in ast.walk(tree) + if isinstance(node, ast.FunctionDef) and node.name == "_main" + ) + + for solver_name in ("calibrate", "calibrate_l0_refit"): + calls = [ + node + for node in ast.walk(main_fn) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == solver_name + ] + assert len(calls) == 1 + assert ast.unparse(calls[0].args[0]) == "target_frame" + assert ast.unparse(calls[0].args[1]) == "registry.to_target_set()" + + +def test_main_rechecks_immigration_composition_on_export_for_terminal_gates() -> None: + import ast + + builder = _load_builder_module() + tree = ast.parse(Path(builder.__file__).read_text()) + main_fn = next( + node + for node in ast.walk(tree) + if isinstance(node, ast.FunctionDef) and node.name == "_main" + ) + final_assignments = [ + node + for node in ast.walk(main_fn) + if isinstance(node, ast.Assign) + and any( + isinstance(target, ast.Name) and target.id == "final_immigration_gate" + for target in node.targets + ) + ] + assert len(final_assignments) == 1 + final_assignment = final_assignments[0] + assert isinstance(final_assignment.value, ast.Call) + assert getattr(final_assignment.value.func, "id", None) == ( + "us_immigration_composition_gate" + ) + assert ast.unparse(final_assignment.value.args[0]) == "export_frame" + assert [ + (keyword.arg, ast.unparse(keyword.value)) + for keyword in final_assignment.value.keywords + ] == [("time_period", "PERIOD")] + + terminal_calls = [ + node + for node in ast.walk(main_fn) + if isinstance(node, ast.Call) + and getattr(node.func, "id", None) == "_release_gate_failures" + and node.lineno > final_assignment.lineno + ] + assert len(terminal_calls) == 1 + assert ast.unparse(terminal_calls[0].args[6]) == "final_immigration_gate" + + def test_unknown_ledger_filter_metadata_fails_closed() -> None: builder = _load_builder_module() target = TargetSpec( diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_immigration.py b/packages/microcosm-build/tests/engine_free/us/test_us_immigration.py index 23a1cb19fb..4255d01144 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_immigration.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_immigration.py @@ -6,6 +6,7 @@ import pandas as pd import pytest +import microcosm.build.us_runtime.immigration as immigration_module from microcosm.build.source_manifest import SourceStageSpec from microcosm.build.source_runtime import ( SourceRuntimeConfig, @@ -13,6 +14,7 @@ run_source_stage, ) from microcosm.build.us_runtime import ( + HUMANITARIAN_STATUS_CATEGORIES, IMMIGRATION_STATUS_VALUES, SSN_CARD_TYPE_VALUES, US_DONORS, @@ -20,6 +22,8 @@ US_IMMIGRATION_STAGE_NAME, US_SOURCE_MANIFEST, US_STAGE_NAMES, + HumanitarianDraw, + ImmigrationControls, UndocumentedControls, derive_us_immigration_status_from_manifest, us_immigration_composition_gate, @@ -27,18 +31,66 @@ us_immigration_stage_spec, with_us_immigration_inputs, ) +from microcosm.build.us_runtime.immigration import ( + us_immigration_controls, + us_immigration_humanitarian_draw_mask, +) from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights _HANDLERS = {"derive_immigration_status": derive_us_immigration_status_from_manifest} TIME_PERIOD = 2024 +_PAROLE_ORIGIN_KEYS = ("afghanistan", "ukraine", "nicaragua", "venezuela") +_TPS_ORIGIN_KEYS = ( + "venezuela", + "el_salvador", + "honduras", + "nicaragua", + "nepal", + "other_designated", +) + + +def _humanitarian_block( + *, + paroled_one_year: dict[str, float] | None = None, + refugee: float = 0.0, + asylee: float = 0.0, + deportation_withheld: float = 0.0, + tps: dict[str, float] | None = None, +) -> dict: + """Manifest-shaped humanitarian block; unnamed targets default to zero.""" + + parole_targets = dict.fromkeys(_PAROLE_ORIGIN_KEYS, 0.0) | (paroled_one_year or {}) + tps_targets = dict.fromkeys(_TPS_ORIGIN_KEYS, 0.0) | (tps or {}) + return { + "paroled_one_year": { + origin: { + "target": target, + "source": f"https://example.com/parole/{origin}", + } + for origin, target in parole_targets.items() + }, + "refugee": {"target": refugee, "source": "https://example.com/refugee"}, + "asylee": {"target": asylee, "source": "https://example.com/asylee"}, + "deportation_withheld": { + "target": deportation_withheld, + "source": "https://example.com/withheld", + }, + "tps": { + origin: {"target": target, "source": f"https://example.com/tps/{origin}"} + for origin, target in tps_targets.items() + }, + } + def _stage_spec( *, workers: float, students: float, anchor: float, + humanitarian: dict | None = None, ) -> SourceStageSpec: return SourceStageSpec.from_mapping( { @@ -64,6 +116,11 @@ def _stage_spec( "value": anchor, "source": "https://example.com/population", }, + "humanitarian_status_stocks": ( + humanitarian + if humanitarian is not None + else _humanitarian_block() + ), }, ], "outputs": list(US_IMMIGRATION_OUTPUT_COLUMNS), @@ -82,6 +139,7 @@ def _person_table(rows: list[dict]) -> pd.DataFrame: "A_MARITL": 7, "A_SPOUSE": 0, "A_HSCOL": 0, + "A_LFSR": 7, "WSAL_VAL": 0.0, "SEMP_VAL": 0.0, "MCARE": 2, @@ -126,10 +184,16 @@ def _run( workers: float = 100.0, students: float = 100.0, anchor: float = 10.0, + humanitarian: dict | None = None, seed: int = 0, ) -> pd.DataFrame: return run_source_stage( - _stage_spec(workers=workers, students=students, anchor=anchor), + _stage_spec( + workers=workers, + students=students, + anchor=anchor, + humanitarian=humanitarian, + ), tables={"person": person}, operation_handlers=_HANDLERS, config=SourceRuntimeConfig(seed=seed, target_year=TIME_PERIOD), @@ -180,13 +244,15 @@ def test_noncitizen_without_indicators_is_undocumented(self) -> None: assert output.loc[0, "ssn_card_type"] == "NONE" assert output.loc[0, "immigration_status_str"] == "UNDOCUMENTED" - def test_worker_spill_leaves_undocumented_workers_at_control(self) -> None: + def test_worker_spill_leaves_pew_unauthorized_labor_force_at_control( + self, + ) -> None: person = _person_table( - [_noncitizen(WSAL_VAL=10_000.0) for _ in range(10)] + [_noncitizen(A_LFSR=1, WSAL_VAL=10_000.0) for _ in range(10)] + [_noncitizen(), _noncitizen()] ) output = _run(person, workers=4.0) - workers = output["WSAL_VAL"] > 0 + workers = output["A_LFSR"].isin([1, 2, 3, 4]) undocumented_workers = ((output["ssn_card_type"] == "NONE") & workers).sum() ead_workers = ( (output["ssn_card_type"] == "NON_CITIZEN_VALID_EAD") & workers @@ -196,8 +262,147 @@ def test_worker_spill_leaves_undocumented_workers_at_control(self) -> None: # Non-workers are untouched by the worker spill. assert (output.loc[~workers, "ssn_card_type"] == "NONE").all() + def test_worker_spill_uses_labor_force_status_not_prior_year_earnings( + self, + ) -> None: + person = _person_table( + [ + # Looking for work now, despite having no prior-year earnings. + _noncitizen(A_LFSR=3), + # Earned wages last year, but is now outside the labor force. + _noncitizen(A_LFSR=7, WSAL_VAL=10_000.0), + ] + ) + output = _run(person, workers=0.001) + assert output.loc[0, "ssn_card_type"] == "NON_CITIZEN_VALID_EAD" + assert output.loc[1, "ssn_card_type"] == "NONE" + + def test_worker_control_retains_residual_cuban_haitian_ead_rows(self) -> None: + person = _person_table( + [ + _noncitizen(A_LFSR=1, PENATVTY=327, PEINUSYR=24), + _noncitizen(A_LFSR=1, PENATVTY=332, PEINUSYR=24), + *[_noncitizen(A_LFSR=1) for _ in range(8)], + ] + ) + output = _run(person, workers=4.0) + labor_force = output["A_LFSR"].isin([1, 2, 3, 4]) + pew_unauthorized = output["immigration_status_str"].isin( + [ + "UNDOCUMENTED", + "DACA", + "PAROLED_ONE_YEAR", + "DEPORTATION_WITHHELD", + "TPS", + ] + ) | ( + output["immigration_status_str"].eq("CUBAN_HAITIAN_ENTRANT") + & output["ssn_card_type"].eq("NON_CITIZEN_VALID_EAD") + ) + assert int((labor_force & pew_unauthorized).sum()) == 4 + + def test_worker_control_uses_pew_age_16_labor_force_universe(self) -> None: + person = _person_table( + [ + _noncitizen(A_AGE=15, A_LFSR=1), + _noncitizen(A_AGE=16, A_LFSR=3), + ] + ) + output = _run(person, workers=0.001) + assert output.loc[0, "ssn_card_type"] == "NONE" + assert output.loc[1, "ssn_card_type"] == "NON_CITIZEN_VALID_EAD" + + def test_invalid_labor_force_status_is_refused(self) -> None: + with pytest.raises(SourceRuntimeError, match="A_LFSR"): + _run(_person_table([_noncitizen(A_LFSR=6)])) + + def test_worker_control_counts_tps_before_spilling_residual_workers( + self, + ) -> None: + person = _person_table( + [ + _noncitizen( + PENATVTY=373, + PEINUSYR=24, + A_LFSR=1, + ), + _noncitizen(A_LFSR=1), + ] + ) + output = _run( + person, + workers=1.0, + humanitarian=_humanitarian_block(tps={"venezuela": 1.0}), + ) + assert output.loc[0, "immigration_status_str"] == "TPS" + # Pew includes TPS holders in its unauthorized estimate. The TPS row + # therefore exhausts the one-person labor-force control, so the other + # residual worker must spill to EAD rather than remain undocumented. + assert output.loc[1, "ssn_card_type"] == "NON_CITIZEN_VALID_EAD" + assert output.loc[1, "immigration_status_str"] == ("LEGAL_PERMANENT_RESIDENT") + + @pytest.mark.parametrize("seed", range(6)) + def test_student_spill_exhausts_nonworkers_before_touching_a_worker( + self, seed: int + ) -> None: + # Three residual students, control 2: a nonworking DACA-cohort row + # (an EAD leaves it inside Pew's universe, so spilling it closes + # nothing), a nonworking non-cohort row, and the only worker. The + # corrective draw must take the nonworker: the worker pass cannot + # restore a worker the student pass removed. + person = _person_table( + [ + _noncitizen(A_HSCOL=2, PEINUSYR=17, A_AGE=30), + _noncitizen(A_HSCOL=2), + _noncitizen(A_HSCOL=2, A_LFSR=1), + ] + ) + output = _run(person, students=2.0, workers=1.0, seed=seed) + status = output["immigration_status_str"] + in_pew_universe = status.isin(["UNDOCUMENTED", "DACA"]) + in_labor_force = output["A_LFSR"].isin([1, 2, 3, 4]) + assert int(in_pew_universe.sum()) == 2 + assert int((in_pew_universe & in_labor_force).sum()) == 1 + assert status.iloc[2] == "UNDOCUMENTED" + + def test_exact_student_closure_residue_never_spills_a_worker( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + # Regression for 5a89fed1e. The student control 0.3 equals, in exact + # arithmetic, the two working students' 0.1 + 0.2, so spilling the + # one nonworking student (weight 1.0) closes the student excess + # exactly. In binary floating point the recomputed remainder is + # 0.1 + 0.2 - 0.3 = 5.6e-17 > 0; compared with zero, that residue + # opened the second tier and spilled a worker the worker control + # still needed. + person = _person_table( + [ + _noncitizen(A_HSCOL=2, person_weight=1.0), + _noncitizen(A_HSCOL=2, A_LFSR=1, person_weight=0.1), + _noncitizen(A_HSCOL=2, A_LFSR=1, person_weight=0.2), + ] + ) + assert float(np.asarray([0.1, 0.2]).sum()) - 0.3 > 0.0 + for seed in range(4): + output = _run(person, students=0.3, workers=100.0, seed=seed) + assert output["ssn_card_type"].tolist() == [ + "NON_CITIZEN_VALID_EAD", + "NONE", + "NONE", + ], seed + + # Sensitivity: without the closure tolerance the same case spills a + # worker, so the assertions above bind on the tolerance. + monkeypatch.setattr( + immigration_module, "_SPILL_CLOSURE_RELATIVE_TOLERANCE", 0.0 + ) + output = _run(person, students=0.3, workers=100.0, seed=0) + assert output.loc[1:, "ssn_card_type"].eq("NON_CITIZEN_VALID_EAD").any() + def test_below_control_counts_spill_nothing(self) -> None: - person = _person_table([_noncitizen(WSAL_VAL=10_000.0), _noncitizen(A_HSCOL=2)]) + person = _person_table( + [_noncitizen(A_LFSR=1, WSAL_VAL=10_000.0), _noncitizen(A_HSCOL=2)] + ) output = _run(person, workers=50.0, students=50.0) assert (output["ssn_card_type"] == "NONE").all() @@ -210,16 +415,16 @@ def test_student_spill_leaves_undocumented_students_at_control(self) -> None: def test_weights_drive_the_spill_amounts(self) -> None: person = _person_table( [ - _noncitizen(WSAL_VAL=10_000.0, person_weight=6.0), - _noncitizen(WSAL_VAL=10_000.0, person_weight=6.0), + _noncitizen(A_LFSR=1, WSAL_VAL=10_000.0, person_weight=6.0), + _noncitizen(A_LFSR=1, WSAL_VAL=10_000.0, person_weight=6.0), ] ) output = _run(person, workers=6.0) assert set(output["ssn_card_type"]) == {"NON_CITIZEN_VALID_EAD", "NONE"} def test_indicator_holders_never_flip_to_undocumented(self) -> None: - # The total undocumented population is emergent: a short count is - # never topped up from people with legal-status indicators. + # The total Pew-defined unauthorized population is emergent: a short + # count is never topped up from people with legal-status indicators. person = _person_table( [_noncitizen(), _noncitizen(CAID=1, person_household_id=1)] ) @@ -230,6 +435,18 @@ def test_prcitshp_outside_domain_raises(self) -> None: with pytest.raises(SourceRuntimeError, match="PRCITSHP"): _run(_person_table([{"PRCITSHP": 7}])) + def test_2026_asec_peinusyr_29_is_in_the_domain(self) -> None: + # The 2026 ASEC (income year 2025, in the default pool) codes + # 2024-2026 arrivals as 29. A Medicaid-reporting arrival there is a + # documented non-citizen like any other. + output = _run(_person_table([_noncitizen(PEINUSYR=29, CAID=1)])) + assert output.loc[0, "immigration_status_str"] == "LEGAL_PERMANENT_RESIDENT" + assert output.loc[0, "ssn_card_type"] == "OTHER_NON_CITIZEN" + + def test_peinusyr_past_the_2026_asec_domain_raises(self) -> None: + with pytest.raises(SourceRuntimeError, match=r"PEINUSYR.*0\.\.29"): + _run(_person_table([_noncitizen(PEINUSYR=30)])) + def test_missing_required_column_raises(self) -> None: person = _person_table([_noncitizen()]).drop(columns=["PEINUSYR"]) with pytest.raises(SourceRuntimeError, match="PEINUSYR"): @@ -245,14 +462,18 @@ class TestImmigrationStatusTags: def test_daca_statutory_cohort_among_ead_holders(self) -> None: # Arrived 2005 (code 19) aged 10 → age at entry < 16, now 29, EAD via # worker spill with a zero control. - person = _person_table([_noncitizen(PEINUSYR=19, A_AGE=29, WSAL_VAL=20_000.0)]) + person = _person_table( + [_noncitizen(PEINUSYR=19, A_AGE=29, A_LFSR=1, WSAL_VAL=20_000.0)] + ) output = _run(person, workers=0.001) assert output.loc[0, "ssn_card_type"] == "NON_CITIZEN_VALID_EAD" assert output.loc[0, "immigration_status_str"] == "DACA" def test_ead_outside_daca_cohort_is_lpr(self) -> None: # Arrived 2015 as an adult: fails the DACA arrival test. - person = _person_table([_noncitizen(PEINUSYR=24, A_AGE=40, WSAL_VAL=20_000.0)]) + person = _person_table( + [_noncitizen(PEINUSYR=24, A_AGE=40, A_LFSR=1, WSAL_VAL=20_000.0)] + ) output = _run(person, workers=0.001) assert output.loc[0, "ssn_card_type"] == "NON_CITIZEN_VALID_EAD" assert output.loc[0, "immigration_status_str"] == "LEGAL_PERMANENT_RESIDENT" @@ -279,7 +500,7 @@ def test_undocumented_tag_matches_none_ssn_exactly(self) -> None: [ _noncitizen(), _noncitizen(CAID=1), - _noncitizen(WSAL_VAL=10_000.0), + _noncitizen(A_LFSR=1, WSAL_VAL=10_000.0), {"PRCITSHP": 1}, ] ) @@ -295,10 +516,15 @@ def test_emitted_values_stay_inside_engine_enum_domains(self) -> None: for overrides in ( {}, {"CAID": 1}, - {"WSAL_VAL": 10_000.0}, + {"A_LFSR": 1, "WSAL_VAL": 10_000.0}, {"A_HSCOL": 2}, {"PENATVTY": 327}, - {"PEINUSYR": 19, "A_AGE": 25, "WSAL_VAL": 5_000.0}, + { + "PEINUSYR": 19, + "A_AGE": 25, + "A_LFSR": 1, + "WSAL_VAL": 5_000.0, + }, ) ] + [{"PRCITSHP": 1}] @@ -308,9 +534,593 @@ def test_emitted_values_stay_inside_engine_enum_domains(self) -> None: assert set(output["immigration_status_str"]) <= set(IMMIGRATION_STATUS_VALUES) +class TestHumanitarianDraws: + def test_parole_draw_takes_documented_origin_cohort(self) -> None: + person = _person_table( + [ + # 2022-arrived Ukrainian reporting Medicaid: the U4U signature. + _noncitizen(PENATVTY=164, PEINUSYR=28, CAID=1), + # Same profile from a non-parole origin stays LPR. + _noncitizen(PENATVTY=303, PEINUSYR=28, CAID=1), + ] + ) + output = _run( + person, + humanitarian=_humanitarian_block(paroled_one_year={"ukraine": 1.0}), + ) + assert output.loc[0, "immigration_status_str"] == "PAROLED_ONE_YEAR" + assert output.loc[0, "ssn_card_type"] == "OTHER_NON_CITIZEN" + assert output.loc[1, "immigration_status_str"] == "LEGAL_PERMANENT_RESIDENT" + + def test_parole_origin_targets_do_not_cross(self) -> None: + person = _person_table( + [ + _noncitizen(PENATVTY=315, PEINUSYR=28, CAID=1), # Nicaraguan + ] + ) + output = _run( + person, + humanitarian=_humanitarian_block(paroled_one_year={"ukraine": 5.0}), + ) + assert output.loc[0, "immigration_status_str"] == "LEGAL_PERMANENT_RESIDENT" + + def test_parole_needs_recent_arrival(self) -> None: + # A 2014-2015-arrived Ukrainian predates every parole program. + person = _person_table([_noncitizen(PENATVTY=164, PEINUSYR=24, CAID=1)]) + output = _run( + person, + humanitarian=_humanitarian_block(paroled_one_year={"ukraine": 5.0}), + ) + assert output.loc[0, "immigration_status_str"] == "LEGAL_PERMANENT_RESIDENT" + + def test_2026_asec_code_29_draws_only_programs_running_in_2024(self) -> None: + # Code 29 is the 2026 ASEC's 2024-2026 bin: inside the U4U, CHNV and + # FY2023-FY2024 refugee windows, after Operation Allies Welcome, the + # FY2022-FY2024 asylum grants and every TPS residence date. + def status_at_29(birth: int, **block) -> str: + person = _person_table([_noncitizen(PENATVTY=birth, PEINUSYR=29, CAID=1)]) + output = _run(person, humanitarian=_humanitarian_block(**block)) + return output.loc[0, "immigration_status_str"] + + drawn = "PAROLED_ONE_YEAR" + assert status_at_29(164, paroled_one_year={"ukraine": 1.0}) == drawn + assert status_at_29(315, paroled_one_year={"nicaragua": 1.0}) == drawn + assert status_at_29(373, paroled_one_year={"venezuela": 1.0}) == drawn + assert status_at_29(412, refugee=1.0) == "REFUGEE" + kept = "LEGAL_PERMANENT_RESIDENT" + assert status_at_29(200, paroled_one_year={"afghanistan": 1.0}) == kept + assert status_at_29(207, asylee=1.0) == kept + assert status_at_29(373, tps={"venezuela": 1.0}) == kept + assert status_at_29(205, tps={"other_designated": 1.0}) == kept + + def test_u4u_rejects_2020_2021_ukrainian_arrivals(self) -> None: + person = _person_table( + [ + _noncitizen(PENATVTY=164, PEINUSYR=27, CAID=1), + _noncitizen(PENATVTY=164, PEINUSYR=28, CAID=1), + ] + ) + output = _run( + person, + humanitarian=_humanitarian_block(paroled_one_year={"ukraine": 2.0}), + ) + assert output["immigration_status_str"].tolist() == [ + "LEGAL_PERMANENT_RESIDENT", + "PAROLED_ONE_YEAR", + ] + + compatibility = _us_frame( + [ + _noncitizen( + PENATVTY=164, + PEINUSYR=27, + ssn_card_type="OTHER_NON_CITIZEN", + immigration_status_str="PAROLED_ONE_YEAR", + ), + _noncitizen( + PENATVTY=164, + PEINUSYR=28, + ssn_card_type="OTHER_NON_CITIZEN", + immigration_status_str="PAROLED_ONE_YEAR", + ), + ] + ) + draw = HumanitarianDraw( + category="paroled_one_year", + origin="ukraine", + status="PAROLED_ONE_YEAR", + target=1.0, + source="https://example.com/u4u", + ) + assert us_immigration_humanitarian_draw_mask( + compatibility, + draw, + time_period=TIME_PERIOD, + ).tolist() == [False, True] + + tables = { + entity: compatibility.table(entity).copy() + for entity in compatibility.entities + } + tables["person"] = tables["person"].drop( + columns=["PRCITSHP", "PENATVTY", "PEINUSYR"] + ) + tables["person"]["CIT"] = [5, 5] + tables["person"]["POBP"] = [164, 164] + tables["person"]["YOEP"] = [2021, 2022] + acs_compatibility = Frame( + tables, + compatibility.schema, + { + entity: compatibility.weights_for(entity) + for entity in compatibility.weighted_entities + }, + ) + assert us_immigration_humanitarian_draw_mask( + acs_compatibility, + draw, + time_period=TIME_PERIOD, + ).tolist() == [False, True] + + def test_acs_venezuela_tps_includes_2023_designation_cohort(self) -> None: + compatibility = _us_frame( + [ + _noncitizen( + PENATVTY=373, + PEINUSYR=28, + ssn_card_type="NON_CITIZEN_VALID_EAD", + immigration_status_str=status, + ) + for status in ("TPS", "TPS", "TPS", "PAROLED_ONE_YEAR") + ] + ) + tables = { + entity: compatibility.table(entity).copy() + for entity in compatibility.entities + } + person = tables["person"].drop(columns=["PRCITSHP", "PENATVTY", "PEINUSYR"]) + person["CIT"] = [5, 5, 5, 5] + person["POBP"] = [373, 373, 373, 373] + person["YOEP"] = [2021, 2023, 2024, 2023] + tables["person"] = person + frame = Frame( + tables, + compatibility.schema, + { + entity: compatibility.weights_for(entity) + for entity in compatibility.weighted_entities + }, + ) + tps = HumanitarianDraw( + category="tps", + origin="venezuela", + status="TPS", + target=1.0, + source="https://example.com/venezuela-tps", + ) + parole = HumanitarianDraw( + category="paroled_one_year", + origin="venezuela", + status="PAROLED_ONE_YEAR", + target=1.0, + source="https://example.com/chnv", + ) + + assert us_immigration_humanitarian_draw_mask( + frame, + tps, + time_period=TIME_PERIOD, + ).tolist() == [True, True, False, False] + assert us_immigration_humanitarian_draw_mask( + frame, + parole, + time_period=TIME_PERIOD, + ).tolist() == [False, False, False, True] + + @pytest.mark.parametrize( + ("citizenship", "arrival_year"), + ( + (4, np.nan), + (5, np.nan), + (4, 1899), + (5, TIME_PERIOD + 1), + (5, 2020.5), + ), + ) + def test_acs_foreign_born_arrival_evidence_fails_closed( + self, + citizenship: int, + arrival_year: float, + ) -> None: + compatibility = _us_frame( + [ + _noncitizen( + PENATVTY=373, + PEINUSYR=28, + ssn_card_type="NON_CITIZEN_VALID_EAD", + immigration_status_str="TPS", + ) + ] + ) + tables = { + entity: compatibility.table(entity).copy() + for entity in compatibility.entities + } + person = tables["person"].drop(columns=["PRCITSHP", "PENATVTY", "PEINUSYR"]) + person["CIT"] = [citizenship] + person["POBP"] = [373] + person["YOEP"] = [arrival_year] + tables["person"] = person + frame = Frame( + tables, + compatibility.schema, + { + entity: compatibility.weights_for(entity) + for entity in compatibility.weighted_entities + }, + ) + draw = HumanitarianDraw( + category="tps", + origin="venezuela", + status="TPS", + target=1.0, + source="https://example.com/venezuela-tps", + ) + + with pytest.raises( + SourceRuntimeError, + match="ACS YOEP must be an integral 1900..time_period year", + ): + us_immigration_humanitarian_draw_mask( + frame, + draw, + time_period=TIME_PERIOD, + ) + + @pytest.mark.parametrize( + ("birth_country", "origin", "cutoff"), + ( + (373, "venezuela", 2023), + (312, "el_salvador", 2001), + (314, "honduras", 1998), + (315, "nicaragua", 1998), + (229, "nepal", 2015), + (205, "other_designated", 2024), + (239, "other_designated", 2024), + (248, "other_designated", 2024), + (224, "other_designated", 2024), + (407, "other_designated", 2023), + (416, "other_designated", 2024), + (448, "other_designated", 2024), + (451, "other_designated", 2023), + ), + ) + def test_acs_tps_continuous_residence_cutoff( + self, + birth_country: int, + origin: str, + cutoff: int, + ) -> None: + compatibility = _us_frame( + [ + _noncitizen( + PENATVTY=birth_country, + PEINUSYR=28, + A_AGE=70, + ssn_card_type="NON_CITIZEN_VALID_EAD", + immigration_status_str="TPS", + ), + _noncitizen( + PENATVTY=birth_country, + PEINUSYR=28, + A_AGE=70, + ssn_card_type="NON_CITIZEN_VALID_EAD", + immigration_status_str="TPS", + ), + ] + ) + tables = { + entity: compatibility.table(entity).copy() + for entity in compatibility.entities + } + person = tables["person"].drop(columns=["PRCITSHP", "PENATVTY", "PEINUSYR"]) + person["CIT"] = [5, 5] + person["POBP"] = [birth_country, birth_country] + person["YOEP"] = [cutoff, cutoff + 1] + tables["person"] = person + frame = Frame( + tables, + compatibility.schema, + { + entity: compatibility.weights_for(entity) + for entity in compatibility.weighted_entities + }, + ) + draw = HumanitarianDraw( + category="tps", + origin=origin, + status="TPS", + target=1.0, + source="https://example.com/tps-cutoff", + ) + + assert us_immigration_humanitarian_draw_mask( + frame, + draw, + time_period=max(TIME_PERIOD, cutoff + 1), + ).tolist() == [True, False] + + def test_refugee_draw_excludes_residual_pool(self) -> None: + # A 2022-arrived Congolese person with no legal-status indicator is + # not refugee-consistent (refugees carry immediate benefits access). + person = _person_table( + [ + _noncitizen(PENATVTY=412, PEINUSYR=28), + _noncitizen(PENATVTY=412, PEINUSYR=28, CAID=1), + ] + ) + output = _run(person, humanitarian=_humanitarian_block(refugee=1.0)) + assert output.loc[0, "immigration_status_str"] == "UNDOCUMENTED" + assert output.loc[1, "immigration_status_str"] == "REFUGEE" + + def test_asylee_draw_uses_backlog_window(self) -> None: + person = _person_table( + [ + # 2018-2019 arrival from China with Medicaid: grant-consistent. + _noncitizen(PENATVTY=207, PEINUSYR=26, CAID=1), + # 2004-2005 arrival predates the grant backlog window. + _noncitizen(PENATVTY=207, PEINUSYR=19, CAID=1, A_AGE=50), + ] + ) + output = _run(person, humanitarian=_humanitarian_block(asylee=1.0)) + assert output.loc[0, "immigration_status_str"] == "ASYLEE" + assert output.loc[1, "immigration_status_str"] == "LEGAL_PERMANENT_RESIDENT" + + def test_tps_residual_draw_flips_ssn_to_ead(self) -> None: + person = _person_table([_noncitizen(PENATVTY=373, PEINUSYR=28)]) + output = _run(person, humanitarian=_humanitarian_block(tps={"venezuela": 1.0})) + assert output.loc[0, "immigration_status_str"] == "TPS" + assert output.loc[0, "ssn_card_type"] == "NON_CITIZEN_VALID_EAD" + none_ssn = output["ssn_card_type"] == "NONE" + undocumented = output["immigration_status_str"] == "UNDOCUMENTED" + assert (none_ssn == undocumented).all() + + def test_tps_legacy_continuous_residence_window_binds(self) -> None: + person = _person_table( + [ + # Arrived 1996-1997: inside the El Salvador 2001 cutoff. + _noncitizen(PENATVTY=312, PEINUSYR=15, CAID=1, A_AGE=55), + # Arrived 2014-2015: cannot hold El Salvador TPS. + _noncitizen(PENATVTY=312, PEINUSYR=24, CAID=1, A_AGE=40), + ] + ) + output = _run( + person, humanitarian=_humanitarian_block(tps={"el_salvador": 5.0}) + ) + assert output.loc[0, "immigration_status_str"] == "TPS" + assert output.loc[1, "immigration_status_str"] == "LEGAL_PERMANENT_RESIDENT" + + def test_daca_cohort_is_excluded_from_humanitarian_draws(self) -> None: + # Arrived 2000-2001 aged ~7: the DACA statutory cohort. The worker + # spill gives the EAD card and the DACA tag survives TPS targeting. + person = _person_table( + [ + _noncitizen( + PENATVTY=312, + PEINUSYR=17, + A_AGE=30, + A_LFSR=1, + WSAL_VAL=20_000.0, + ) + ] + ) + output = _run( + person, + workers=0.001, + humanitarian=_humanitarian_block(tps={"el_salvador": 5.0}), + ) + assert output.loc[0, "immigration_status_str"] == "DACA" + + def test_cuban_haitian_entrants_survive_humanitarian_targets(self) -> None: + person = _person_table([_noncitizen(PENATVTY=327, PEINUSYR=28, CAID=1)]) + output = _run( + person, + humanitarian=_humanitarian_block( + paroled_one_year={"venezuela": 5.0}, tps={"venezuela": 5.0} + ), + ) + assert output.loc[0, "immigration_status_str"] == "CUBAN_HAITIAN_ENTRANT" + + def test_draw_order_gives_parole_precedence_over_tps(self) -> None: + person = _person_table([_noncitizen(PENATVTY=373, PEINUSYR=28, CAID=1)]) + both = _run( + person, + humanitarian=_humanitarian_block( + paroled_one_year={"venezuela": 1.0}, tps={"venezuela": 1.0} + ), + ) + tps_only = _run( + person, humanitarian=_humanitarian_block(tps={"venezuela": 1.0}) + ) + assert both.loc[0, "immigration_status_str"] == "PAROLED_ONE_YEAR" + assert tps_only.loc[0, "immigration_status_str"] == "TPS" + + def test_draws_are_weight_targeted(self) -> None: + person = _person_table( + [ + _noncitizen(PENATVTY=164, PEINUSYR=28, CAID=1, person_weight=3.0), + _noncitizen(PENATVTY=164, PEINUSYR=28, CAID=1, person_weight=3.0), + ] + ) + output = _run( + person, + humanitarian=_humanitarian_block(paroled_one_year={"ukraine": 3.0}), + ) + statuses = sorted(output["immigration_status_str"]) + assert statuses == ["LEGAL_PERMANENT_RESIDENT", "PAROLED_ONE_YEAR"] + + def test_zero_targets_draw_nothing(self) -> None: + person = _person_table( + [ + _noncitizen(PENATVTY=164, PEINUSYR=28, CAID=1), + _noncitizen(PENATVTY=373, PEINUSYR=28), + _noncitizen(PENATVTY=412, PEINUSYR=28, CAID=1), + ] + ) + output = _run(person, humanitarian=_humanitarian_block()) + assert set(output["immigration_status_str"]) <= { + "LEGAL_PERMANENT_RESIDENT", + "UNDOCUMENTED", + } + + def test_humanitarian_spill_interaction_preserves_worker_control(self) -> None: + # TPS extraction removes a residual worker before the spill, so the + # remaining undocumented workers still land on the control exactly. + person = _person_table( + [ + _noncitizen( + PENATVTY=373, + PEINUSYR=28, + A_LFSR=1, + WSAL_VAL=10_000.0, + ) + ] + + [_noncitizen(A_LFSR=1, WSAL_VAL=10_000.0) for _ in range(10)] + ) + output = _run( + person, + workers=4.0, + humanitarian=_humanitarian_block(tps={"venezuela": 1.0}), + ) + workers = output["A_LFSR"].isin([1, 2, 3, 4]) + pew_workers = workers & output["immigration_status_str"].isin( + [ + "UNDOCUMENTED", + "DACA", + "PAROLED_ONE_YEAR", + "DEPORTATION_WITHHELD", + "TPS", + ] + ) + assert output.loc[0, "immigration_status_str"] == "TPS" + assert pew_workers.sum() == 4 + + def test_missing_humanitarian_block_is_refused(self) -> None: + spec_mapping = { + "stage": US_IMMIGRATION_STAGE_NAME, + "survey": "test", + "source": "https://example.com", + "grain": "person", + "operations": [ + {"kind": "read_table", "table": "person"}, + { + "kind": "derive_immigration_status", + "seed_from_build_config": True, + "time_period_from_build_config": True, + "undocumented_workers": { + "target": 1, + "source": "https://example.com", + }, + "undocumented_students": { + "target": 1, + "source": "https://example.com", + }, + "undocumented_population_anchor": { + "value": 1, + "source": "https://example.com", + }, + }, + ], + "outputs": list(US_IMMIGRATION_OUTPUT_COLUMNS), + } + with pytest.raises(SourceRuntimeError, match="humanitarian_status_stocks"): + run_source_stage( + SourceStageSpec.from_mapping(spec_mapping), + tables={"person": _person_table([_noncitizen()])}, + operation_handlers=_HANDLERS, + config=SourceRuntimeConfig(seed=0, target_year=TIME_PERIOD), + ) + + @pytest.mark.parametrize( + "mutate, match", + [ + (lambda block: block.pop("refugee"), "missing category"), + ( + lambda block: block.__setitem__( + "mystery", {"target": 1, "source": "https://example.com"} + ), + "unsupported category", + ), + ( + lambda block: block["paroled_one_year"].pop("ukraine"), + "missing origin", + ), + ( + lambda block: block["tps"].__setitem__( + "atlantis", {"target": 1, "source": "https://example.com"} + ), + "outside the stage's codebook table", + ), + ( + lambda block: block["refugee"].pop("source"), + "source citation", + ), + ( + lambda block: block["refugee"].__setitem__("target", -1), + "non-negative", + ), + ], + ) + def test_malformed_humanitarian_blocks_are_refused(self, mutate, match) -> None: + block = _humanitarian_block(refugee=1.0) + mutate(block) + with pytest.raises(SourceRuntimeError, match=match): + _run(_person_table([_noncitizen()]), humanitarian=block) + + def test_humanitarian_clones_stay_consistent(self) -> None: + rows = [ + _noncitizen( + PENATVTY=164, + PEINUSYR=28, + CAID=1, + person_id=index + 1, + source_year=2024, + source_person_id=f"P{index % 5}", + ) + for index in range(10) + ] + output = _run( + _person_table(rows), + humanitarian=_humanitarian_block(paroled_one_year={"ukraine": 5.0}), + ) + by_source = output.groupby("source_person_id")["immigration_status_str"] + assert (by_source.nunique() == 1).all() + + class TestDeterminism: def _worker_pool(self) -> pd.DataFrame: - return _person_table([_noncitizen(WSAL_VAL=10_000.0) for _ in range(20)]) + return _person_table( + [_noncitizen(A_LFSR=1, WSAL_VAL=10_000.0) for _ in range(20)] + ) + + def test_partial_household_lineage_uses_complete_legacy_alternative(self) -> None: + person = _person_table([_noncitizen(), _noncitizen()]) + person["source_year"] = [2024, 2024] + person["source_household_id"] = [100, np.nan] + person["source_person_id"] = [1, 2] + + partial_full = immigration_module._stable_person_draws( + person, + seed=17, + salt="lineage-regression", + ) + legacy_only = immigration_module._stable_person_draws( + person.drop(columns="source_household_id"), + seed=17, + salt="lineage-regression", + ) + + np.testing.assert_array_equal(partial_full, legacy_only) def test_same_seed_is_bit_reproducible(self) -> None: first = _run(self._worker_pool(), workers=10.0, seed=7) @@ -325,6 +1135,7 @@ def test_different_seeds_select_different_ead_holders(self) -> None: def test_source_identity_keys_make_clones_consistent(self) -> None: rows = [ _noncitizen( + A_LFSR=1, WSAL_VAL=10_000.0, person_id=index + 1, source_year=2024, @@ -363,6 +1174,52 @@ def test_manifest_controls_carry_citations(self) -> None: assert float(block[value_key]) > 0 assert str(block["source"]).startswith("https://") + def test_manifest_humanitarian_stocks_carry_citations(self) -> None: + humanitarian = ( + us_immigration_stage_spec() + .operations[1] + .parameters["humanitarian_status_stocks"] + ) + assert set(humanitarian) == set(HUMANITARIAN_STATUS_CATEGORIES) + flat_blocks = { + "refugee": humanitarian["refugee"], + "asylee": humanitarian["asylee"], + "deportation_withheld": humanitarian["deportation_withheld"], + **{ + f"paroled_one_year:{origin}": block + for origin, block in humanitarian["paroled_one_year"].items() + }, + **{f"tps:{origin}": block for origin, block in humanitarian["tps"].items()}, + } + assert set(humanitarian["paroled_one_year"]) == set(_PAROLE_ORIGIN_KEYS) + assert set(humanitarian["tps"]) == set(_TPS_ORIGIN_KEYS) + for label, block in flat_blocks.items(): + assert float(block["target"]) >= 0, label + assert str(block["source"]).startswith("https://"), label + assert humanitarian["refugee"] == { + "target": 160_000, + "source": ( + "https://ohss.dhs.gov/topics/immigration/refugees/" + "annual-flow-report/fy-24-refugees-flow-report" + ), + } + assert humanitarian["asylee"] == { + "target": 155_000, + "source": ( + "https://ohss.dhs.gov/topics/immigration/asylees/" + "annual-flow-report/fy24-asylees-flow-report" + ), + } + # The withheld stock is an explicit cited zero (no published stock); + # every other category carries a positive stock. + assert float(humanitarian["deportation_withheld"]["target"]) == 0 + positive = [ + label + for label, block in flat_blocks.items() + if label != "deportation_withheld" + ] + assert all(float(flat_blocks[label]["target"]) > 0 for label in positive) + def test_unexpected_parameter_is_refused(self) -> None: spec = SourceStageSpec.from_mapping( { @@ -481,7 +1338,7 @@ def test_with_us_immigration_inputs_writes_both_columns(self) -> None: rows = ( [{"PRCITSHP": 1} for _ in range(93)] + [_noncitizen(CAID=1) for _ in range(2)] - + [_noncitizen(WSAL_VAL=10_000.0) for _ in range(12)] + + [_noncitizen(A_LFSR=1, WSAL_VAL=10_000.0) for _ in range(12)] + [_noncitizen() for _ in range(5)] ) frame = _us_frame(rows, household_weights=[1e6] * len(rows)) @@ -491,7 +1348,7 @@ def test_with_us_immigration_inputs_writes_both_columns(self) -> None: assert column in person.columns assert set(person["ssn_card_type"]) <= set(SSN_CARD_TYPE_VALUES) assert (person.loc[person["PRCITSHP"] == 1, "ssn_card_type"] == "CITIZEN").all() - # 12M weighted undocumented workers against the 8.3M Pew control: + # 12M weighted unauthorized workers against the 9.7M Pew control: # some spill to EAD, the rest stay undocumented. assert (person["ssn_card_type"] == "NON_CITIZEN_VALID_EAD").any() assert (person["ssn_card_type"] == "NONE").any() @@ -521,16 +1378,41 @@ def test_missing_raw_columns_raise_loudly(self) -> None: with_us_immigration_inputs(stripped, seed=0, time_period=TIME_PERIOD) -def _plausible_controls() -> UndocumentedControls: - return UndocumentedControls( - workers=20.0, - students=5.0, - population_anchor=30.0, - sources={ - "undocumented_workers": "https://example.com/workers", - "undocumented_students": "https://example.com/students", - "undocumented_population_anchor": "https://example.com/population", - }, +def _humanitarian_draws(**targets: float) -> tuple[HumanitarianDraw, ...]: + """Zero-target draw per category, overridden by keyword (national sums).""" + + status_by_category = { + "paroled_one_year": "PAROLED_ONE_YEAR", + "refugee": "REFUGEE", + "asylee": "ASYLEE", + "deportation_withheld": "DEPORTATION_WITHHELD", + "tps": "TPS", + } + return tuple( + HumanitarianDraw( + category=category, + origin=None, + status=status_by_category[category], + target=float(targets.get(category, 0.0)), + source=f"https://example.com/{category}", + ) + for category in HUMANITARIAN_STATUS_CATEGORIES + ) + + +def _plausible_controls(**humanitarian_targets: float) -> ImmigrationControls: + return ImmigrationControls( + undocumented=UndocumentedControls( + workers=20.0, + students=5.0, + population_anchor=30.0, + sources={ + "undocumented_workers": "https://example.com/workers", + "undocumented_students": "https://example.com/students", + "undocumented_population_anchor": "https://example.com/population", + }, + ), + humanitarian=_humanitarian_draws(**humanitarian_targets), ) @@ -540,6 +1422,7 @@ def _composition_frame( other: int = 30, ead: int = 10, none: int = 30, + humanitarian: dict[str, int] | None = None, ) -> Frame: rows: list[dict] = [] values: list[tuple[str, str]] = ( @@ -547,13 +1430,30 @@ def _composition_frame( + [("OTHER_NON_CITIZEN", "LEGAL_PERMANENT_RESIDENT")] * other + [("NON_CITIZEN_VALID_EAD", "LEGAL_PERMANENT_RESIDENT")] * ead + [("NONE", "UNDOCUMENTED")] * none + + [ + ( + ("NON_CITIZEN_VALID_EAD" if status == "DACA" else "OTHER_NON_CITIZEN"), + status, + ) + for status, count in (humanitarian or {}).items() + for _ in range(count) + ] ) + evidence_by_status = { + "PAROLED_ONE_YEAR": {"PENATVTY": 164, "PEINUSYR": 28}, + "REFUGEE": {"PENATVTY": 412, "PEINUSYR": 28}, + "ASYLEE": {"PENATVTY": 207, "PEINUSYR": 26}, + "DEPORTATION_WITHHELD": {"PENATVTY": 303, "PEINUSYR": 24}, + "TPS": {"PENATVTY": 373, "PEINUSYR": 24}, + "DACA": {"PENATVTY": 303, "PEINUSYR": 19, "A_AGE": 29}, + } for ssn, status in values: rows.append( { "PRCITSHP": 1 if ssn == "CITIZEN" else 5, "ssn_card_type": ssn, "immigration_status_str": status, + **evidence_by_status.get(status, {}), } ) return _us_frame(rows) @@ -562,13 +1462,78 @@ def _composition_frame( class TestCompositionGate: def test_passes_on_plausible_composition(self) -> None: gate = us_immigration_composition_gate( - _composition_frame(), controls=_plausible_controls() + time_period=TIME_PERIOD, + frame=_composition_frame(), + controls=_plausible_controls(), ) assert gate.passed, gate.failures + def test_gate_rederives_the_daca_cohort_at_the_stage_period(self) -> None: + # Regression for 5a89fed1e: the gate re-derived cohort evidence at a + # hard-coded 2024. A worker who arrived in 2007 (PEINUSYR 20, the + # 2006-2007 bin) and is 33 entered at 16 when measured in 2024, + # outside the DACA cohort ("arrived by 2007 before age 16"), but at + # 15 when measured in 2025, inside it. The residual worker spill + # gives the row an EAD at either period. + rows = ( + [{"PRCITSHP": 1} for _ in range(90)] + + [_noncitizen() for _ in range(9)] + + [_noncitizen(PEINUSYR=20, A_AGE=33, A_LFSR=1)] + ) + controls = ImmigrationControls( + undocumented=UndocumentedControls( + workers=0.5, + students=100.0, + population_anchor=10.0, + sources=_plausible_controls().undocumented.sources, + ), + humanitarian=_humanitarian_draws(), + ) + + def stage_frame(period: int) -> Frame: + output = run_source_stage( + _stage_spec(workers=0.5, students=100.0, anchor=10.0), + tables={"person": _person_table(rows)}, + operation_handlers=_HANDLERS, + config=SourceRuntimeConfig(seed=0, target_year=period), + ) + return _us_frame(output.drop(columns=["person_weight"]).to_dict("records")) + + by_period = {period: stage_frame(period) for period in (2024, 2025)} + labels = { + period: frame.table("person").iloc[-1][list(US_IMMIGRATION_OUTPUT_COLUMNS)] + for period, frame in by_period.items() + } + assert labels[2024].tolist() == [ + "NON_CITIZEN_VALID_EAD", + "LEGAL_PERMANENT_RESIDENT", + ] + assert labels[2025].tolist() == ["NON_CITIZEN_VALID_EAD", "DACA"] + + cohort_failure = ( + "1 person(s) violate the source-aware Cuban/Haitian entrant or DACA " + "cohort contract." + ) + for stage_period, frame in by_period.items(): + for gate_period in (2024, 2025): + gate = us_immigration_composition_gate( + frame, time_period=gate_period, controls=controls + ) + invalid = gate.details["evidence_derived_status_compatibility"][ + "invalid_rows" + ] + if gate_period == stage_period: + assert gate.passed, (stage_period, gate.failures) + assert invalid == 0 + else: + assert gate.failures == (cohort_failure,) + assert invalid == 1 + def test_fails_when_columns_missing(self) -> None: gate = us_immigration_composition_gate( - _us_frame([{"PRCITSHP": 1}]), controls=_plausible_controls() + time_period=TIME_PERIOD, + frame=_us_frame([{"PRCITSHP": 1}]), + controls=_plausible_controls(), ) assert not gate.passed assert any("missing person column" in failure for failure in gate.failures) @@ -583,7 +1548,9 @@ def test_fails_on_the_225_failure_mode_all_citizens(self) -> None: for _ in range(50) ] gate = us_immigration_composition_gate( - _us_frame(rows), controls=_plausible_controls() + time_period=TIME_PERIOD, + frame=_us_frame(rows), + controls=_plausible_controls(), ) assert not gate.passed assert any("constant" in failure for failure in gate.failures) @@ -592,7 +1559,9 @@ def test_fails_on_values_outside_engine_enum_domain(self) -> None: frame = _composition_frame() person = frame.table("person") person.loc[0, "ssn_card_type"] = "5" - gate = us_immigration_composition_gate(frame, controls=_plausible_controls()) + gate = us_immigration_composition_gate( + time_period=TIME_PERIOD, frame=frame, controls=_plausible_controls() + ) assert not gate.passed assert any("enum domain" in failure for failure in gate.failures) @@ -600,13 +1569,16 @@ def test_fails_when_columns_disagree_about_citizenship(self) -> None: frame = _composition_frame() person = frame.table("person") person.loc[0, "immigration_status_str"] = "LEGAL_PERMANENT_RESIDENT" - gate = us_immigration_composition_gate(frame, controls=_plausible_controls()) + gate = us_immigration_composition_gate( + time_period=TIME_PERIOD, frame=frame, controls=_plausible_controls() + ) assert not gate.passed assert any("citizenship" in failure for failure in gate.failures) def test_fails_when_undocumented_far_from_anchor(self) -> None: gate = us_immigration_composition_gate( - _composition_frame(citizens=930, other=48, ead=10, none=2), + time_period=TIME_PERIOD, + frame=_composition_frame(citizens=930, other=48, ead=10, none=2), controls=_plausible_controls(), ) assert not gate.passed @@ -615,23 +1587,184 @@ def test_fails_when_undocumented_far_from_anchor(self) -> None: def test_fails_when_non_citizen_share_implausible(self) -> None: gate = us_immigration_composition_gate( - _composition_frame(citizens=40, other=20, ead=20, none=20), - controls=UndocumentedControls( - workers=8.0, - students=1.0, - population_anchor=20.0, - sources=_plausible_controls().sources, + time_period=TIME_PERIOD, + frame=_composition_frame(citizens=40, other=20, ead=20, none=20), + controls=ImmigrationControls( + undocumented=UndocumentedControls( + workers=8.0, + students=1.0, + population_anchor=20.0, + sources=_plausible_controls().undocumented.sources, + ), + humanitarian=_humanitarian_draws(), ), ) assert not gate.passed assert any("non-citizen weighted share" in failure for failure in gate.failures) def test_gate_reads_packaged_controls_by_default(self) -> None: - gate = us_immigration_composition_gate(_us_frame([{"PRCITSHP": 1}])) + packaged = us_immigration_controls() + assert packaged.undocumented.workers == 9_700_000 + assert packaged.humanitarian_target("refugee") == 160_000 + gate = us_immigration_composition_gate( + time_period=TIME_PERIOD, frame=_us_frame([{"PRCITSHP": 1}]) + ) assert not gate.passed controls = gate.details["controls"] - assert controls["undocumented_workers"] == 8_300_000 - assert controls["undocumented_population_anchor"] == 11_000_000 + assert controls["undocumented_workers"] == 9_700_000 + assert controls["undocumented_population_anchor"] == 14_000_000 + stocks = controls["humanitarian_status_stocks"] + assert stocks["paroled_one_year:afghanistan"]["target"] == 73_566 + assert stocks["refugee"] == { + "target": 160_000.0, + "source": ( + "https://ohss.dhs.gov/topics/immigration/refugees/" + "annual-flow-report/fy-24-refugees-flow-report" + ), + } + assert stocks["asylee"] == { + "target": 155_000.0, + "source": ( + "https://ohss.dhs.gov/topics/immigration/asylees/" + "annual-flow-report/fy24-asylees-flow-report" + ), + } + assert stocks["tps:venezuela"]["target"] == 605_015 + assert stocks["deportation_withheld"]["target"] == 0 + + def test_gate_passes_with_in_band_humanitarian_masses(self) -> None: + gate = us_immigration_composition_gate( + time_period=TIME_PERIOD, + frame=_composition_frame(humanitarian={"REFUGEE": 4, "TPS": 6}), + controls=_plausible_controls(refugee=4.0, tps=6.0), + ) + assert gate.passed, gate.failures + achieved = gate.details["humanitarian_achieved"] + assert achieved["refugee"]["population"] == 4.0 + assert achieved["refugee"]["relative"] == 1.0 + assert achieved["tps"]["target"] == 6.0 + + def test_gate_counts_temporary_protections_in_pew_population(self) -> None: + gate = us_immigration_composition_gate( + time_period=TIME_PERIOD, + frame=_composition_frame( + none=10, + humanitarian={"DACA": 5, "PAROLED_ONE_YEAR": 10, "TPS": 10}, + ), + controls=_plausible_controls(paroled_one_year=10.0, tps=10.0), + ) + assert gate.passed, gate.failures + assert gate.details["pew_unauthorized_population"] == 35.0 + + def test_gate_counts_only_residual_ead_cuban_haitian_entrants(self) -> None: + frame = _composition_frame() + person = frame.table("person") + ead_row = person.index[person["ssn_card_type"].eq("NON_CITIZEN_VALID_EAD")][0] + documented_row = person.index[person["ssn_card_type"].eq("OTHER_NON_CITIZEN")][ + 0 + ] + for row in (ead_row, documented_row): + person.loc[row, "immigration_status_str"] = "CUBAN_HAITIAN_ENTRANT" + person.loc[row, "PENATVTY"] = 327 + person.loc[row, "PEINUSYR"] = 24 + + gate = us_immigration_composition_gate( + time_period=TIME_PERIOD, + frame=frame, + controls=_plausible_controls(), + ) + assert gate.passed, gate.failures + assert gate.details["pew_unauthorized_population"] == 31.0 + assert gate.details["pew_unauthorized_paired_status"] == { + "immigration_status_str": "CUBAN_HAITIAN_ENTRANT", + "ssn_card_type": "NON_CITIZEN_VALID_EAD", + } + + def test_gate_excludes_refugees_and_asylees_from_pew_population(self) -> None: + controls = ImmigrationControls( + undocumented=UndocumentedControls( + workers=20.0, + students=5.0, + population_anchor=10.0, + sources=_plausible_controls().undocumented.sources, + ), + humanitarian=_humanitarian_draws(refugee=10.0, asylee=10.0), + ) + gate = us_immigration_composition_gate( + time_period=TIME_PERIOD, + frame=_composition_frame( + none=10, + humanitarian={"REFUGEE": 10, "ASYLEE": 10}, + ), + controls=controls, + ) + assert gate.passed, gate.failures + assert gate.details["pew_unauthorized_population"] == 10.0 + + def test_gate_fails_when_humanitarian_category_collapses(self) -> None: + gate = us_immigration_composition_gate( + time_period=TIME_PERIOD, + frame=_composition_frame(), + controls=_plausible_controls(refugee=4.0), + ) + assert not gate.passed + assert any( + "REFUGEE" in failure and "degenerate" in failure + for failure in gate.failures + ) + + def test_gate_fails_when_explicit_zero_category_emits(self) -> None: + gate = us_immigration_composition_gate( + time_period=TIME_PERIOD, + frame=_composition_frame(humanitarian={"DEPORTATION_WITHHELD": 2}), + controls=_plausible_controls(), + ) + assert not gate.passed + assert any("explicit zero target" in failure for failure in gate.failures) + + def test_gate_accepts_saturated_draws_inside_band(self) -> None: + # The ASEC undercovers 2022-24 arrivals; a draw that saturates at + # roughly three-quarters of its admin target stays inside the band. + gate = us_immigration_composition_gate( + time_period=TIME_PERIOD, + frame=_composition_frame(humanitarian={"PAROLED_ONE_YEAR": 3}), + controls=_plausible_controls(paroled_one_year=4.0), + ) + assert gate.passed, gate.failures + + def test_gate_checks_per_origin_draws_not_only_category_total(self) -> None: + source = "https://example.com/parole" + controls = ImmigrationControls( + undocumented=_plausible_controls().undocumented, + humanitarian=( + HumanitarianDraw( + category="paroled_one_year", + origin="afghanistan", + status="PAROLED_ONE_YEAR", + target=1.0, + source=source, + ), + HumanitarianDraw( + category="paroled_one_year", + origin="ukraine", + status="PAROLED_ONE_YEAR", + target=1.0, + source=source, + ), + ), + ) + # Both rows carry Ukraine evidence. The category total is exactly two, + # but the per-origin targets are a zero/two redistribution. + gate = us_immigration_composition_gate( + time_period=TIME_PERIOD, + frame=_composition_frame(humanitarian={"PAROLED_ONE_YEAR": 2}), + controls=controls, + ) + assert not gate.passed + assert any( + "paroled_one_year:afghanistan" in failure for failure in gate.failures + ) + assert any("paroled_one_year:ukraine" in failure for failure in gate.failures) def test_summary_reports_weighted_composition(self) -> None: summary = us_immigration_composition_summary( @@ -641,3 +1774,41 @@ def test_summary_reports_weighted_composition(self) -> None: assert ssn["population"]["CITIZEN"] == 3.0 assert ssn["population"]["NONE"] == 1.0 assert summary["person_population"] == 5.0 + + +def test_exact_year_parole_windows_close_when_their_program_closed() -> None: + # Operation Allies Welcome paroled Afghans who arrived in 2021 and 2022, so + # an exact-year (ACS YOEP) row is its candidate only in those years. + # Uniting for Ukraine began in 2022 and has no last year. + years = [2020.0, 2021.0, 2022.0, 2023.0, 2024.0] + person = pd.DataFrame( + { + "CIT": [5.0] * 10, + "POBP": [200.0] * 5 + [164.0] * 5, + "YOEP": years * 2, + "age": [30.0] * 10, + } + ) + profile = immigration_module._source_aware_immigration_profile( + person, time_period=2024 + ) + draws = { + draw.label: draw + for draw in immigration_module.us_immigration_controls().humanitarian + } + + def candidates(label: str) -> list[bool]: + return immigration_module._humanitarian_draw_candidates( + draws[label], + ssn_codes=np.full(len(person), 3), + profile=profile, + ).tolist() + + assert candidates("paroled_one_year:afghanistan") == [ + *(False, True, True, False, False), + *([False] * 5), + ] + assert candidates("paroled_one_year:ukraine") == [ + *([False] * 5), + *(False, False, True, True, True), + ] diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_immigration_pool_scaling.py b/packages/microcosm-build/tests/engine_free/us/test_us_immigration_pool_scaling.py new file mode 100644 index 0000000000..45820b87da --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/us/test_us_immigration_pool_scaling.py @@ -0,0 +1,329 @@ +"""Pooled immigration absolute-control mass regressions (microcosm #767).""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.us_runtime import immigration as immigration_module +from microcosm.build.us_runtime import multispine_pool as multispine_pool_module +from microcosm.build.us_runtime.immigration import ( + US_IMMIGRATION_OUTPUT_COLUMNS, + US_IMMIGRATION_STAGE_NAME, + with_us_immigration_inputs, +) +from microcosm.build.us_runtime.puf_support import clone_us_frame_for_puf_support +from microcosm.build.us_runtime.spine_assembly import assemble_spines +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights + + +def _humanitarian_controls(*, ukraine: float) -> dict[str, object]: + parole_origins = ("afghanistan", "ukraine", "nicaragua", "venezuela") + tps_origins = ( + "venezuela", + "el_salvador", + "honduras", + "nicaragua", + "nepal", + "other_designated", + ) + return { + "paroled_one_year": { + origin: { + "target": ukraine if origin == "ukraine" else 0.0, + "source": f"https://example.com/parole/{origin}", + } + for origin in parole_origins + }, + "refugee": {"target": 0.0, "source": "https://example.com/refugee"}, + "asylee": {"target": 0.0, "source": "https://example.com/asylee"}, + "deportation_withheld": { + "target": 0.0, + "source": "https://example.com/withheld", + }, + "tps": { + origin: { + "target": 0.0, + "source": f"https://example.com/tps/{origin}", + } + for origin in tps_origins + }, + } + + +def _stage_spec() -> SourceStageSpec: + return SourceStageSpec.from_mapping( + { + "stage": US_IMMIGRATION_STAGE_NAME, + "survey": "test ASEC", + "source": "https://example.com", + "grain": "person", + "operations": [ + {"kind": "read_table", "table": "person"}, + { + "kind": "derive_immigration_status", + "seed_from_build_config": True, + "time_period_from_build_config": True, + "undocumented_workers": { + "target": 4.0, + "source": "https://example.com/workers", + }, + "undocumented_students": { + "target": 2.0, + "source": "https://example.com/students", + }, + "undocumented_population_anchor": { + "value": 10.0, + "source": "https://example.com/population", + }, + "humanitarian_status_stocks": _humanitarian_controls(ukraine=4.0), + }, + ], + "outputs": list(US_IMMIGRATION_OUTPUT_COLUMNS), + } + ) + + +def _immigration_frame() -> Frame: + rows: list[dict[str, object]] = [] + rows.extend({"PRCITSHP": 5, "A_LFSR": 1, "WSAL_VAL": 10_000.0} for _ in range(4)) + rows.extend({"PRCITSHP": 5, "A_HSCOL": 2} for _ in range(3)) + rows.extend( + { + "PRCITSHP": 5, + "PENATVTY": 164, + "PEINUSYR": 28, + "CAID": 1, + } + for _ in range(8) + ) + baseline: dict[str, object] = { + "PRCITSHP": 1, + "PEINUSYR": 0, + "PENATVTY": 57, + "A_AGE": 40, + "A_MARITL": 7, + "A_SPOUSE": 0, + "A_HSCOL": 0, + "A_LFSR": 7, + "WSAL_VAL": 0.0, + "SEMP_VAL": 0.0, + "MCARE": 2, + "CAID": 2, + "IHSFLG": 2, + "CHAMPVA": 2, + "MIL": 2, + "PEN_SC1": 0, + "PEN_SC2": 0, + "RESNSS1": 0, + "RESNSS2": 0, + "SS_YN": 2, + "SSI_YN": 2, + "PEIO1COW": 0, + "A_MJOCC": 0, + "PEAFEVER": 2, + "SPM_CAPHOUSESUB": 0.0, + } + records = [] + for position, overrides in enumerate(rows, start=1): + record = dict(baseline) + record.update(overrides) + record.update( + { + "person_id": position, + "person_household_id": position, + "person_tax_unit_id": position, + "person_spm_unit_id": position, + "person_family_id": position, + "person_marital_unit_id": position, + } + ) + records.append(record) + person = pd.DataFrame(records) + ids = np.arange(1, len(person) + 1, dtype=np.int64) + return Frame( + { + "person": person, + **{ + entity: pd.DataFrame({f"{entity}_id": ids}) + for entity in US_SCHEMA.group_entities + }, + }, + US_SCHEMA, + { + "household": Weights( + np.ones(len(person), dtype=np.float64), + WeightKind.DESIGN, + ) + }, + ) + + +def _weighted_count(frame: Frame, mask: pd.Series) -> float: + weights = np.asarray(frame.resolve_weights("person").values, dtype=np.float64) + return float(weights[mask.to_numpy(dtype=bool)].sum()) + + +def test_stage_weight_scale_allocates_every_absolute_control_to_source_share( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(immigration_module, "us_immigration_stage_spec", _stage_spec) + + standalone = with_us_immigration_inputs( + _immigration_frame(), seed=0, time_period=2024 + ) + pooled_projection = with_us_immigration_inputs( + _immigration_frame(), + seed=0, + time_period=2024, + person_weight_scale=2.0, + ) + + standalone_person = standalone.table("person") + pooled_person = pooled_projection.table("person") + assert ( + _weighted_count( + standalone, + standalone_person["immigration_status_str"].eq("PAROLED_ONE_YEAR"), + ) + == 4.0 + ) + assert ( + _weighted_count( + pooled_projection, + pooled_person["immigration_status_str"].eq("PAROLED_ONE_YEAR"), + ) + == 2.0 + ) + + standalone_workers = standalone_person["A_LFSR"].isin([1, 2, 3, 4]) + pooled_workers = pooled_person["A_LFSR"].isin([1, 2, 3, 4]) + assert ( + _weighted_count( + standalone, + standalone_workers & standalone_person["ssn_card_type"].eq("NONE"), + ) + == 4.0 + ) + assert ( + _weighted_count( + pooled_projection, + pooled_workers & pooled_person["ssn_card_type"].eq("NONE"), + ) + == 2.0 + ) + + standalone_students = standalone_person["A_HSCOL"].eq(2) + pooled_students = pooled_person["A_HSCOL"].eq(2) + assert ( + _weighted_count( + standalone, + standalone_students & standalone_person["ssn_card_type"].eq("NONE"), + ) + == 2.0 + ) + assert ( + _weighted_count( + pooled_projection, + pooled_students & pooled_person["ssn_card_type"].eq("NONE"), + ) + == 1.0 + ) + + +def _spine(*, cps: bool, offset: int = 0) -> Frame: + ids = np.asarray([1, 2], dtype=np.int64) + person = pd.DataFrame( + { + "person_id": ids, + "person_household_id": ids, + "person_tax_unit_id": ids, + "person_spm_unit_id": ids, + "person_family_id": ids, + "person_marital_unit_id": ids, + "age": [30.0, 50.0], + **({"PERIDNUM": [f"p{offset + 1}", f"p{offset + 2}"]} if cps else {}), + } + ) + return Frame( + { + "person": person, + **{ + entity: pd.DataFrame({f"{entity}_id": ids}) + for entity in US_SCHEMA.group_entities + }, + }, + US_SCHEMA, + { + "household": Weights( + np.asarray([2.0, 2.0]), + WeightKind.DESIGN, + ) + }, + ) + + +def _replace_person(frame: Frame, person: pd.DataFrame) -> Frame: + return Frame( + { + "person": person, + **{ + entity: frame.table(entity) + for entity in frame.entities + if entity != "person" + }, + }, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +def test_postclone_chain_derives_and_receipts_inverse_cps_person_mass_share() -> None: + assembled = assemble_spines( + {"asec": _spine(cps=True), "acs": _spine(cps=False, offset=100)}, + household_mass_shares={"asec": 0.5, "acs": 0.5}, + ) + cloned = clone_us_frame_for_puf_support(assembled) + observed_scale: list[float] = [] + + def immigration( + available: Frame, + *, + person_weight_scale: float, + ) -> Frame: + observed_scale.append(person_weight_scale) + person = available.table("person").copy() + person["ssn_card_type"] = "CITIZEN" + person["immigration_status_str"] = "CITIZEN" + return _replace_person(available, person) + + completed = multispine_pool_module._run_source_operator_chain( + cloned, + phase="post_clone", + operator_names=("with_us_immigration_inputs",), + operators={"with_us_immigration_inputs": immigration}, + ) + + person = cloned.table("person") + weights = np.asarray(cloned.resolve_weights("person").values, dtype=np.float64) + cps = person["PERIDNUM"].notna().to_numpy(dtype=bool) + full_mass = float(weights.sum()) + cps_mass = float(weights[cps].sum()) + expected_scale = full_mass / cps_mass + assert cps_mass / full_mass == pytest.approx(0.5) + assert observed_scale == [pytest.approx(expected_scale)] + + scaling = completed.receipt["suboperators"][0]["kernel_receipt"][ + "person_design_weight_scaling" + ] + assert scaling == { + "full_pool_person_design_weight_mass": pytest.approx(full_mass), + "cps_projection_person_design_weight_mass": pytest.approx(cps_mass), + "person_weight_scale": pytest.approx(expected_scale), + "cps_person_mass_share": pytest.approx(cps_mass / full_mass), + } diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_immigration_properties.py b/packages/microcosm-build/tests/engine_free/us/test_us_immigration_properties.py new file mode 100644 index 0000000000..c9b79f4314 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/us/test_us_immigration_properties.py @@ -0,0 +1,2185 @@ +"""Property tests for the US immigration-status stage (microcosm #225, #767). + +Every property below quotes the docstring, comment or runtime check in +``microcosm.build.us_runtime.immigration`` that promises it. Properties marked +*deduced* combine two stated mechanisms rather than restating one sentence +(for example, draws keyed by person identity plus threshold selection imply +row-order equivariance). The example tests live in ``test_us_immigration.py``. + +Invariants checked, by function: + +- ``_select_weight_to_target``: the selection is the candidates at or below + one draw threshold, so tie groups are never split; a non-positive amount + selects nothing; a feasible selection reaches the amount and overshoots it + by less than its final tie group, and an infeasible one takes every + candidate; it equals an independent group-walk reference; it is row-order + equivariant (deduced). +- ``_stable_person_draws``: draws depend only on seed, salt and the most + complete stable lineage (equal keys draw equally, different keys draw + differently), reruns repeat and row order does not matter (deduced). +- ``_spill_pew_unauthorized_excess``: only in-scope residual non-citizens move, + and only to EAD; Pew-universe membership never grows; a margin within the + closure tolerance of its control spills nothing; every closable excess is + closed and an unclosable one exhausts the removable pool; the undershoot is + less than one flipped row; rows inside the other margin are touched only + after every removable row outside it, and only when removing those leaves + more than rounding residue (explicit example: the minimal exact-closure + case, where 0.1 + 0.2 - 0.3 > 0 in binary floating point). +- The stage (``derive_us_immigration_status_from_manifest``): ``NONE`` pairs + exactly with ``UNDOCUMENTED`` and ``CITIZEN`` with measured citizenship; + source columns pass through; the status column is the documented function of + SSN code, humanitarian mark and evidence; the indicator-documented pool is + exactly the non-citizens carrying a documented indicator; humanitarian + labels respect their exclusions and SSN pools, are mutually exclusive, reach + their targets or exhaust their pools, precede the EAD spill, and sit inside + the ASEC arrival windows the module's comments state; the output is + bit-reproducible, row-order equivariant (deduced) and consistent across + support clones; the Pew margins close whenever the residual pool can close + them. +- ``with_us_immigration_inputs``: labels every person exactly as the manifest + handler does, leaves weights and other tables alone, and is idempotent. +- Differential: stage output satisfies every structural check of + ``us_immigration_composition_gate`` at the stage's own period (explicit + example: a row whose DACA-cohort membership changes between 2024 and + 2025), and the spill's ``_pew_unauthorized_projection_mask`` selects + exactly the rows the gate counts as Pew-unauthorized. +- ``reconcile_us_immigration_humanitarian_transfer``: its output satisfies the + gate's structural contract; each receipt's achieved population is its + immutable plus selected population; a draw whose candidates hold less than + its residual takes every candidate no earlier draw took, and its receipt + records the shortfall; a draw that then reaches less than the composition + band's floor of its target is refused; and + ``us_immigration_humanitarian_transfer_selection_masks`` replays the + reconciliation's mutable-row selections exactly (differential). + +Independence limits, stated so no test claims more than it shows: + +- The candidate pools (origin x arrival window x SSN pool) of the draw-target + and transfer properties come from the module's own + ``_humanitarian_draw_candidates``; those properties check selection against + target, not the windows. Only + ``test_humanitarian_rows_sit_inside_the_asec_windows_the_comments_state`` + checks windows, against values transcribed from the module's comments. + That is a comment-versus-table consistency check, not external validation. +- Reconciliation raises ``RuntimeError`` before returning a receipt that + misses its tie-mass bounds or changes an immutable row, so asserting those + on returned receipts only confirms the receipt records what was checked. +- ``_facts`` reuses the module's arrival-midpoint and Cuba/Haiti code tables; + the DACA thresholds come from the module docstring. + +Hypothesis reaches the engine-free CI job through ``uv sync --all-packages`` +(microcosm-graph's dev group), so it is imported directly: a missing +dependency fails loudly rather than skipping these tests. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass, replace + +import numpy as np +import pandas as pd +import pytest +from hypothesis import event, example, given, settings +from hypothesis import strategies as st + +import microcosm.build.us_runtime.immigration as immigration +from microcosm.build.source_manifest import SourceOperationSpec +from microcosm.build.source_runtime import ( + SourceRuntimeConfig, + SourceRuntimeContext, + SourceRuntimeError, +) +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights + +#: Unit properties are cheap; stage properties run the full derivation per +#: example. ``database=None`` keeps runs from saving examples into the +#: checkout. +_UNIT = settings(max_examples=100, deadline=None, database=None) +_STAGE = settings(max_examples=50, deadline=None, database=None) +_TRANSFER = settings(max_examples=30, deadline=None, database=None) +_FRAME = settings(max_examples=20, deadline=None, database=None) + +_TIME_PERIOD = 2024 +#: Stage periods. The DACA cohort's age-at-entry test moves with the period, +#: so the stage and the composition gate must agree at each of them. +_PERIODS = (2024, 2025, 2030) + +_HUMANITARIAN_STATUSES = frozenset( + immigration._HUMANITARIAN_STATUS_BY_CATEGORY.values() +) +_INDICATOR_POOL_STATUSES = frozenset({"REFUGEE", "ASYLEE", "DEPORTATION_WITHHELD"}) + +#: Every origin in the humanitarian codebook tables. +_HUMANITARIAN_ORIGIN_CODES: tuple[int, ...] = tuple( + sorted( + { + *( + code + for codes in immigration._PAROLE_ORIGIN_CODES.values() + for code in codes + ), + *immigration._REFUGEE_ORIGIN_CODES, + *immigration._ASYLEE_ORIGIN_CODES, + *( + code + for codes, _ in immigration._TPS_ORIGIN_CODES.values() + for code in codes + ), + } + ) +) +#: Birth countries the stage distinguishes: the United States, a generic +#: origin, Cuba/Haiti, and the humanitarian origins. +_BIRTH_CODES: tuple[int, ...] = tuple( + sorted( + {57, 120, 303, *immigration._CUBAN_HAITIAN_BIRTH_CODES} + | set(_HUMANITARIAN_ORIGIN_CODES) + ) +) + +#: ASEC-UA legal-status indicators as the module docstring lists them +#: ("Medicare/Medicaid/SSI/Social Security receipt, federal pensions, +#: IHS/CHAMPVA/military coverage, government employment, subsidized housing, +#: veteran status"), with the neutral value each column takes otherwise. +#: Pre-1982 arrival is drawn through ``PEINUSYR``. Naturalization-eligibility +#: is omitted: the code reads it from ``PRCITSHP == 4``, a citizen code, so it +#: can never move a ``PRCITSHP == 5`` row and no property here depends on it. +_NEUTRAL_INDICATORS: Mapping[str, float | int] = { + "MCARE": 2, + "CAID": 2, + "IHSFLG": 2, + "CHAMPVA": 2, + "MIL": 2, + "PEN_SC1": 0, + "PEN_SC2": 0, + "RESNSS1": 0, + "RESNSS2": 0, + "SS_YN": 2, + "SSI_YN": 2, + "PEIO1COW": 0, + "A_MJOCC": 0, + "PEAFEVER": 2, + "SPM_CAPHOUSESUB": 0.0, +} +_INDICATORS: tuple[tuple[str, float | int], ...] = ( + ("MCARE", 1), + ("CAID", 1), + ("IHSFLG", 1), + ("CHAMPVA", 1), + ("MIL", 1), + ("PEN_SC1", 3), + ("PEN_SC2", 3), + ("RESNSS1", 2), + ("RESNSS2", 2), + ("SS_YN", 1), + ("SSI_YN", 1), + ("PEIO1COW", 1), + ("PEIO1COW", 2), + ("PEIO1COW", 3), + ("A_MJOCC", 11), + ("PEAFEVER", 1), + ("SPM_CAPHOUSESUB", 250.0), +) + +#: ASEC arrival windows transcribed from the module's comments, not read from +#: its tables. Birth codes are the module's own (its comment says they were +#: "verified against Census CPS technical documentation"); the windows are the +#: comments' words. ``_RECENT_ARRIVAL_CODES``: "Operation Allies Welcome +#: (2021-2022 arrivals) draws from 27 and 28. U4U and CHNV (2022 to early +#: 2025) and refugees admitted in FY2023-FY2024 draw from 28 and 29, never +#: from 27"; ``_PAROLE_ORIGIN_CODES``: "Operation Allies Welcome +#: (Afghanistan), Uniting for Ukraine, and the non-Cuban/Haitian CHNV +#: nationalities". +_PAROLE_DOCUMENTED_WINDOWS: Mapping[int, frozenset[int]] = { + 200: frozenset({27, 28}), # Afghanistan (Operation Allies Welcome) + 164: frozenset({28, 29}), # Ukraine (Uniting for Ukraine) + 315: frozenset({28, 29}), # Nicaragua (CHNV) + 373: frozenset({28, 29}), # Venezuela (CHNV) +} +_REFUGEE_DOCUMENTED_WINDOW = frozenset({28, 29}) +#: ``_TPS_ORIGIN_CODES``: "Legacy designations bind hard (El Salvador +#: 2001-02-13 -> code 17; Honduras/Nicaragua 1998-12-30 -> code 16; Nepal +#: 2015-06-24 -> code 24)". +_TPS_DOCUMENTED_MAX_ARRIVAL_CODE: Mapping[int, int] = { + 312: 17, # El Salvador + 314: 16, # Honduras + 315: 16, # Nicaragua + 229: 24, # Nepal +} +#: ``_TPS_ORIGIN_CODES``: "Ukraine and Afghanistan TPS registrants are carried +#: by the parole draw ...; Haiti TPS is carried by CUBAN_HAITIAN_ENTRANT", and +#: Cuba/Haiti-born persons "are excluded from every humanitarian draw". +_TPS_DOCUMENTED_EXCLUDED_BIRTHS = frozenset({164, 200, 327, 332}) +#: ``_ASYLEE_ARRIVAL_CODES``: "grantees in 2022-2024 overwhelmingly arrived +#: 2016+"; the 2016-2017 bin (midpoint 2017) is code 25. Its comment and +#: ``_TPS_ORIGIN_CODES``'s both say "The 2026 ASEC's code 29 (2024-2026) is +#: left out". +_ASYLEE_DOCUMENTED_MIN_ARRIVAL_CODE = 25 +_ASYLEE_AND_TPS_DOCUMENTED_EXCLUDED_ARRIVAL_CODE = 29 + + +def _draw_labels() -> tuple[str, ...]: + labels: list[str] = [] + for category in immigration.HUMANITARIAN_STATUS_CATEGORIES: + origins = immigration._PER_ORIGIN_HUMANITARIAN_CATEGORIES.get(category) + if origins is None: + labels.append(category) + else: + labels.extend(f"{category}:{origin}" for origin in origins) + return tuple(labels) + + +_DRAW_LABELS = _draw_labels() + +#: One zero-target draw per manifest label, for evaluating candidate pools. +_TEMPLATE_DRAWS: tuple[immigration.HumanitarianDraw, ...] = tuple( + immigration.HumanitarianDraw( + category=label.split(":", 1)[0], + origin=label.split(":", 1)[1] if ":" in label else None, + status=immigration._HUMANITARIAN_STATUS_BY_CATEGORY[label.split(":", 1)[0]], + target=0.0, + source="https://example.org/template", + ) + for label in _DRAW_LABELS +) + + +def _tolerance(weights: np.ndarray) -> float: + return 1e-9 * max(1.0, float(np.abs(weights).sum())) + + +# --------------------------------------------------------------------------- +# _select_weight_to_target +# --------------------------------------------------------------------------- + +#: A few repeated draw values force the tie groups clones produce. +_TIED_DRAWS = (0.0, 0.125, 0.25, 0.5, 0.75, 0.875) + + +@st.composite +def _selection_cases(draw, *, integer_weights: bool = False): + n = draw(st.integers(0, 24)) + weight = ( + st.integers(0, 50).map(float) + if integer_weights + else st.floats(0.0, 1e6, allow_nan=False, allow_subnormal=False) + ) + weights = np.asarray( + draw(st.lists(weight, min_size=n, max_size=n)), dtype=np.float64 + ) + draws = np.asarray( + draw( + st.lists( + st.one_of( + st.sampled_from(_TIED_DRAWS), + st.floats(0.0, 1.0, exclude_max=True), + ), + min_size=n, + max_size=n, + ) + ), + dtype=np.float64, + ) + candidates = np.asarray( + draw(st.lists(st.booleans(), min_size=n, max_size=n)), dtype=bool + ) + total = float(weights[candidates].sum()) + amount = draw( + st.one_of( + st.floats(-10.0, 0.0), + st.floats(0.0, max(1.5 * total, 1.0)), + st.just(total), + ) + ) + return candidates, weights, draws, amount + + +def _reference_selection( + candidates: np.ndarray, weights: np.ndarray, draws: np.ndarray, amount: float +) -> np.ndarray: + """Walk distinct draw values upward; stop at the first group reaching it.""" + + if amount <= 0 or not candidates.any(): + return np.zeros(len(candidates), dtype=bool) + cumulative = 0.0 + for value in sorted(set(draws[candidates].tolist())): + cumulative += float(weights[candidates & (draws == value)].sum()) + if cumulative >= amount: + return candidates & (draws <= value) + return candidates.copy() + + +@_UNIT +@given(_selection_cases()) +def test_selection_is_a_draw_threshold_prefix_that_never_splits_ties(case) -> None: + """``_select_weight_to_target``: "every candidate at or below the draw + where the cumulative weight first reaches ``amount`` is selected. Records + sharing a draw ... land on the same side of the threshold" and "A + non-positive ``amount`` selects nothing".""" + + candidates, weights, draws, amount = case + selected = immigration._select_weight_to_target(candidates, weights, draws, amount) + assert selected.dtype == bool and selected.shape == candidates.shape + assert not (selected & ~candidates).any() + if amount <= 0 or not candidates.any(): + assert not selected.any() + return + threshold = draws[selected].max() + np.testing.assert_array_equal(selected, candidates & (draws <= threshold)) + + +@_UNIT +@given(_selection_cases()) +def test_selection_reaches_amount_and_overshoots_by_less_than_its_tie_group( + case, +) -> None: + """``_select_weight_to_target`` selects "~``amount`` of weighted mass": + the cumulative weight "first reaches ``amount``" at the threshold and "the + final tie group may overshoot the amount". When the candidates cannot + reach it, every candidate is selected.""" + + candidates, weights, draws, amount = case + selected = immigration._select_weight_to_target(candidates, weights, draws, amount) + if amount <= 0 or not candidates.any(): + return + tolerance = _tolerance(weights) + total = float(weights[candidates].sum()) + chosen = float(weights[selected].sum()) + if total < amount - tolerance: + event("amount infeasible") + np.testing.assert_array_equal(selected, candidates) + return + event("amount feasible") + tie = selected & (draws == draws[selected].max()) + assert chosen >= amount - tolerance + assert chosen - float(weights[tie].sum()) < amount + tolerance + + +@_UNIT +@given(_selection_cases(integer_weights=True)) +def test_selection_matches_an_independent_group_walk(case) -> None: + """Differential: the ``np.searchsorted`` implementation equals a plain + group walk over distinct draw values (integer weights keep both sums + exact, so the comparison is bitwise).""" + + candidates, weights, draws, amount = case + np.testing.assert_array_equal( + immigration._select_weight_to_target(candidates, weights, draws, amount), + _reference_selection(candidates, weights, draws, amount), + ) + + +@_UNIT +@given(_selection_cases(integer_weights=True), st.data()) +def test_selection_is_row_order_equivariant(case, data) -> None: + """Deduced: a threshold on draws with whole tie groups on one side depends + only on the (draw, weight) multiset, not on row positions. Integer weights + keep cumulative sums order-independent.""" + + candidates, weights, draws, amount = case + order = np.asarray( + data.draw(st.permutations(range(len(candidates)))), dtype=np.int64 + ) + selected = immigration._select_weight_to_target(candidates, weights, draws, amount) + permuted = immigration._select_weight_to_target( + candidates[order], weights[order], draws[order], amount + ) + np.testing.assert_array_equal(permuted, selected[order]) + + +# --------------------------------------------------------------------------- +# _stable_person_draws +# --------------------------------------------------------------------------- + + +@_UNIT +@given( + keys=st.lists(st.integers(0, 6), min_size=1, max_size=16), + seed=st.integers(0, 2**32), + salt=st.text(max_size=8), + data=st.data(), +) +def test_person_draws_are_keyed_by_identity_alone(keys, seed, salt, data) -> None: + """``_stable_person_draws``: "Deterministic uniform draws keyed by stable + person identity" that give "every clone of one source person the same + draw". Row-order equivariance is deduced from the per-key hash.""" + + person = pd.DataFrame({"person_id": keys}) + draws = immigration._stable_person_draws(person, seed=seed, salt=salt) + assert draws.shape == (len(keys),) + assert ((draws >= 0.0) & (draws < 1.0)).all() + key_array = np.asarray(keys) + for key in set(keys): + assert len(set(draws[key_array == key].tolist())) == 1 + np.testing.assert_array_equal( + draws, immigration._stable_person_draws(person, seed=seed, salt=salt) + ) + order = list(data.draw(st.permutations(range(len(keys))))) + reordered = person.iloc[order].reset_index(drop=True) + np.testing.assert_array_equal( + immigration._stable_person_draws(reordered, seed=seed, salt=salt), + draws[order], + ) + + +def _equal_draws_iff_equal_keys(draws: np.ndarray, keys: list[tuple]) -> None: + """Draws agree exactly for equal keys and differ for different ones (a + 64-bit blake2b collision between distinct keys has probability ~2**-64).""" + + for i in range(len(keys)): + for j in range(i + 1, len(keys)): + assert (draws[i] == draws[j]) == (keys[i] == keys[j]), (i, j) + + +@st.composite +def _lineage_rows(draw) -> list[tuple[int | None, int | None]]: + """(source_household_id, source_person_id) rows, each lineage alternative + about equally often: complete, household missing somewhere (legacy pair), + or source person missing somewhere (``person_id`` fallback). Small id + ranges make one source person recur across households.""" + + n = draw(st.integers(2, 10)) + ids = st.integers(1, 3) + rows = draw(st.lists(st.tuples(ids, ids), min_size=n, max_size=n)) + mode = draw(st.sampled_from(("full", "legacy", "person_id"))) + if mode == "full": + return rows + gaps = draw(st.sets(st.integers(0, n - 1), min_size=1)) + if mode == "legacy": + return [(None, p) if i in gaps else (h, p) for i, (h, p) in enumerate(rows)] + household_gaps = draw(st.booleans()) + return [ + ((None if household_gaps else h), None) if i in gaps else (h, p) + for i, (h, p) in enumerate(rows) + ] + + +@_UNIT +@given(rows=_lineage_rows(), seed=st.integers(0, 2**32)) +def test_person_draws_use_the_most_complete_lineage(rows, seed) -> None: + """``_stable_person_draws``: support clones key on ``source_year`` / + ``source_household_id`` / ``source_person_id``, and "frames without full + source ids fall back to the legacy source pair or ``person_id``"; with no + complete alternative it raises. Draws are equal exactly when the chosen + lineage is equal, so two rows sharing ``source_year`` and + ``source_person_id`` but not ``source_household_id`` draw differently + whenever the full lineage is complete (a draw that ignored the household + would tie them).""" + + household = [household for household, _ in rows] + source_person = [person for _, person in rows] + person = pd.DataFrame( + { + "source_year": [2024] * len(rows), + "source_household_id": pd.array(household, dtype="Int64"), + "source_person_id": pd.array(source_person, dtype="Int64"), + "person_id": np.arange(1, len(rows) + 1), + } + ) + + def draws(frame: pd.DataFrame) -> np.ndarray: + return immigration._stable_person_draws(frame, seed=seed, salt="lineage") + + full = draws(person) + if None not in source_person and None not in household: + event("full lineage complete") + if len(set(source_person)) < len( + set(zip(household, source_person, strict=True)) + ): + event("same source person in different households") + np.testing.assert_array_equal(full, draws(person.drop(columns="person_id"))) + _equal_draws_iff_equal_keys( + full, list(zip(household, source_person, strict=True)) + ) + elif None not in source_person: + event("legacy lineage") + np.testing.assert_array_equal( + full, draws(person.drop(columns="source_household_id")) + ) + _equal_draws_iff_equal_keys(full, [(key,) for key in source_person]) + else: + event("person_id fallback") + np.testing.assert_array_equal(full, draws(person[["person_id"]])) + _equal_draws_iff_equal_keys(full, [(key,) for key in range(len(rows))]) + with pytest.raises(SourceRuntimeError, match="lineage"): + draws(person.drop(columns="person_id")) + + +# --------------------------------------------------------------------------- +# _spill_pew_unauthorized_excess +# --------------------------------------------------------------------------- + +_MARKS = ("", "", *sorted(_HUMANITARIAN_STATUSES)) +#: Decimal weights whose binary sums leave rounding residue (0.1 + 0.2 is not +#: 0.3), the case the spill's closure tolerance exists for. +_DECIMAL_WEIGHTS = (0.1, 0.2, 0.3, 0.7, 1.1) +_WEIGHT = st.one_of( + st.just(0.0), + st.integers(1, 5_000).map(float), + st.sampled_from(_DECIMAL_WEIGHTS), + st.floats(0.5, 5e5, allow_nan=False, allow_subnormal=False), +) +#: Spill rows lean further toward decimal weights: the exact-closure residue +#: needs them on both sides of the tier boundary. +_SPILL_WEIGHT = st.one_of(st.sampled_from(_DECIMAL_WEIGHTS), _WEIGHT) +#: ``_spill_pew_unauthorized_excess``: "Treat anything within a billionth of +#: the in-scope weight as closed". +_DOCUMENTED_CLOSURE_TOLERANCE = 1e-9 + + +@dataclass(frozen=True) +class _SpillCase: + ssn: np.ndarray + marks: np.ndarray + weights: np.ndarray + noncitizens: np.ndarray + scope: np.ndarray + preserve: np.ndarray + retained: np.ndarray + target: float + seed: int + + +def _projection_of( + ssn: np.ndarray, marks: np.ndarray, retained: np.ndarray +) -> np.ndarray: + return immigration._pew_unauthorized_projection_mask( + ssn_codes=ssn, + humanitarian_marks=marks, + retained_ead_cohort=retained, + ) + + +@st.composite +def _spill_cases(draw) -> _SpillCase: + """Mostly residual in-scope non-citizens, with a target that is a share of + the in-scope Pew count (so the spill has work to do), exactly that count + less a subset of first-tier removable rows (the exact-closure case), or at + or above the count (so it must do nothing).""" + + n = draw(st.integers(2, 24), label="rows") + rows = draw( + st.lists( + st.tuples( + st.sampled_from((True, True, True, False)), + st.sampled_from((0, 0, 0, 2, 3)), + st.sampled_from(_MARKS), + st.sampled_from((True, True, False)), + st.booleans(), + st.sampled_from((False, False, False, True)), + _SPILL_WEIGHT, + ), + min_size=n, + max_size=n, + ) + ) + noncitizens = np.asarray([row[0] for row in rows], dtype=bool) + ssn = np.where(noncitizens, [row[1] for row in rows], 1).astype(np.int64) + # The stage never marks a residual (code 0) row: parole/TPS selections + # leave that pool when they are marked. + marks = np.where(noncitizens & (ssn != 0), [row[2] for row in rows], "").astype( + "U24" + ) + scope = np.asarray([row[3] for row in rows], dtype=bool) + preserve = np.asarray([row[4] for row in rows], dtype=bool) + retained = np.asarray([row[5] for row in rows], dtype=bool) + weights = np.asarray([row[6] for row in rows], dtype=np.float64) + initial = float(weights[scope & _projection_of(ssn, marks, retained)].sum()) + kind = draw( + st.sampled_from(("share", "exact", "exact", "above")), label="target kind" + ) + if kind == "share": + target = draw(st.floats(0.0, 1.0)) * initial + elif kind == "exact": + # Removing the whole first tier closes the gap exactly in real + # arithmetic, so the first tier's passes must take all of it; with + # decimal weights the recomputed sum then differs from the target by + # rounding residue alone. + first_tier = scope & ~preserve & noncitizens & (ssn == 0) & ~retained + subset = first_tier + if not draw(st.booleans(), label="whole first tier"): + subset = first_tier & np.asarray( + draw(st.lists(st.booleans(), min_size=n, max_size=n)), dtype=bool + ) + target = max(0.0, initial - float(weights[subset].sum())) + else: + target = initial * draw(st.floats(1.0, 1.5)) + event(f"spill target: {kind}") + return _SpillCase( + ssn=ssn, + marks=marks, + weights=weights, + noncitizens=noncitizens, + scope=scope, + preserve=preserve, + retained=retained, + target=target, + seed=draw(st.integers(0, 2**31 - 1)), + ) + + +def _projection(case: _SpillCase, ssn: np.ndarray) -> np.ndarray: + return _projection_of(ssn, case.marks, case.retained) + + +def _spill(case: _SpillCase) -> tuple[np.ndarray, np.ndarray]: + """Run the spill on copies; return (codes after, marks after).""" + + ssn = case.ssn.copy() + marks = case.marks.copy() + immigration._spill_pew_unauthorized_excess( + pd.DataFrame({"person_id": np.arange(1, len(ssn) + 1)}), + ssn, + marks, + case.weights.copy(), + noncitizens=case.noncitizens, + scope=case.scope, + preserve_scope=case.preserve, + retained_ead_cohort=case.retained, + target=case.target, + seed=case.seed, + salt="immigration:property", + ) + return ssn, marks + + +@_UNIT +@given(_spill_cases()) +def test_spill_moves_only_in_scope_residual_noncitizens_and_only_to_ead( + case, +) -> None: + """``_spill_pew_unauthorized_excess`` "Spill[s] a broad Pew-universe margin + to EAD": candidates are ``(ssn_codes == 0) & noncitizens & scope`` and the + only write is ``ssn_codes[selected] = 2``. It is deterministic.""" + + after, marks = _spill(case) + changed = after != case.ssn + if changed.any(): + event("the spill moved rows") + assert not (changed & ~((case.ssn == 0) & case.noncitizens & case.scope)).any() + assert (after[changed] == 2).all() + np.testing.assert_array_equal(marks, case.marks) + np.testing.assert_array_equal(_spill(case)[0], after) + + +@_UNIT +@given(_spill_cases()) +def test_spill_never_grows_the_pew_universe(case) -> None: + """The spill exists to reduce a Pew-universe excess: moving a residual row + to EAD either removes it from ``_pew_unauthorized_projection_mask`` or, + for "DACA and residual Cuban/Haitian cohort rows", leaves it counted. + Membership therefore only shrinks, in scope and everywhere else, so the + weighted Pew count never rises.""" + + after, _ = _spill(case) + before_mask = _projection(case, case.ssn) + after_mask = _projection(case, after) + assert not (after_mask & ~before_mask).any() + + +@_UNIT +@given(_spill_cases()) +def test_spill_leaves_a_margin_within_tolerance_of_its_control_untouched( + case, +) -> None: + """``_select_weight_to_target``: "a count already at or below its control + spills nothing"; the spill returns while ``current_excess() <= + closed_within``, and its comment says "Treat anything within a billionth + of the in-scope weight as closed".""" + + included = _projection(case, case.ssn) + excess = float(case.weights[case.scope & included].sum()) - case.target + closed_within = _DOCUMENTED_CLOSURE_TOLERANCE * max( + 1.0, float(case.weights[case.scope].sum()) + ) + if excess > closed_within: + return + event("margin already closed") + after, _ = _spill(case) + np.testing.assert_array_equal(after, case.ssn) + + +@_UNIT +@given(_spill_cases()) +def test_spill_closes_every_closable_excess_and_otherwise_exhausts_the_pool( + case, +) -> None: + """The workers/students controls are enforced "until the broad reported + universes match their controls" (module docstring, step 3): each pass + selects the current excess, and the corrective draw "spills rows outside + those retained cohorts". So the in-scope Pew count ends at or below the + target whenever removing every in-scope residual non-retained row could + get it there, and otherwise every such row has been spilled.""" + + tolerance = _tolerance(case.weights) + after, _ = _spill(case) + initial = float(case.weights[case.scope & _projection(case, case.ssn)].sum()) + final = float(case.weights[case.scope & _projection(case, after)].sum()) + removable = case.scope & case.noncitizens & (case.ssn == 0) & ~case.retained + floor = initial - float(case.weights[removable].sum()) + if floor <= case.target - tolerance: + event("excess closable") + assert final <= case.target + tolerance + elif floor > case.target + tolerance: + event("excess not closable") + assert (after[removable] == 2).all() + + +@_UNIT +@given(_spill_cases()) +def test_spill_undershoots_its_control_by_less_than_one_spilled_row(case) -> None: + """Each pass asks ``_select_weight_to_target`` for exactly the current + excess, which "may overshoot the amount" only by its final tie group + (one row here: every row has its own ``person_id`` draw). The in-scope + Pew count therefore never lands a full spilled row below the target.""" + + after, _ = _spill(case) + changed = after != case.ssn + if not changed.any(): + return + final = float(case.weights[case.scope & _projection(case, after)].sum()) + largest = float(case.weights[changed].max()) + assert final > case.target - largest - _tolerance(case.weights) + + +#: The minimal exact-closure case: in real arithmetic the student control 0.3 +#: equals the two worker-students' 0.1 + 0.2, so spilling the one non-worker +#: student (weight 1.0) closes the gap, yet ``0.1 + 0.2 - 0.3`` is 5.6e-17 in +#: binary floating point. ``test_us_immigration.py`` pins the same case end +#: to end through the stage. +_EXACT_CLOSURE_RESIDUE_CASE = _SpillCase( + ssn=np.zeros(3, dtype=np.int64), + marks=np.full(3, "", dtype="U24"), + weights=np.asarray([1.0, 0.1, 0.2]), + noncitizens=np.ones(3, dtype=bool), + scope=np.ones(3, dtype=bool), + preserve=np.asarray([False, True, True]), + retained=np.zeros(3, dtype=bool), + target=0.3, + seed=0, +) + + +@_UNIT +@given(_spill_cases()) +@example(_EXACT_CLOSURE_RESIDUE_CASE) +def test_spill_exhausts_rows_outside_the_other_margin_first(case) -> None: + """``_spill_pew_unauthorized_excess``: "Rows outside the other controlled + margin are exhausted before any overlapping row is touched" and "a row + inside the other margin is spilled only once no row outside it can close + the gap"; the closure tolerance ("anything within a billionth of the + in-scope weight") exists so that rounding residue "never opens the next + tier". If any row inside the preserved margin moved, every removable + (non-retained residual) in-scope row outside it moved too, and removing + all of them still left more than rounding residue above the target.""" + + after, _ = _spill(case) + changed = after != case.ssn + if not (changed & case.preserve).any(): + return + event("a row inside the other margin moved") + outside = ( + case.scope + & ~case.preserve + & case.noncitizens + & (case.ssn == 0) + & ~case.retained + ) + assert (after[outside] == 2).all() + initial = float(case.weights[case.scope & _projection(case, case.ssn)].sum()) + closed_within = _DOCUMENTED_CLOSURE_TOLERANCE * max( + 1.0, float(case.weights[case.scope].sum()) + ) + left_after_outside = initial - float(case.weights[outside].sum()) - case.target + assert left_after_outside > 0.5 * closed_within + + +# --------------------------------------------------------------------------- +# The stage +# --------------------------------------------------------------------------- + + +def _generic_asec_row(weight): + return st.fixed_dictionaries( + { + "PRCITSHP": st.sampled_from((1, 2, 3, 4, 5, 5, 5, 5, 5)), + # Biased toward the origins and recent arrivals the humanitarian + # draws need. + "PENATVTY": st.one_of( + st.sampled_from(_BIRTH_CODES), + st.sampled_from(_HUMANITARIAN_ORIGIN_CODES), + ), + "PEINUSYR": st.one_of( + st.integers(0, 29), st.integers(17, 29), st.integers(24, 29) + ), + "A_AGE": st.integers(0, 85), + "A_MARITL": st.sampled_from((1, 2, 4, 7)), + "A_SPOUSE": st.sampled_from((0, 1)), + "A_HSCOL": st.sampled_from((0, 1, 2)), + "A_LFSR": st.sampled_from(immigration._CPS_LABOR_FORCE_STATUS_DOMAIN), + "indicator": st.one_of(st.none(), st.none(), st.sampled_from(_INDICATORS)), + "person_weight": weight, + } + ) + + +@st.composite +def _humanitarian_profile_row(draw, weight) -> dict: + """A non-citizen shaped for one humanitarian draw or for the DACA cohort, + or one arrival bin outside its window (so windows are exercised from both + sides). + + Inputs only: the origin and arrival tables here come from the module so + the candidate pools are reachable; no oracle reads them. + """ + + category = draw( + st.sampled_from((*immigration.HUMANITARIAN_STATUS_CATEGORIES, "daca")) + ) + age = draw(st.one_of(st.integers(25, 60), st.integers(0, 85))) + if category == "daca": + # Arrived by 2007 (module docstring, step 6) at an age measured at + # 2024: under 16 (the cohort at every period drawn) or 16-21 (outside + # it at 2024, inside it by 2030, so cohort membership depends on the + # period). Sometimes Cuba/Haiti-born, where DACA and the entrant + # class compete. + birth = draw(st.sampled_from((303, 312, 327, 332))) + arrival = draw(st.integers(8, 20)) + midpoint = immigration._ARRIVAL_YEAR_MIDPOINTS[arrival] + entry_age_at_2024 = draw(st.one_of(st.integers(0, 15), st.integers(16, 21))) + age = max(15, (_TIME_PERIOD - midpoint) + entry_age_at_2024) + elif category == "paroled_one_year": + origin = draw(st.sampled_from(tuple(immigration._PAROLE_ORIGIN_CODES))) + birth = draw(st.sampled_from(immigration._PAROLE_ORIGIN_CODES[origin])) + arrival = draw(st.sampled_from(immigration._PAROLE_ASEC_ARRIVAL_CODES[origin])) + elif category == "refugee": + birth = draw(st.sampled_from(immigration._REFUGEE_ORIGIN_CODES)) + arrival = draw(st.sampled_from(immigration._POST_2021_ARRIVAL_CODES)) + elif category == "asylee": + birth = draw(st.sampled_from(immigration._ASYLEE_ORIGIN_CODES)) + arrival = draw(st.sampled_from(immigration._ASYLEE_ARRIVAL_CODES)) + elif category == "deportation_withheld": + birth = draw(st.sampled_from(_BIRTH_CODES)) + arrival = draw(st.integers(1, immigration._WITHHELD_MAX_ARRIVAL_CODE)) + else: + codes, latest = draw( + st.sampled_from(tuple(immigration._TPS_ORIGIN_CODES.values())) + ) + birth = draw(st.sampled_from(codes)) + arrival = draw(st.integers(1, latest)) + if draw(st.integers(0, 4)) == 0: + arrival = min(29, max(1, arrival + draw(st.sampled_from((-1, 1))))) + return { + "PRCITSHP": draw(st.sampled_from((5, 5, 5, 4))), + "PENATVTY": birth, + "PEINUSYR": arrival, + "A_AGE": age, + "A_MARITL": draw(st.sampled_from((1, 2, 4, 7))), + "A_SPOUSE": draw(st.sampled_from((0, 1))), + "A_HSCOL": draw(st.sampled_from((0, 1, 2))), + "A_LFSR": draw(st.sampled_from(immigration._CPS_LABOR_FORCE_STATUS_DOMAIN)), + # Refugee, asylee and withholding draws need an indicator; parole and + # TPS draw from either pool; a DACA-cohort row needs the residual + # pool to reach an EAD. + "indicator": draw( + st.none() + if category == "daca" + else st.one_of( + st.none(), st.sampled_from(_INDICATORS), st.sampled_from(_INDICATORS) + ) + ), + "person_weight": draw(weight), + } + + +def _asec_row(weight): + return st.one_of( + _generic_asec_row(weight), + _generic_asec_row(weight), + _humanitarian_profile_row(weight), + ) + + +def _asec_table(rows: list[dict]) -> pd.DataFrame: + records = [] + for index, row in enumerate(rows): + record = {**_NEUTRAL_INDICATORS, **row, "person_id": index + 1} + indicator = record.pop("indicator") + if indicator is not None: + column, value = indicator + record[column] = value + records.append(record) + return pd.DataFrame(records) + + +def _manifest_parameters( + *, + workers: float, + students: float, + anchor: float, + targets: Mapping[str, float], +) -> dict[str, object]: + """``derive_immigration_status`` parameters in the packaged manifest shape.""" + + humanitarian: dict[str, object] = {} + for category in immigration.HUMANITARIAN_STATUS_CATEGORIES: + origins = immigration._PER_ORIGIN_HUMANITARIAN_CATEGORIES.get(category) + if origins is None: + humanitarian[category] = { + "target": targets[category], + "source": f"https://example.org/{category}", + } + else: + humanitarian[category] = { + origin: { + "target": targets[f"{category}:{origin}"], + "source": f"https://example.org/{category}/{origin}", + } + for origin in origins + } + return { + "seed_from_build_config": True, + "time_period_from_build_config": True, + "undocumented_workers": { + "target": workers, + "source": "https://example.org/workers", + }, + "undocumented_students": { + "target": students, + "source": "https://example.org/students", + }, + "undocumented_population_anchor": { + "value": anchor, + "source": "https://example.org/population", + }, + "humanitarian_status_stocks": humanitarian, + } + + +@dataclass(frozen=True) +class _StageCase: + person: pd.DataFrame + workers: float + students: float + anchor: float + targets: Mapping[str, float] + seed: int + time_period: int + + @property + def parameters(self) -> dict[str, object]: + return _manifest_parameters( + workers=self.workers, + students=self.students, + anchor=self.anchor, + targets=self.targets, + ) + + @property + def controls(self) -> immigration.ImmigrationControls: + return immigration._controls_from_parameters(self.parameters) + + +def _indicator_mask(person: pd.DataFrame) -> np.ndarray: + """Rows carrying a documented indicator (see ``_INDICATORS``).""" + + indicator = np.isin(person["PEINUSYR"].to_numpy(), range(1, 8)) + for name, value in _INDICATORS: + indicator |= person[name].to_numpy() == value + return indicator + + +def _scopes(person: pd.DataFrame) -> dict[str, np.ndarray]: + """The two controlled margins as step 3 defines them: workers "measured + with ASEC ``A_LFSR`` at age 16+" and students.""" + + return { + "workers": (person["A_AGE"].to_numpy() >= 16) + & np.isin(person["A_LFSR"].to_numpy(), (1, 2, 3, 4)), + "students": person["A_HSCOL"].to_numpy() == 2, + } + + +def _residual_masses(person: pd.DataFrame) -> dict[str, float]: + """Each margin's pre-stage residual mass (indicator-free non-citizens). + + The stage's pre-spill Pew count in a margin is at least this: residual + rows either stay residual or, if a parole/TPS draw takes them, keep a + Pew-included mark. A control below it therefore makes the spill work. + """ + + residual = (person["PRCITSHP"].to_numpy() == 5) & ~_indicator_mask(person) + weights = person["person_weight"].to_numpy(dtype=np.float64) + return { + name: float(weights[residual & scope].sum()) + for name, scope in _scopes(person).items() + } + + +def _pool_masses(person: pd.DataFrame, time_period: int) -> dict[str, float]: + """Each draw's weighted candidate pool before any draw runs. + + Scaling targets to these pools exercises both sides of every draw (a pool + that suffices and one that runs short) instead of mostly saturating. + """ + + noncitizen = person["PRCITSHP"].to_numpy() == 5 + codes = np.where(noncitizen, np.where(_indicator_mask(person), 3, 0), 1) + profile = immigration._source_aware_immigration_profile( + person, time_period=time_period + ) + weights = person["person_weight"].to_numpy(dtype=np.float64) + return { + template.label: float( + weights[ + immigration._humanitarian_draw_candidates( + template, ssn_codes=codes, profile=profile + ) + ].sum() + ) + for template in _TEMPLATE_DRAWS + } + + +@st.composite +def _target(draw, pool: float, scale: float) -> float: + """Zero, a share of the draw's own pool, or occasionally a pool-blind stock.""" + + if draw(st.integers(0, 9)) == 0: + return draw(st.floats(1e-3, 0.1)) * scale + return draw(st.one_of(st.just(0.0), st.floats(0.01, 1.5))) * pool + + +@st.composite +def _control(draw, mass: float, scale: float) -> float: + """A worker/student control as a share of the margin's residual mass: + below it (the spill must act), near it, or above it (it may not).""" + + share = draw( + st.one_of(st.floats(0.01, 0.99), st.floats(0.99, 1.01), st.floats(1.01, 2.0)) + ) + return share * (mass if mass > 0 else scale) + + +@st.composite +def _stage_cases( + draw, + *, + integer_weights: bool = False, + min_size: int = 6, + max_size: int = 32, + time_periods: tuple[int, ...] = _PERIODS, +) -> _StageCase: + weight = st.integers(0, 2_000).map(float) if integer_weights else _WEIGHT + n = draw(st.integers(min_size, max_size), label="rows") + rows = draw(st.lists(_asec_row(weight), min_size=n, max_size=n)) + person = _asec_table(rows) + time_period = draw(st.sampled_from(time_periods), label="time period") + scale = max(1.0, float(person["person_weight"].sum())) + pools = _pool_masses(person, time_period) + residual = _residual_masses(person) + return _StageCase( + person=person, + workers=draw(_control(residual["workers"], scale), label="workers"), + students=draw(_control(residual["students"], scale), label="students"), + anchor=draw(st.floats(0.1, 2.0)) * scale, + targets={label: draw(_target(pools[label], scale)) for label in _DRAW_LABELS}, + seed=draw(st.integers(0, 2**31 - 1)), + time_period=time_period, + ) + + +def _derive( + person: pd.DataFrame, + parameters: Mapping[str, object], + *, + seed: int, + time_period: int, +) -> pd.DataFrame: + operation = SourceOperationSpec.from_mapping( + {"kind": "derive_immigration_status", **parameters} + ) + context = SourceRuntimeContext( + config=SourceRuntimeConfig(seed=seed, target_year=time_period), + tables={}, + ) + return immigration.derive_us_immigration_status_from_manifest( + person, operation, context + ) + + +def _run(case: _StageCase) -> pd.DataFrame: + return _derive( + case.person, + case.parameters, + seed=case.seed, + time_period=case.time_period, + ) + + +@dataclass(frozen=True) +class _Facts: + """Evidence recomputed from the raw columns and the documented rules.""" + + weights: np.ndarray + noncitizen: np.ndarray + ssn: np.ndarray + status: np.ndarray + humanitarian: np.ndarray + cuban_haitian_born: np.ndarray + cuban_haitian_class: np.ndarray + daca_cohort: np.ndarray + indicator: np.ndarray + worker: np.ndarray + student: np.ndarray + + @property + def retained(self) -> np.ndarray: + """The spill's retained EAD cohort: DACA plus the Cuban/Haitian class + outside it (module docstring, step 3: "Residual Cuban/Haitian rows + that receive an EAD also stay in that universe").""" + + return self.daca_cohort | (self.cuban_haitian_class & ~self.daca_cohort) + + +def _facts(person: pd.DataFrame, output: pd.DataFrame, time_period: int) -> _Facts: + def column(name: str) -> np.ndarray: + return person[name].to_numpy() + + arrival_code = column("PEINUSYR").astype(np.int64) + arrival_year = np.asarray( + [ + immigration._ARRIVAL_YEAR_MIDPOINTS.get(int(code), time_period) + for code in arrival_code + ], + dtype=np.int64, + ) + age = column("A_AGE").astype(np.int64) + age_at_entry = np.maximum(0, age - (time_period - arrival_year)) + # "DACA applies the statutory cohort test (arrived by 2007 before age 16, + # aged 15+)" (module docstring, step 6). + daca_cohort = (arrival_year <= 2007) & (age_at_entry < 16) & (age >= 15) + cuban_haitian_born = np.isin( + column("PENATVTY"), immigration._CUBAN_HAITIAN_BIRTH_CODES + ) + status = output["immigration_status_str"].astype(str).to_numpy() + scopes = _scopes(person) + return _Facts( + weights=person["person_weight"].to_numpy(dtype=np.float64), + noncitizen=column("PRCITSHP") == 5, + ssn=output["ssn_card_type"].astype(str).to_numpy(), + status=status, + humanitarian=np.isin(status, list(_HUMANITARIAN_STATUSES)), + cuban_haitian_born=cuban_haitian_born, + # "the nationality-plus-arrival class" (module docstring, step 6). + cuban_haitian_class=cuban_haitian_born + & (arrival_year >= immigration._CUBAN_HAITIAN_ARRIVAL_CUTOFF), + daca_cohort=daca_cohort, + indicator=_indicator_mask(person), + worker=scopes["workers"], + student=scopes["students"], + ) + + +def _frame(person: pd.DataFrame, weights: np.ndarray) -> Frame: + """One person per household, weighted at the household.""" + + person = person.drop(columns=["person_weight"], errors="ignore").copy() + ids = np.arange(1, len(person) + 1, dtype=np.int64) + person = person.reset_index(drop=True) + for entity in US_SCHEMA.group_entities: + person[US_SCHEMA.membership_column(entity)] = ids + tables = { + "person": person, + **{ + entity: pd.DataFrame({US_SCHEMA.id_column(entity): ids}) + for entity in US_SCHEMA.group_entities + }, + } + return Frame( + tables, + US_SCHEMA, + { + "household": Weights( + np.asarray(weights, dtype=np.float64), WeightKind.DESIGN + ) + }, + ) + + +def _codes(ssn: np.ndarray) -> np.ndarray: + """Engine SSN names to the stage's integer codes, via the module's map.""" + + return immigration._ssn_name_codes(pd.DataFrame({"ssn_card_type": ssn})) + + +def _pre_draw_codes(ssn: np.ndarray) -> np.ndarray: + """SSN codes as the humanitarian draws saw them. + + Indicators set code 3 and nothing later changes it; the draws and the + spill only move code 0 to code 2. So every final EAD row was residual when + the draws ran. + """ + + return np.select( + [ssn == "CITIZEN", ssn == "OTHER_NON_CITIZEN"], [1, 3], default=0 + ).astype(np.int64) + + +@_STAGE +@given(_stage_cases()) +def test_stage_pairs_undocumented_with_no_ssn_and_citizens_with_citizenship( + case, +) -> None: + """Module docstring: "``UNDOCUMENTED`` enum remains exactly paired with + ``ssn_card_type=NONE``"; "Citizenship is measured, not imputed. + ``PRCITSHP`` in {1..4} → citizens"; "``CONDITIONAL_ENTRANT`` is + deliberately not emitted"; and "The stage writes two PolicyEngine-US + person input columns" (the source columns pass through unchanged).""" + + output = _run(case) + assert list(output.columns) == [ + *case.person.columns, + *immigration.US_IMMIGRATION_OUTPUT_COLUMNS, + ] + pd.testing.assert_frame_equal(output[list(case.person.columns)], case.person) + ssn = output["ssn_card_type"].to_numpy() + status = output["immigration_status_str"].to_numpy() + assert set(ssn) <= set(immigration.SSN_CARD_TYPE_VALUES) + assert set(status) <= set(immigration.IMMIGRATION_STATUS_VALUES) + assert "CONDITIONAL_ENTRANT" not in set(status) + np.testing.assert_array_equal(ssn == "NONE", status == "UNDOCUMENTED") + citizen = np.isin(case.person["PRCITSHP"].to_numpy(), (1, 2, 3, 4)) + np.testing.assert_array_equal(ssn == "CITIZEN", citizen) + np.testing.assert_array_equal(status == "CITIZEN", citizen) + if (ssn == "NONE").any() and (ssn == "NON_CITIZEN_VALID_EAD").any(): + event("output carries both NONE and EAD rows") + + +@_STAGE +@given(_stage_cases()) +def test_stage_status_is_the_documented_function_of_ssn_marks_and_evidence( + case, +) -> None: + """Reference implementation of step 6 and ``_derive_immigration_status``: + citizens are ``CITIZEN``, ``NONE`` is ``UNDOCUMENTED``, a humanitarian + mark wins, then "``DACA`` applies the statutory cohort test ... to EAD + holders; ``CUBAN_HAITIAN_ENTRANT`` applies the nationality-plus-arrival + class to documented non-citizens; every other documented non-citizen stays + ``LEGAL_PERMANENT_RESIDENT``". DACA outranks the Cuban/Haitian class only + for EAD holders (``_special_status_masks``: "a qualifying EAD holder in + both cohorts retains DACA"), so an indicator-documented Cuba-born member + of the DACA cohort is a Cuban/Haitian entrant. A mark never hides one of + those tags: "Humanitarian draws exclude Cuba/Haiti-born and the DACA + cohort, so the marks are disjoint from both tags".""" + + output = _run(case) + facts = _facts(case.person, output, case.time_period) + documented = np.isin(facts.ssn, ("NON_CITIZEN_VALID_EAD", "OTHER_NON_CITIZEN")) + daca = (facts.ssn == "NON_CITIZEN_VALID_EAD") & facts.daca_cohort + cuban_haitian = documented & facts.cuban_haitian_class & ~daca + if daca.any(): + event("output carries DACA rows") + if cuban_haitian.any(): + event("output carries Cuban/Haitian entrants") + expected = np.select( + [ + facts.ssn == "CITIZEN", + facts.ssn == "NONE", + facts.humanitarian, + daca, + cuban_haitian, + ], + ["CITIZEN", "UNDOCUMENTED", facts.status, "DACA", "CUBAN_HAITIAN_ENTRANT"], + default="LEGAL_PERMANENT_RESIDENT", + ) + np.testing.assert_array_equal(facts.status, expected) + assert not (facts.humanitarian & (daca | cuban_haitian)).any() + + +@_STAGE +@given(_stage_cases()) +def test_indicator_pool_is_exactly_the_noncitizens_with_a_documented_indicator( + case, +) -> None: + """Module docstring, step 2: "Non-citizens with any indicator of authorized + status — pre-1982 IRCA-cohort arrival, ... — move to + ``OTHER_NON_CITIZEN``", and nothing later moves them (the draws and the + spill only rewrite code 0). No other row lands there. (The docstring also + lists naturalization-eligibility, which the code cannot fire for a + non-citizen; see ``_NEUTRAL_INDICATORS``.)""" + + output = _run(case) + facts = _facts(case.person, output, case.time_period) + np.testing.assert_array_equal( + facts.ssn == "OTHER_NON_CITIZEN", facts.noncitizen & facts.indicator + ) + + +@_STAGE +@given(_stage_cases()) +def test_humanitarian_labels_respect_exclusions_pools_and_draw_masks(case) -> None: + """Module docstring, step 5: "Cuba/Haiti-born persons are excluded from + every humanitarian draw ..., as is the DACA statutory cohort"; draws are + "mutually exclusive"; "Draws from the residual pool (TPS and parole only) + flip ``ssn_card_type`` ``NONE`` to ``NON_CITIZEN_VALID_EAD``"; + ``_humanitarian_draw_candidates``: "``REFUGEE``/``ASYLEE``/ + ``DEPORTATION_WITHHELD`` draw from the indicator-documented pool only". + The public ``us_immigration_humanitarian_draw_mask`` partitions the + humanitarian rows by draw, and an explicit zero target emits nothing + (``HumanitarianDraw``: "Zero ... means the category is explicitly not + imputed").""" + + output = _run(case) + facts = _facts(case.person, output, case.time_period) + if facts.humanitarian.any(): + event("output carries humanitarian rows") + assert not (facts.humanitarian & ~facts.noncitizen).any() + assert not (facts.humanitarian & facts.cuban_haitian_born).any() + assert not (facts.humanitarian & facts.daca_cohort).any() + indicator_pool = np.isin(facts.status, list(_INDICATOR_POOL_STATUSES)) + assert (facts.ssn[indicator_pool] == "OTHER_NON_CITIZEN").all() + temporary = facts.humanitarian & ~indicator_pool + if (temporary & (facts.ssn == "NON_CITIZEN_VALID_EAD")).any(): + event("a residual row drew parole/TPS") + assert np.isin( + facts.ssn[temporary], ("OTHER_NON_CITIZEN", "NON_CITIZEN_VALID_EAD") + ).all() + + frame = _frame(output, np.ones(len(output))) + covered = np.zeros(len(output), dtype=int) + for draw in case.controls.humanitarian: + mask = immigration.us_immigration_humanitarian_draw_mask( + frame, draw, time_period=case.time_period + ) + if draw.target <= 0: + assert not mask.any(), draw.label + covered += mask + assert (covered <= 1).all() + np.testing.assert_array_equal(covered == 1, facts.humanitarian) + + +@_STAGE +@given(_stage_cases()) +def test_humanitarian_rows_sit_inside_the_asec_windows_the_comments_state( + case, +) -> None: + """Independent of ``_humanitarian_draw_candidates``: every emitted parole, + TPS and asylee row satisfies the ASEC origin/arrival windows transcribed + above from the module's comments (``_PAROLE_DOCUMENTED_WINDOWS``, + ``_REFUGEE_DOCUMENTED_WINDOW``, ``_TPS_DOCUMENTED_MAX_ARRIVAL_CODE``, + ``_TPS_DOCUMENTED_EXCLUDED_BIRTHS``, + ``_ASYLEE_DOCUMENTED_MIN_ARRIVAL_CODE``, + ``_ASYLEE_AND_TPS_DOCUMENTED_EXCLUDED_ARRIVAL_CODE``). The withholding + window has no numeric statement in a comment and is not checked here.""" + + output = _run(case) + status = output["immigration_status_str"].to_numpy() + birth = case.person["PENATVTY"].to_numpy() + arrival = case.person["PEINUSYR"].to_numpy() + + parole = status == "PAROLED_ONE_YEAR" + assert set(birth[parole].tolist()) <= set(_PAROLE_DOCUMENTED_WINDOWS) + for origin, window in _PAROLE_DOCUMENTED_WINDOWS.items(): + rows = parole & (birth == origin) + if rows.any(): + event("parole rows emitted") + if ((birth == origin) & ~np.isin(arrival, list(window))).any(): + event("a parole-origin row sits outside its window") + assert np.isin(arrival[rows], list(window)).all(), origin + + refugee = status == "REFUGEE" + if refugee.any(): + event("refugee rows emitted") + assert np.isin(arrival[refugee], list(_REFUGEE_DOCUMENTED_WINDOW)).all() + + tps = status == "TPS" + assert not np.isin(birth[tps], list(_TPS_DOCUMENTED_EXCLUDED_BIRTHS)).any() + assert not (arrival[tps] == _ASYLEE_AND_TPS_DOCUMENTED_EXCLUDED_ARRIVAL_CODE).any() + assert (arrival[tps] >= 1).all() + for origin, latest in _TPS_DOCUMENTED_MAX_ARRIVAL_CODE.items(): + rows = tps & (birth == origin) + if rows.any(): + event("legacy-designation TPS rows emitted") + if ((birth == origin) & (arrival > latest)).any(): + event("a legacy TPS-origin row arrived after its cutoff") + assert (arrival[rows] <= latest).all(), origin + + asylee = status == "ASYLEE" + if asylee.any(): + event("asylee rows emitted") + assert (arrival[asylee] >= _ASYLEE_DOCUMENTED_MIN_ARRIVAL_CODE).all() + assert not ( + arrival[asylee] == _ASYLEE_AND_TPS_DOCUMENTED_EXCLUDED_ARRIVAL_CODE + ).any() + + +@_STAGE +@given(_stage_cases()) +def test_each_humanitarian_draw_reaches_its_target_or_exhausts_its_pool( + case, +) -> None: + """Step 5 draws each status "to a manifest-cited weighted stock target", + "in that order", and "a person two cohorts could claim takes the earlier + status" (``_assign_humanitarian_statuses``: "a person marked by an earlier + draw is out of every later candidate pool"). The band constant adds: "The + draw forces the target when candidates suffice". Replaying the candidate + pools in draw order, each draw's rows reach its target and overshoot it by + less than the last-drawn row, or take the whole pool when it is short; a + zero target takes nothing; and the draws account for every humanitarian + row. The pools come from the module's ``_humanitarian_draw_candidates``, + so this checks selection against target, not the windows themselves.""" + + output = _run(case) + facts = _facts(case.person, output, case.time_period) + tolerance = _tolerance(facts.weights) + profile = immigration._source_aware_immigration_profile( + case.person, time_period=case.time_period + ) + pre_draw = _pre_draw_codes(facts.ssn) + marked = np.zeros(len(output), dtype=bool) + for draw in case.controls.humanitarian: + pool = ( + immigration._humanitarian_draw_candidates( + draw, ssn_codes=pre_draw, profile=profile + ) + & ~marked + ) + chosen = pool & (facts.status == draw.status) + weight = float(facts.weights[chosen].sum()) + available = float(facts.weights[pool].sum()) + if draw.target <= 0: + assert not chosen.any(), draw.label + elif available >= draw.target + tolerance: + event("a draw's pool sufficed") + assert weight >= draw.target - tolerance, draw.label + draws = immigration._stable_person_draws( + case.person, seed=case.seed, salt=draw.salt + ) + last = chosen & (draws == draws[chosen].max()) + assert ( + weight - float(facts.weights[last].sum()) < draw.target + tolerance + ), draw.label + elif available < draw.target - tolerance: + event("a draw's pool was short") + np.testing.assert_array_equal(chosen, pool, err_msg=draw.label) + marked |= chosen + np.testing.assert_array_equal(marked, facts.humanitarian) + + +@_STAGE +@given(_stage_cases(), st.data()) +def test_humanitarian_draws_are_sequential_and_precede_the_spill(case, data) -> None: + """``HUMANITARIAN_STATUS_CATEGORIES``: "Draws are sequential and mutually + exclusive"; ``_assign_humanitarian_statuses`` "Runs after the legal-status + indicators and before the EAD worker/student spill". So retargeting every + category from some point on, and changing the worker and student controls, + leaves the rows of every earlier category unchanged.""" + + categories = immigration.HUMANITARIAN_STATUS_CATEGORIES + cut = data.draw(st.integers(0, len(categories)), label="first retargeted") + scale = max(1.0, float(case.person["person_weight"].sum())) + pools = _pool_masses(case.person, case.time_period) + residual = _residual_masses(case.person) + later = set(categories[cut:]) + targets = { + label: ( + data.draw(_target(pools[label], scale), label=label) + if label.split(":", 1)[0] in later + else target + ) + for label, target in case.targets.items() + } + changed = replace( + case, + workers=data.draw(_control(residual["workers"], scale), label="workers"), + students=data.draw(_control(residual["students"], scale), label="students"), + targets=targets, + ) + first = _run(case)["immigration_status_str"].to_numpy() + second = _run(changed)["immigration_status_str"].to_numpy() + for category in categories[:cut]: + status = immigration._HUMANITARIAN_STATUS_BY_CATEGORY[category] + if (first == status).any(): + event("an earlier category emitted rows") + np.testing.assert_array_equal( + first == status, second == status, err_msg=category + ) + + +@_STAGE +@given(_stage_cases(integer_weights=True), st.data()) +def test_stage_is_bit_reproducible_and_row_order_equivariant(case, data) -> None: + """Module docstring: "reruns are bit-reproducible without global RNG + state". Row-order equivariance is deduced: draws are "keyed by the + person's stable source identity" and every selection is a draw threshold + (integer weights keep the weighted sums order-independent).""" + + output = _run(case) + pd.testing.assert_frame_equal(_run(case), output) + order = list(data.draw(st.permutations(range(len(case.person))))) + shuffled = replace(case, person=case.person.iloc[order].reset_index(drop=True)) + pd.testing.assert_frame_equal( + _run(shuffled), + output.iloc[order].reset_index(drop=True), + ) + + +@_STAGE +@given(_stage_cases(max_size=16), st.data()) +def test_support_clones_share_their_source_persons_labels(case, data) -> None: + """Module docstring: draws keyed by + ``source_year``/``source_household_id``/``source_person_id`` mean + "support-channel clones of one source person always receive the same + status"; ``_select_weight_to_target``: "a clone pair is never split". + Clones copy their source person's survey record but may carry any + weight.""" + + source = case.person.assign( + source_year=2024, + source_household_id=case.person["person_id"] + 100, + source_person_id=case.person["person_id"], + ) + copies = data.draw( + st.lists(st.integers(0, len(source) - 1), min_size=1, max_size=16), + label="cloned rows", + ) + clones = source.iloc[copies].copy() + clones["person_weight"] = data.draw( + st.lists(_WEIGHT, min_size=len(copies), max_size=len(copies)), + label="clone weights", + ) + person = pd.concat([source, clones], ignore_index=True) + person["person_id"] = np.arange(1, len(person) + 1) + output = _derive( + person, case.parameters, seed=case.seed, time_period=case.time_period + ) + grouped = output.groupby("source_person_id")[ + list(immigration.US_IMMIGRATION_OUTPUT_COLUMNS) + ].nunique() + assert (grouped == 1).all().all() + + +@_FRAME +@given(_stage_cases(max_size=16), st.floats(0.25, 4.0), st.data()) +def test_frame_entry_point_matches_the_manifest_handler_and_is_idempotent( + case, weight_scale, data +) -> None: + """``with_us_immigration_inputs`` runs "the ``immigration_status`` + manifest stage over a US frame" and returns "A new frame whose person + table carries ``ssn_card_type`` and ``immigration_status_str``"; + ``person_weight_scale`` is a "Stage-only multiplier for person design + weights"; and "Existing output columns are preserved, making the + transform idempotent". Differential: with non-monotone person ids, the + frame entry point labels every person exactly as the packaged manifest + handler does on design weight x scale, leaves the frame's weights and + every other column and table unchanged, and a second call (any seed or + period) returns the frame as it is.""" + + ids = data.draw(st.permutations(range(1, len(case.person) + 1)), label="ids") + person = case.person.assign(person_id=np.asarray(ids, dtype=np.int64)) + # A Frame refuses all-zero weights, which the handler alone accepts. + weights = (person["person_weight"].to_numpy(dtype=np.float64) + 1.0) * 1e3 + frame = _frame(person, weights) + result = immigration.with_us_immigration_inputs( + frame, + seed=case.seed, + time_period=case.time_period, + person_weight_scale=weight_scale, + ) + packaged = [ + operation + for operation in immigration.us_immigration_stage_spec().operations + if operation.kind == "derive_immigration_status" + ] + assert len(packaged) == 1 + expected = _derive( + frame.table("person").assign(person_weight=weights * weight_scale), + packaged[0].parameters, + seed=case.seed, + time_period=case.time_period, + ) + labelled = result.table("person") + columns = list(immigration.US_IMMIGRATION_OUTPUT_COLUMNS) + for column in columns: + np.testing.assert_array_equal( + labelled[column].to_numpy(), expected[column].to_numpy(), err_msg=column + ) + pd.testing.assert_frame_equal(labelled.drop(columns=columns), frame.table("person")) + for entity in US_SCHEMA.group_entities: + pd.testing.assert_frame_equal(result.table(entity), frame.table(entity)) + np.testing.assert_array_equal( + np.asarray(result.resolve_weights("household").values), + np.asarray(frame.resolve_weights("household").values), + ) + again = immigration.with_us_immigration_inputs( + result, + seed=case.seed + 1, + time_period=case.time_period + 1, + person_weight_scale=2.0, + ) + pd.testing.assert_frame_equal(again.table("person"), labelled) + + +#: Composition-gate failures that judge plausibility of the weighted totals +#: (non-citizen share, Pew anchor band, humanitarian bands, a small table +#: that happens to be all one value). Every other failure is structural. +_PLAUSIBILITY_FAILURES = ( + "constant value", + "non-citizen weighted share", + "emergent Pew-defined unauthorized population", + "x the cited stock target", +) + + +def _structural_failures(gate) -> list[str]: + return [ + failure + for failure in gate.failures + if not any(fragment in failure for fragment in _PLAUSIBILITY_FAILURES) + ] + + +def _binary_weights(n: int) -> np.ndarray: + """Distinct powers of two: a weighted total then names its rows exactly.""" + + return 2.0 ** np.arange(n, dtype=np.float64) + + +def _rows_from_binary_total(total: float, n: int) -> np.ndarray: + bits = int(round(total)) + assert float(bits) == total + return np.asarray([(bits >> index) & 1 for index in range(n)], dtype=bool) + + +def _person_row(**overrides) -> dict: + """One ``_asec_row``-shaped record: a US-born citizen adult by default.""" + + row = { + "PRCITSHP": 1, + "PENATVTY": 57, + "PEINUSYR": 0, + "A_AGE": 30, + "A_MARITL": 7, + "A_SPOUSE": 0, + "A_HSCOL": 0, + "A_LFSR": 7, + "indicator": None, + "person_weight": 1.0, + } + return row | overrides + + +#: A worker who arrived in 2007 (PEINUSYR 20) and is 33 entered at 16 when +#: measured in 2024 but at 15 in 2025, so the stage at 2025 emits DACA once +#: the worker spill gives the row an EAD. A gate that re-derived the cohort +#: at 2024 (the defect fixed in 5a89fed1e) flags that row. +_PERIOD_SENSITIVE_DACA_CASE = _StageCase( + person=_asec_table( + [ + _person_row(), + _person_row(PRCITSHP=5, PENATVTY=303, PEINUSYR=20, A_AGE=33, A_LFSR=1), + ] + ), + workers=0.5, + students=1.0, + anchor=1.0, + targets=dict.fromkeys(_DRAW_LABELS, 0.0), + seed=0, + time_period=2025, +) + + +@_STAGE +@given(_stage_cases()) +@example(_PERIOD_SENSITIVE_DACA_CASE) +def test_stage_output_satisfies_the_composition_gates_structural_contract( + case, +) -> None: + """Differential against ``us_immigration_composition_gate``, which fails + "when a value falls outside the engine enum domain, when the two columns + disagree about citizenship or undocumented status", on source-evidence and + Cuban/Haitian/DACA cohort violations, on humanitarian rows outside every + draw's cohort, and when "an explicit zero target" emits. Its + ``time_period`` "must be the period the stage ran for", so the gate runs + at the stage's period (several periods are drawn). The stage and the gate + encode those contracts separately; stage output must pass all of them + (the weighted plausibility bands are not structural).""" + + output = _run(case) + event(f"time period {case.time_period}") + gate = immigration.us_immigration_composition_gate( + _frame(output, _binary_weights(len(output))), + time_period=case.time_period, + controls=case.controls, + ) + assert _structural_failures(gate) == [] + assert gate.details["evidence_derived_status_compatibility"]["invalid_rows"] == 0 + + +@_STAGE +@given(_stage_cases()) +def test_spill_projection_selects_exactly_the_gates_pew_universe(case) -> None: + """Differential: ``_pew_unauthorized_projection_mask`` "Project[s] final + membership in Pew's broad unauthorized universe" that the composition + gate then measures (``_PEW_UNAUTHORIZED_STATUS_VALUES`` plus residual + Cuban/Haitian EAD rows). On final stage output the two definitions select + the same rows; power-of-two gate weights make the gate's total name its + rows.""" + + output = _run(case) + facts = _facts(case.person, output, case.time_period) + gate = immigration.us_immigration_composition_gate( + _frame(output, _binary_weights(len(output))), + time_period=case.time_period, + controls=case.controls, + ) + gate_rows = _rows_from_binary_total( + gate.details["pew_unauthorized_population"], len(output) + ) + projection = immigration._pew_unauthorized_projection_mask( + ssn_codes=_codes(facts.ssn), + humanitarian_marks=np.where(facts.humanitarian, facts.status, ""), + retained_ead_cohort=facts.retained, + ) + if (projection & (facts.ssn != "NONE")).any(): + event("Pew universe includes non-NONE rows") + np.testing.assert_array_equal(projection, gate_rows) + + +@_STAGE +@given(_stage_cases()) +def test_pew_margins_close_whenever_the_residual_pool_can_close_them(case) -> None: + """Module docstring, step 3: workers and students "spill to + ``NON_CITIZEN_VALID_EAD`` in deterministic seeded order until the broad + reported universes match their controls"; the worker spill "runs last and + is therefore authoritative"; and "a count already at or below its control + spills nothing". Each margin ends at or below its control unless every + removable (residual, non-retained) row in it was spilled, and no row is + spilled when both margins start below their controls.""" + + output = _run(case) + facts = _facts(case.person, output, case.time_period) + tolerance = _tolerance(facts.weights) + pew = immigration._pew_unauthorized_projection_mask( + ssn_codes=_codes(facts.ssn), + humanitarian_marks=np.where(facts.humanitarian, facts.status, ""), + retained_ead_cohort=facts.retained, + ) + residual = ( + facts.noncitizen + & ~facts.humanitarian + & np.isin(facts.ssn, ("NONE", "NON_CITIZEN_VALID_EAD")) + ) + spilled = residual & (facts.ssn == "NON_CITIZEN_VALID_EAD") + initial: dict[str, float] = {} + for name, scope, target in ( + ("students", facts.student, case.students), + ("workers", facts.worker, case.workers), + ): + final = float(facts.weights[scope & pew].sum()) + removable = scope & residual & ~facts.retained + if (scope & spilled).any(): + event(f"the {name} margin spilled rows") + if final <= target + tolerance and (removable & ~spilled).any(): + event(f"the {name} margin closed with residual rows left") + assert final <= target + tolerance or not (removable & ~spilled).any(), name + initial[name] = final + float( + facts.weights[scope & spilled & ~facts.retained].sum() + ) + if ( + initial["students"] < case.students - tolerance + and initial["workers"] < case.workers - tolerance + ): + assert not spilled.any() + + +# --------------------------------------------------------------------------- +# ACS transfer reconciliation +# --------------------------------------------------------------------------- + +_ACS_ROW = st.fixed_dictionaries( + { + "CIT": st.sampled_from((1, 4, 5, 5, 5, 5)), + # Biased toward the humanitarian origins so recipients can fill draws. + "POBP": st.one_of( + st.sampled_from(_BIRTH_CODES), st.sampled_from(_HUMANITARIAN_ORIGIN_CODES) + ), + "YOEP": st.one_of(st.integers(1975, 2024), st.integers(2016, 2024)), + "age": st.integers(0, 85), + "ssn_card_type": st.sampled_from(immigration.SSN_CARD_TYPE_VALUES), + # The joint-QRF baseline never emits an evidence-constrained label. + "immigration_status_str": st.sampled_from( + ("CITIZEN", "LEGAL_PERMANENT_RESIDENT", "UNDOCUMENTED") + ), + "weight": st.one_of( + st.integers(1, 5_000).map(float), + st.floats(0.5, 5e5, allow_nan=False, allow_subnormal=False), + ), + } +) + + +@dataclass(frozen=True) +class _TransferCase: + person: pd.DataFrame + weights: np.ndarray + mutable: np.ndarray + controls: immigration.ImmigrationControls + seed: int + #: Each draw's recipient candidates before earlier draws remove theirs. + pools: Mapping[str, np.ndarray] + #: The draw whose target no selection can bring to the band floor. + floor_breaker: str | None = None + + +#: Exhausting cases keep each draw at this share of its target or more, above +#: the composition band's floor, so only the floor test reaches the refusal. +_FLOOR_SAFE_SHARE = immigration._HUMANITARIAN_TARGET_RELATIVE_BAND[0] + 0.01 + + +@st.composite +def _transfer_cases(draw, *, floor_breaker: bool = False) -> _TransferCase: + """ASEC rows labeled by the stage (immutable) stacked with ACS recipients + carrying unconstrained donor labels (mutable), as in the ACS transfer. + + Each draw's target is at least half its immutable ASEC mass (so a draw + with immutable rows never has an explicit-zero target) plus a share of + the recipient mass that this draw, and no earlier draw, can claim; + ``selected_once`` can only remove rows an earlier draw claims. Some + targets then add an excess the candidates cannot fill, so those draws + exhaust them. The excess keeps the immutable mass plus the exclusive + recipient mass at ``_FLOOR_SAFE_SHARE`` of the target or more, so the + refusal below the composition band's floor never applies and any error is + a finding. With ``floor_breaker``, one draw instead takes a target that + even its immutable mass plus every one of its candidates leaves below the + floor. Some recipients are ACS twins of ASEC rows (same origin, age and + the ASEC bin's midpoint year), so a draw often has both an immutable + contribution and recipients to select. + """ + + stage = draw(_stage_cases(min_size=4, max_size=16, time_periods=(_TIME_PERIOD,))) + asec = _run(stage).assign(CIT=np.nan, POBP=np.nan, YOEP=np.nan, age=np.nan) + acs_rows = draw(st.lists(_ACS_ROW, min_size=2, max_size=12)) + for index in draw( + st.lists(st.integers(0, len(asec) - 1), max_size=8), label="ACS twins" + ): + source = asec.iloc[index] + noncitizen = int(source["PRCITSHP"]) == 5 + acs_rows.append( + { + "CIT": 5 if noncitizen else 1, + "POBP": int(source["PENATVTY"]), + "YOEP": immigration._ARRIVAL_YEAR_MIDPOINTS.get( + int(source["PEINUSYR"]), _TIME_PERIOD + ), + "age": int(source["A_AGE"]), + "ssn_card_type": draw( + st.sampled_from( + ("OTHER_NON_CITIZEN", "NONE", "NON_CITIZEN_VALID_EAD") + if noncitizen + else ("CITIZEN",) + ) + ), + "immigration_status_str": draw( + st.sampled_from(("LEGAL_PERMANENT_RESIDENT", "UNDOCUMENTED")) + if noncitizen + else st.just("CITIZEN") + ), + "weight": draw(st.integers(1, 5_000).map(float)), + } + ) + acs = pd.DataFrame(acs_rows) + acs = acs.rename(columns={"weight": "person_weight"}).assign( + PRCITSHP=np.nan, PENATVTY=np.nan, PEINUSYR=np.nan, A_AGE=np.nan + ) + person = pd.concat([asec, acs], ignore_index=True) + person["person_id"] = np.arange(1, len(person) + 1) + weights = person.pop("person_weight").to_numpy(dtype=np.float64) + mutable = np.arange(len(person)) >= len(asec) + + frame = _frame(person, weights) + profile = immigration._source_aware_immigration_profile( + person, time_period=_TIME_PERIOD + ) + # Pair repair turns a recipient non-citizen's CITIZEN code into + # OTHER_NON_CITIZEN and leaves every other non-citizen code alone; no + # citizen is ever a humanitarian candidate. + codes = immigration._ssn_name_codes(person) + codes = np.where(~profile.is_citizen & (codes == 1), 3, codes) + claimed = np.zeros(len(person), dtype=bool) + targets: dict[str, float] = {} + pools: dict[str, np.ndarray] = {} + breaker = ( + draw( + st.sampled_from([c.label for c in stage.controls.humanitarian]), + label="floor breaker", + ) + if floor_breaker + else None + ) + for control in stage.controls.humanitarian: + immutable_mass = float( + weights[ + immigration.us_immigration_humanitarian_draw_mask( + frame, control, time_period=_TIME_PERIOD + ) + & ~mutable + ].sum() + ) + candidate_codes = ( + np.where(codes == 2, 0, codes) + if control.category in {"paroled_one_year", "tps"} + else codes + ) + pool = mutable & immigration._humanitarian_draw_candidates( + control, ssn_codes=candidate_codes, profile=profile + ) + exclusive = float(weights[pool & ~claimed].sum()) + claimed |= pool + pools[control.label] = pool + if control.label == breaker: + reachable = immutable_mass + float(weights[pool].sum()) + floor = immigration._HUMANITARIAN_TARGET_RELATIVE_BAND[0] + targets[control.label] = ( + (reachable + 1.0) + / floor + * draw(st.floats(1.05, 3.0), label="floor breach") + ) + continue + immutable_share = draw( + st.one_of(st.just(1.0), st.floats(0.5, 1.0)), label="immutable share" + ) + exclusive_share = draw( + st.one_of(st.just(0.0), st.just(1.0), st.floats(0.0, 1.0)), + label="exclusive share", + ) + base_target = immutable_share * immutable_mass + exclusive_share * exclusive + headroom = max( + 0.0, (immutable_mass + exclusive) / _FLOOR_SAFE_SHARE - base_target + ) + excess_share = draw( + st.one_of(st.just(0.0), st.just(1.0), st.floats(0.0, 1.0)), + label="excess share", + ) + targets[control.label] = base_target + excess_share * headroom + return _TransferCase( + person=person, + weights=weights, + mutable=mutable, + controls=immigration._controls_from_parameters( + _manifest_parameters( + workers=stage.workers, + students=stage.students, + anchor=stage.anchor, + targets=targets, + ) + ), + seed=draw(st.integers(0, 2**31 - 1)), + pools=pools, + floor_breaker=breaker, + ) + + +def _reconcile(case: _TransferCase): + return immigration.reconcile_us_immigration_humanitarian_transfer( + case.person, + weights=case.weights, + mutable_rows=case.mutable, + seed=case.seed, + time_period=_TIME_PERIOD, + controls=case.controls, + ) + + +@_TRANSFER +@given(_transfer_cases()) +def test_reconciliation_output_satisfies_the_gates_structural_contract(case) -> None: + """Differential: ``reconcile_us_immigration_humanitarian_transfer``'s own + final checks restate the gate's citizenship/``NONE``⇔``UNDOCUMENTED`` + pairing and Cuban/Haitian/DACA cohort contracts; the independently coded + ``us_immigration_composition_gate`` must find no structural failure in + the reconciled frame. Reconciliation is deterministic. Immutable rows are + compared too, but the function already raises "changed immutable ASEC + rows" before returning otherwise, so that comparison confirms the guard + rather than adding evidence.""" + + result, receipt = _reconcile(case) + if any( + entry["selected_recipient_population"] > 0 + for entry in receipt["draws"].values() + ): + event("reconciliation selected recipients") + immutable = ~case.mutable + for column in immigration.US_IMMIGRATION_OUTPUT_COLUMNS: + np.testing.assert_array_equal( + result.loc[immutable, column].to_numpy(), + case.person.loc[immutable, column].to_numpy(), + ) + gate = immigration.us_immigration_composition_gate( + _frame(result, case.weights), + time_period=_TIME_PERIOD, + controls=case.controls, + ) + assert _structural_failures(gate) == [] + again_result, again_receipt = _reconcile(case) + pd.testing.assert_frame_equal(again_result, result) + assert again_receipt == receipt + + +@_TRANSFER +@given(_transfer_cases()) +def test_reconciliation_receipts_obey_their_accounting_identities(case) -> None: + """Each draw "receives the residual of its national target after its + compatible, immutable contribution, in manifest order", and the receipt + records it. + + Independent evidence: ``achieved_population`` is recomputed from the + post-selection emitted mask, separately from the immutable and selected + masks, and must equal immutable + selected; a zero target selects + nothing; the selection never exceeds the eligible pool. + + A draw whose eligible candidates hold less than its residual "takes every + one of them and its receipt records the shortfall": its selection equals + its eligible pool, it crosses no threshold, its shortfall is the residual + less the pool, and it keeps the composition band's floor. + + Receipt consistency only (not independent evidence): the function raises + ``RuntimeError`` before returning a receipt whose unexhausted selection + misses its residual by more than its threshold tie mass or whose achieved + error exceeds the discrete bound, and it sets both bound flags to the + negation of ``candidates_exhausted``. Those assertions confirm the receipt + records the quantities it checked.""" + + _, receipt = _reconcile(case) + tolerance = float(receipt["floating_tolerance"]) + assert receipt["selection_order"] == [ + draw.label for draw in case.controls.humanitarian + ] + assert receipt["mutable_rows"] == int(case.mutable.sum()) + for draw in case.controls.humanitarian: + entry = receipt["draws"][draw.label] + target = float(draw.target) + immutable = entry["immutable_population"] + selected = entry["selected_recipient_population"] + achieved = entry["achieved_population"] + tie = entry["selection_threshold_tie_population"] + if selected > 0 and immutable > 0: + event("a draw combined immutable and selected rows") + if immutable > target: + event("immutable rows overshoot a target") + # Independent identities. + assert achieved == pytest.approx(immutable + selected, abs=tolerance) + assert selected <= entry["eligible_recipient_population"] + tolerance + if target <= 0: + assert selected == 0.0 + # Receipt consistency with the function's own guards. + assert entry["target"] == target + residual = entry["residual_target"] + eligible = entry["eligible_recipient_population"] + assert residual == pytest.approx(max(0.0, target - immutable), abs=tolerance) + exhausted = eligible + tolerance < residual + assert entry["candidates_exhausted"] is exhausted + assert entry["within_residual_discrete_weight_bound"] is not exhausted + assert entry["within_discrete_weight_bound"] is not exhausted + if exhausted: + event("a draw exhausted its candidates") + assert selected == eligible + assert tie == 0.0 + assert entry["candidate_shortfall_population"] == residual - eligible + floor = immigration._HUMANITARIAN_TARGET_RELATIVE_BAND[0] + assert achieved + tolerance >= floor * target + else: + assert entry["candidate_shortfall_population"] == 0.0 + assert abs(selected - residual) <= tie + tolerance + assert abs(achieved - target) <= ( + max(0.0, immutable - target) + tie + tolerance + ) + + +@_TRANSFER +@given(_transfer_cases()) +def test_selection_replay_reproduces_the_reconciled_mutable_rows(case) -> None: + """Differential: ``us_immigration_humanitarian_transfer_selection_masks`` + promises to "Replay the exact deterministic mutable-row selection for + every draw" from the post-transfer frame. For every draw, its replayed + mask equals the reconciled mutable rows that + ``us_immigration_humanitarian_draw_mask`` reports as emitted (the + comparison ``stacked_spine`` makes at release).""" + + result, _ = _reconcile(case) + frame = _frame(result, case.weights) + replay = immigration.us_immigration_humanitarian_transfer_selection_masks( + frame, + mutable_rows=case.mutable, + seed=case.seed, + time_period=_TIME_PERIOD, + controls=case.controls, + ) + assert list(replay) == [draw.label for draw in case.controls.humanitarian] + for draw in case.controls.humanitarian: + emitted = immigration.us_immigration_humanitarian_draw_mask( + frame, draw, time_period=_TIME_PERIOD + ) + if (emitted & case.mutable).any(): + event("a replayed draw selected recipients") + np.testing.assert_array_equal( + replay[draw.label], emitted & case.mutable, err_msg=draw.label + ) + + +@_TRANSFER +@given(_transfer_cases()) +def test_exhausted_draws_take_every_candidate_no_earlier_draw_took(case) -> None: + """A draw whose candidates hold less than its residual "takes every one of + them". Independent evidence: each draw's candidate pool comes from the + strategy, computed before any selection, and the rows earlier draws took + come from the replay that ``test_selection_replay_reproduces_the_reconciled_ + mutable_rows`` checks against the emitted labels. An exhausted draw's + selection is exactly its pool less those rows, and its receipt's eligible + population is their weight.""" + + result, receipt = _reconcile(case) + replay = immigration.us_immigration_humanitarian_transfer_selection_masks( + _frame(result, case.weights), + mutable_rows=case.mutable, + seed=case.seed, + time_period=_TIME_PERIOD, + controls=case.controls, + ) + taken = np.zeros(len(case.person), dtype=bool) + for draw in case.controls.humanitarian: + remaining = case.pools[draw.label] & ~taken + entry = receipt["draws"][draw.label] + assert entry["eligible_recipient_population"] == pytest.approx( + float(case.weights[remaining].sum()), + abs=float(receipt["floating_tolerance"]), + ) + if entry["candidates_exhausted"]: + event("an exhausted draw took its remaining candidates") + np.testing.assert_array_equal( + replay[draw.label], remaining, err_msg=draw.label + ) + assert ( + result.loc[remaining, "immigration_status_str"] == draw.status + ).all() + taken |= replay[draw.label] + + +@_TRANSFER +@given(_transfer_cases(floor_breaker=True)) +def test_reconciliation_refuses_a_draw_exhausted_below_the_band_floor(case) -> None: + """The reconciliation "refuses only a draw that then reaches less than the + floor of the composition gate's band of its target". The strategy gives + one draw a target that its immutable mass plus every one of its + candidates leaves below that floor, and keeps every other draw above it: + the reconciliation must refuse that draw, by name.""" + + floor = immigration._HUMANITARIAN_TARGET_RELATIVE_BAND[0] + with pytest.raises(ValueError) as refusal: + _reconcile(case) + assert str(refusal.value).startswith( + "Humanitarian transfer candidate shortfall for " + f"{case.floor_breaker!r} leaves it below {floor} of its target" + ) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_late_producer_dag.py b/packages/microcosm-build/tests/engine_free/us/test_us_late_producer_dag.py index 2c2af43514..7e09f88713 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_late_producer_dag.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_late_producer_dag.py @@ -804,7 +804,8 @@ def test_canonical_us_late_schedule_is_import_validated_and_byte_stable() -> Non "virtual_resource_binding": ("exact_kind_specific_semantic_payload_and_sha256"), "top_binding": ( "entry_and_output_frame_sha256_execution_chain_source_" - "completion_and_nineteen_transfer_groups" + "completion_nineteen_transfer_groups_and_constrained_" + "immigration_reconciliation" ), "transition_authority": { "authority_id": "us_stacked_late_producer_transition", @@ -920,6 +921,72 @@ def test_every_transfer_declares_predictors_and_optional_absence_receipts() -> N ) +def test_immigration_transfer_declares_source_scoped_raw_evidence() -> None: + producer = transfer_producer_name("person", "source_operator_immigration") + inventory = US_LATE_TRANSFER_INPUT_INVENTORIES[producer] + by_label = { + requirement.label: requirement for requirement in inventory.requirements + } + expected = { + "immigration_asec_citizenship": ("PRCITSHP", "asec_source"), + "immigration_asec_origin": ("PENATVTY", "asec_source"), + "immigration_asec_arrival": ("PEINUSYR", "asec_source"), + "immigration_acs_citizenship": ("CIT", "acs_source"), + "immigration_acs_origin": ("POBP", "acs_source"), + "immigration_acs_arrival": ("YOEP", "acs_source"), + } + + assert set(expected) <= set(by_label) + for label, (column, scope) in expected.items(): + requirement = by_label[label] + assert requirement.required_scope == scope + assert requirement.optional is False + value_kind = "column_present" if column == "YOEP" else "finite_numeric" + assert requirement.alternatives == ( + (ProducerInputColumn("person", column, value_kind),), + ) + + expected_lineage = { + ("person_id",), + ("source_person_id", "source_year"), + ("source_household_id", "source_person_id", "source_year"), + } + transfer_lineage = by_label["immigration_stable_person_lineage"] + assert transfer_lineage.required_scope is None + assert { + tuple(column.column for column in alternative) + for alternative in transfer_lineage.alternatives + } == expected_lineage + + source_inventory = US_LATE_SOURCE_INPUT_INVENTORIES["with_us_immigration_inputs"] + source_lineage = next( + requirement + for requirement in source_inventory.requirements + if requirement.label == "stable_source_identity" + ) + assert { + tuple(column.column for column in alternative) + for alternative in source_lineage.alternatives + } == expected_lineage + + contract = CANONICAL_US_LATE_PRODUCER_REGISTRY[producer] + effective = { + item.column: item + for item in contract.inputs + if item.column.startswith("@effective:immigration_") + } + assert set(effective) == { + *(f"@effective:{label}" for label in expected), + "@effective:immigration_stable_person_lineage", + } + for label, (_column, scope) in expected.items(): + assert effective[f"@effective:{label}"].required_scope == scope + assert ( + effective["@effective:immigration_stable_person_lineage"].required_scope + == "whole_pool" + ) + + def test_every_transfer_declares_complete_cross_grain_validation_surface() -> None: entities = ( "person", @@ -970,12 +1037,19 @@ def test_every_transfer_declares_complete_cross_grain_validation_surface() -> No def test_every_origin_exclusive_raw_input_has_its_native_scope() -> None: + # The loader keeps the ACS immigration evidence optional (microcosm#1020); + # the late immigration producer still reads it as ACS-only raw input + # (microcosm#767), so it carries the ACS scope like the required columns. + acs_immigration_evidence = ("CIT", "POBP", "YOEP") + assert set(acs_immigration_evidence) <= set(acs_pums_module._PERSON_OPTIONAL) acs_raw_inputs = { *(("household", column) for column in acs_pums_module._HOUSEHOLD_FRAME_COLUMNS), *(("person", column) for column in acs_pums_module._PERSON_REQUIRED), + *(("person", column) for column in acs_immigration_evidence), } asec_person_raw_columns = { "A_HSCOL", + "A_LFSR", "A_MJOCC", "CAID", "CHAMPVA", @@ -1044,6 +1118,12 @@ def test_every_origin_exclusive_raw_input_has_its_native_scope() -> None: "PERIDNUM", } }, + ("person", "PRCITSHP"): 2, + ("person", "PENATVTY"): 2, + ("person", "PEINUSYR"): 2, + ("person", "CIT"): 1, + ("person", "POBP"): 1, + ("person", "YOEP"): 1, } observed: Counter[tuple[str, str, str]] = Counter() @@ -1065,15 +1145,15 @@ def test_every_origin_exclusive_raw_input_has_its_native_scope() -> None: assert observed == Counter( {(*key, required_scope[key]): count for key, count in expected_counts.items()} ) - assert sum(observed.values()) == 101 - assert sum(count for key, count in observed.items() if key[2] == "acs_source") == 43 + assert sum(observed.values()) == 108 + assert sum(count for key, count in observed.items() if key[2] == "acs_source") == 46 assert ( - sum(count for key, count in observed.items() if key[2] == "asec_source") == 58 + sum(count for key, count in observed.items() if key[2] == "asec_source") == 62 ) assert receipts[("household", "TYPEHUGQ")] == set() execution_identity = acs_transfer_module.acs_transfer_execution_contract_identity() - assert execution_identity["schema_version"] == 2 + assert execution_identity["schema_version"] == 3 assert execution_identity["housing"]["head_source_precedence"] == [ {"source": "is_household_head", "head_codes": [True]}, {"source": "A_EXPRRP", "head_codes": [1, 2]}, @@ -1278,8 +1358,7 @@ def test_source_numeric_input_audit_is_fully_executable() -> None: "A_MARITL", "A_SPOUSE", "A_HSCOL", - "WSAL_VAL", - "SEMP_VAL", + "A_LFSR", "MCARE", "CAID", "IHSFLG", diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_legacy_schema9_contract.py b/packages/microcosm-build/tests/engine_free/us/test_us_legacy_schema9_contract.py new file mode 100644 index 0000000000..abdea54396 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/us/test_us_legacy_schema9_contract.py @@ -0,0 +1,368 @@ +"""Attested schema-9 pools keep #767's immigration contracts at schema 16. + +Authenticated legacy scoring compares a historical pool's sealed late-producer +schedule, authority and execution rows against reconstructions. Main derives +those reconstructions from today's registry and relabels them at schema 16. +Microcosm #767 changed the inputs of two producers: the immigration source +stage and its transfer group. A schema-9 pool's execution rows carry those two +producers' schema-16 input surfaces (57 and 93 inputs), which today's +contracts (56 and 100) refuse. So the legacy path takes those two inventories, +and the schema-16 execution receipt contract, from the frozen schema-16 +schedule, and checks each rebuilt contract against the digest the schema-16 +registry derived. It otherwise follows today's registry exactly as main does. + +Main's own drift since schema 16 is out of scope here: microcosm #1239 and +#1258 changed the primary-PUF and PUF-itemization contracts, so the legacy +schedule no longer hashes to the sealed one. +""" + +from __future__ import annotations + +import hashlib +import json + +import pytest +from hypothesis import given +from hypothesis import strategies as st + +from microcosm.build.us_runtime import h5_io, stacked_spine +from microcosm.build.us_runtime import us_late_producer_registry as registry +from microcosm.build.us_runtime.late_producer_dag import _contract_payload + +#: The sealed content hash of the frozen schema-16 schedule, computed on main +#: at 7c4a5ac0 (the base of microcosm #779), which reproduced it. +HISTORICAL_SCHEDULE_PAYLOAD_SHA256 = ( + "02e618cc656eb39990ed99dca2b30a52794e01e2b06a3c2df87ca4a7d85ab086" +) +IMMIGRATION_SOURCE = "source:with_us_immigration_inputs" +IMMIGRATION_TRANSFER = "transfer:person/source_operator_immigration" +#: Canonical-JSON SHA-256 of the two contracts as main at 7c4a5ac0 derived +#: them; that registry reproduced the frozen schedule's ``schedule_sha256``. +HISTORICAL_IMMIGRATION_CONTRACT_SHA256 = { + IMMIGRATION_SOURCE: ( + "8ffdcf5128e54956ea63669033cddcb894af93333fdf61e2262d9f54105ec5af" + ), + IMMIGRATION_TRANSFER: ( + "81c5bdc4567090d8c68aac8101214b9f2db3a16485cc14521b81e530a0860f96" + ), +} +SCHEMA16_TOP_BINDING = ( + "entry_and_output_frame_sha256_execution_chain_source_completion_" + "and_nineteen_transfer_groups" +) + + +def _canonical_sha256(payload: object) -> str: + return hashlib.sha256( + json.dumps( + payload, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode("utf-8") + ).hexdigest() + + +def _frozen() -> dict[str, object]: + return json.loads(registry._verified_legacy_schedule_json()) + + +def _row(rows, operator: str) -> dict[str, object]: + return next(row for row in rows if row["operator"] == operator) + + +def _labels(row) -> set[str]: + return {requirement["label"] for requirement in row["requirements"]} + + +def _clear_legacy_caches() -> None: + registry._verified_legacy_schedule_json.cache_clear() + registry.legacy_us_late_producer_contracts.cache_clear() + registry._legacy_schedule_receipt_json.cache_clear() + + +def test_frozen_schedule_resource_is_the_sealed_historical_content() -> None: + frozen = _frozen() + payload = { + key: value + for key, value in frozen.items() + if key not in registry._LEGACY_SCHEDULE_DERIVED_KEYS + } + assert _canonical_sha256(payload) == HISTORICAL_SCHEDULE_PAYLOAD_SHA256 + assert frozen["payload_sha256"] == HISTORICAL_SCHEDULE_PAYLOAD_SHA256 + assert ( + registry.LEGACY_SCHEMA16_LATE_PRODUCER_SCHEDULE_PAYLOAD_SHA256 + == HISTORICAL_SCHEDULE_PAYLOAD_SHA256 + ) + assert frozen["schema_version"] == 16 + + +def test_legacy_receipt_carries_the_schema16_receipt_contract() -> None: + legacy = registry.legacy_us_late_producer_schedule_receipt() + live = registry.us_late_producer_schedule_receipt() + assert legacy["schema_version"] == 16 + assert ( + legacy["execution_receipt_contract"] == _frozen()["execution_receipt_contract"] + ) + contract = legacy["execution_receipt_contract"] + assert contract["version"] == 3 + assert contract["transition_authority"]["version"] == 1 + assert contract["top_binding"] == SCHEMA16_TOP_BINDING + assert live["execution_receipt_contract"]["top_binding"] != SCHEMA16_TOP_BINDING + + +def test_legacy_receipt_payload_hash_is_a_real_content_hash() -> None: + receipt = dict(registry.legacy_us_late_producer_schedule_receipt()) + payload = { + key: value + for key, value in receipt.items() + if key not in registry._LEGACY_SCHEDULE_DERIVED_KEYS + } + assert _canonical_sha256(payload) == receipt["payload_sha256"] + assert receipt["schedule_sha256"] == ( + registry.legacy_us_late_producer_contracts().schedule.sha256 + ) + + +def test_legacy_receipt_differs_from_live_only_in_767s_surface() -> None: + legacy = dict(registry.legacy_us_late_producer_schedule_receipt()) + live = json.loads(json.dumps(dict(registry.us_late_producer_schedule_receipt()))) + assert set(legacy) == set(live) + changed = {key for key in live if legacy[key] != live[key]} + assert changed <= { + "schema_version", + "execution_receipt_contract", + "schedule_sha256", + "payload_sha256", + "source_input_inventories", + "transfer_input_inventories", + "edges", + "waves", + } + assert { + "schema_version", + "execution_receipt_contract", + "schedule_sha256", + "payload_sha256", + "source_input_inventories", + "transfer_input_inventories", + } <= changed + assert legacy["order"] == live["order"] + for key, operator in ( + ("source_input_inventories", "with_us_immigration_inputs"), + ("transfer_input_inventories", IMMIGRATION_TRANSFER), + ): + differing = { + row["operator"] + for row in live[key] + if row != _row(legacy[key], row["operator"]) + } + assert differing == {operator} + + +def test_legacy_immigration_inventories_are_the_frozen_rows() -> None: + frozen = _frozen() + legacy = registry.legacy_us_late_producer_contracts() + source = registry._inventory_payload( + legacy.source_inventories["with_us_immigration_inputs"] + ) + transfer = registry._inventory_payload( + legacy.transfer_inventories[IMMIGRATION_TRANSFER] + ) + assert source == _row( + frozen["source_input_inventories"], "with_us_immigration_inputs" + ) + assert transfer == _row(frozen["transfer_input_inventories"], IMMIGRATION_TRANSFER) + live = registry._inventory_payload( + registry.US_LATE_SOURCE_INPUT_INVENTORIES["with_us_immigration_inputs"] + ) + # #767 replaced the wage and self-employment reads with the ASEC labor + # force recode; the schema-16 row keeps the reads those pools sealed. + assert {"raw_person:WSAL_VAL", "raw_person:SEMP_VAL"} <= _labels(source) + assert "raw_person:A_LFSR" not in _labels(source) + assert "raw_person:A_LFSR" in _labels(live) + + +def test_legacy_contracts_differ_from_live_exactly_at_767s_two_producers() -> None: + legacy = registry.legacy_us_late_producer_contracts().registry + live = registry.CANONICAL_US_LATE_PRODUCER_REGISTRY + assert set(legacy) == set(live) + changed = {name for name in live if legacy[name] != live[name]} + assert changed == {IMMIGRATION_SOURCE, IMMIGRATION_TRANSFER} + assert len(legacy[IMMIGRATION_SOURCE].inputs) == 57 + assert len(live[IMMIGRATION_SOURCE].inputs) == 56 + assert len(legacy[IMMIGRATION_TRANSFER].inputs) == 93 + assert len(live[IMMIGRATION_TRANSFER].inputs) == 100 + for name in (IMMIGRATION_SOURCE, IMMIGRATION_TRANSFER): + assert legacy[name].outputs == live[name].outputs + assert legacy[name].kind == live[name].kind + + +def test_legacy_immigration_contracts_hash_to_their_sealed_digests() -> None: + legacy = registry.legacy_us_late_producer_contracts().registry + assert dict(registry._LEGACY_SCHEMA16_SEALED_CONTRACT_SHA256) == ( + HISTORICAL_IMMIGRATION_CONTRACT_SHA256 + ) + for name, expected in HISTORICAL_IMMIGRATION_CONTRACT_SHA256.items(): + assert _canonical_sha256(_contract_payload(legacy[name])) == expected + assert ( + _canonical_sha256( + _contract_payload(registry.CANONICAL_US_LATE_PRODUCER_REGISTRY[name]) + ) + != expected + ) + + +def test_legacy_authority_binds_the_legacy_schedule() -> None: + authority = stacked_spine._legacy_stacked_authority_receipt() + assert authority["version"] == 11 + component = authority["components"]["late_producer_schedule"] + legacy = stacked_spine._json_ready( + registry.legacy_us_late_producer_schedule_receipt() + ) + assert stacked_spine._json_ready(component["identity"]) == legacy + assert component["schedule_sha256"] == legacy["schedule_sha256"] + live = stacked_spine._json_ready(registry.us_late_producer_schedule_receipt()) + assert stacked_spine._json_ready(component["identity"]) != live + + +def test_tampered_frozen_schedule_is_refused(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + registry, + "LEGACY_SCHEMA16_LATE_PRODUCER_SCHEDULE_PAYLOAD_SHA256", + "0" * 64, + ) + _clear_legacy_caches() + try: + with pytest.raises(ValueError, match="sealed content hash"): + registry.legacy_us_late_producer_schedule_receipt() + finally: + _clear_legacy_caches() + + +def test_schema9_operator_order_is_the_literal_historical_order() -> None: + historical = h5_io._SCHEMA9_STACKED_POOL_OPERATOR_ORDER + assert historical == ( + "assemble_stacked_spine", + "assign_us_puma_ladder", + "prepare_multispine_source_inputs_for_clone", + "gap_fill_stacked_spine", + "run_stacked_late_producer_dag", + "prepare_stacked_tail_derivation", + "derive_multispine_pool_inputs", + "seed_multispine_pool_inputs", + "materialize_multispine_agreement_outputs", + "stacked_completeness_gate", + "by_origin_battery", + ) + assert "us_immigration_composition_gate" not in historical + assert h5_io.US_STACKED_POOL_OPERATOR_ORDER[-1] == "us_immigration_composition_gate" + + +def test_each_caller_gets_its_own_legacy_schedule() -> None: + first = registry.legacy_us_late_producer_schedule_receipt() + first["execution_receipt_contract"]["version"] = 99 + first["source_input_inventories"].clear() + + second = registry.legacy_us_late_producer_schedule_receipt() + assert second["execution_receipt_contract"]["version"] == 3 + assert second["source_input_inventories"] + + +def test_inventory_payload_round_trips_exactly() -> None: + for inventories in ( + registry.US_LATE_SOURCE_INPUT_INVENTORIES, + registry.US_LATE_TRANSFER_INPUT_INVENTORIES, + ): + for name, inventory in inventories.items(): + assert inventory.operator == name + payload = registry._inventory_payload(inventory) + parsed = registry._inventory_from_payload(payload) + assert parsed == inventory + assert registry._inventory_payload(parsed) == payload + + +_TOKEN = st.text(alphabet="abcdefghijklmnopqrstuvwxyz_:@", min_size=1, max_size=12) +_SCOPED_INPUT = st.builds( + registry.ScopedInput, + _TOKEN, + _TOKEN, + st.sampled_from(("column_present", "non_null", "finite_numeric")), +) +# Constructors canonicalize: alternatives are deduplicated and sorted, so two +# alternatives over the same columns are one repeated alternative (refused), +# and requirement labels must be unique within an inventory. +_REQUIREMENT = st.builds( + registry.EffectiveInputRequirement, + label=_TOKEN, + optional=st.booleans(), + required_scope=st.one_of(st.none(), _TOKEN), + alternatives=st.lists( + st.lists(_SCOPED_INPUT, min_size=1, max_size=3).map(tuple), + min_size=1, + max_size=3, + unique_by=frozenset, + ).map(tuple), +) +_INVENTORY = st.builds( + registry.SourceInputInventory, + operator=_TOKEN, + requirements=st.lists( + _REQUIREMENT, min_size=1, max_size=4, unique_by=lambda item: item.label + ).map(tuple), +) + + +@given(_INVENTORY) +def test_inventory_payload_round_trip_property(inventory) -> None: + payload = registry._inventory_payload(inventory) + assert registry._inventory_from_payload(payload) == inventory + assert registry._inventory_from_payload(json.loads(json.dumps(payload))) == ( + inventory + ) + + +def test_legacy_contracts_refuse_a_drifted_immigration_derivation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + live_inputs = registry._inventory_contract_inputs + + def drifted_inputs(node_name, inventory, *, required_scope): + inputs = live_inputs(node_name, inventory, required_scope=required_scope) + return inputs[1:] if node_name == IMMIGRATION_SOURCE else inputs + + monkeypatch.setattr(registry, "_inventory_contract_inputs", drifted_inputs) + _clear_legacy_caches() + try: + with pytest.raises(ValueError, match="no longer matches the schema-16"): + registry.legacy_us_late_producer_contracts() + finally: + _clear_legacy_caches() + + +@pytest.mark.parametrize("fault", ["missing", "not_round_tripping"]) +def test_legacy_contracts_refuse_an_unusable_sealed_inventory( + monkeypatch: pytest.MonkeyPatch, fault: str +) -> None: + if fault == "missing": + monkeypatch.setattr( + registry, + "LEGACY_SCHEMA16_SEALED_SOURCE_INVENTORIES", + ("not_a_source_operator",), + ) + message = "has no 'not_a_source_operator' inventory" + else: + live_parse = registry._inventory_from_payload + + def lossy_parse(payload): + inventory = live_parse(payload) + return registry.SourceInputInventory( + operator=inventory.operator, + requirements=inventory.requirements[1:], + ) + + monkeypatch.setattr(registry, "_inventory_from_payload", lossy_parse) + message = "does not round-trip" + _clear_legacy_caches() + try: + with pytest.raises(ValueError, match=message): + registry.legacy_us_late_producer_contracts() + finally: + _clear_legacy_caches() diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_multispine_pool.py b/packages/microcosm-build/tests/engine_free/us/test_us_multispine_pool.py index e53afbc40a..a1d623b7c4 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_multispine_pool.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_multispine_pool.py @@ -1156,7 +1156,7 @@ def test_every_pool_transfer_family_accepts_its_produced_physical_dtype( ) assert len(targets) == 120 - assert len(predictors) == 32 + assert len(predictors) == 35 assert len(primary_predictor_sets) == 66 primary_targets = tuple( ( @@ -2242,8 +2242,13 @@ def apply( name: str = operator_name, column: str = output, value: float = float(index + 1), + person_weight_scale: float = 1.0, ) -> Frame: calls.append(name) + if name == "with_us_immigration_inputs": + assert person_weight_scale > 1.0 + else: + assert person_weight_scale == 1.0 assert "us_spine_assembly_manifest" not in available.metadata assert not available.mass_log person = available.table("person") diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_multispine_pool_h5_io.py b/packages/microcosm-build/tests/engine_free/us/test_us_multispine_pool_h5_io.py index 65ff212d40..9f6e736107 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_multispine_pool_h5_io.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_multispine_pool_h5_io.py @@ -3,20 +3,22 @@ import hashlib import json import sys -from collections.abc import Callable +from collections.abc import Callable, Collection, Mapping from copy import deepcopy from dataclasses import replace -from functools import cache from pathlib import Path import numpy as np import pandas as pd import pytest -import microcosm.build.us_runtime.acs_transfer as acs_transfer_module +import microcosm.build.us_runtime.acs_transfer as acs_transfer_runtime import microcosm.build.us_runtime.h5_io as h5_io +import microcosm.build.us_runtime.immigration as immigration_runtime import microcosm.build.us_runtime.post_transfer_calibration as post_transfer_calibration_runtime +import microcosm.build.us_runtime.puf_support as puf_support_runtime import microcosm.build.us_runtime.stacked_spine as stacked_spine_module +import microcosm.build.us_runtime.us_late_producer_registry as late_producer_registry_module import microcosm.build.us_runtime.worker_identity as worker_identity_module from microcosm.build.frame_checkpoint import ( load_frame_checkpoint, @@ -50,7 +52,12 @@ us_multispine_pool_release_receipt, write_nullable_us_h5, ) +from microcosm.build.us_runtime.late_producer_dag import ProducerContract from microcosm.build.us_runtime.support_provenance import ( + BASE_ASEC_SUPPORT_CHANNEL, + spine_assembly_manifest, + spine_provenance_counts, + support_channel_column, support_clone_index_column, support_source_id_column, ) @@ -113,10 +120,7 @@ def _pool_frame() -> Frame: ) -def _pool_frame_with_object_strings_on_every_entity( - *, - stacked: bool = False, -) -> Frame: +def _pool_frame_with_object_strings_on_every_entity() -> Frame: """Match the assembled pool's object-backed source-string shape.""" frame = _pool_frame() @@ -134,21 +138,6 @@ def _pool_frame_with_object_strings_on_every_entity( ), ) tables[entity] = table - if stacked: - household = tables["household"].copy() - household["puma"] = ["0600101", "0600102", "0600102"] - household["congressional_district_geoid"] = np.asarray( - [601, 601, 601], - dtype=np.int64, - ) - household["county_fips"] = ["06001", "06001", "06001"] - household[support_source_id_column("household")] = np.asarray( - [10, 20, 20], dtype=np.int64 - ) - household[support_clone_index_column("household")] = np.asarray( - [0, 0, 1], dtype=np.int64 - ) - tables["household"] = household return Frame( tables, frame.schema, @@ -159,6 +148,181 @@ def _pool_frame_with_object_strings_on_every_entity( ) +def _stacked_pool_frame_with_live_immigration( + *, + include_zero_weight_clone: bool = False, +) -> tuple[Frame, dict[str, object]]: + """Build a persisted-shape final stack and its real immigration receipt.""" + + controls = immigration_runtime.us_immigration_controls() + positive_draws = tuple(draw for draw in controls.humanitarian if draw.target > 0) + acs_evidence = { + "paroled_one_year:afghanistan": (200, 2021, "NONE", "UNDOCUMENTED"), + "paroled_one_year:ukraine": (164, 2022, "NONE", "UNDOCUMENTED"), + "paroled_one_year:nicaragua": (315, 2023, "NONE", "UNDOCUMENTED"), + "paroled_one_year:venezuela": (373, 2022, "NONE", "UNDOCUMENTED"), + "refugee": ( + 412, + 2022, + "OTHER_NON_CITIZEN", + "LEGAL_PERMANENT_RESIDENT", + ), + "asylee": ( + 207, + 2020, + "OTHER_NON_CITIZEN", + "LEGAL_PERMANENT_RESIDENT", + ), + "tps:venezuela": (373, 2015, "NONE", "UNDOCUMENTED"), + "tps:el_salvador": (312, 2001, "NONE", "UNDOCUMENTED"), + "tps:honduras": (314, 1998, "NONE", "UNDOCUMENTED"), + "tps:nicaragua": (315, 1998, "NONE", "UNDOCUMENTED"), + "tps:nepal": (229, 2015, "NONE", "UNDOCUMENTED"), + "tps:other_designated": (248, 2020, "NONE", "UNDOCUMENTED"), + } + expected_labels = {draw.label for draw in positive_draws} + if set(acs_evidence) != expected_labels: + raise AssertionError( + "Stacked H5 immigration evidence must exactly cover the positive " + "canonical humanitarian draws." + ) + + row_count = 1 + len(positive_draws) + int(include_zero_weight_clone) + ids = np.arange(1, row_count + 1, dtype=np.int64) + channels = np.asarray( + [BASE_ASEC_SUPPORT_CHANNEL] + + [stacked_spine_module.ACS_STACKED_SUPPORT_CHANNEL] * len(positive_draws) + + ([BASE_ASEC_SUPPORT_CHANNEL] if include_zero_weight_clone else []), + dtype=object, + ) + clone_indices = np.zeros(row_count, dtype=np.int64) + source_ids = ids.copy() + if include_zero_weight_clone: + clone_indices[-1] = 1 + source_ids[-1] = ids[0] + + person_rows: list[dict[str, object]] = [ + { + "PRCITSHP": 1, + "PENATVTY": 57, + "PEINUSYR": 0, + "CIT": np.nan, + "POBP": np.nan, + "YOEP": np.nan, + "A_AGE": 50, + "age": 50, + "ssn_card_type": "CITIZEN", + "immigration_status_str": "CITIZEN", + } + ] + for draw in positive_draws: + birth_country, arrival_year, ssn_card_type, immigration_status = acs_evidence[ + draw.label + ] + person_rows.append( + { + "PRCITSHP": np.nan, + "PENATVTY": np.nan, + "PEINUSYR": np.nan, + "CIT": 5, + "POBP": birth_country, + "YOEP": arrival_year, + "A_AGE": np.nan, + "age": 50, + "ssn_card_type": ssn_card_type, + "immigration_status_str": immigration_status, + } + ) + if include_zero_weight_clone: + person_rows.append(dict(person_rows[0])) + + person = pd.DataFrame(person_rows) + person.insert(0, "PERIDNUM", pd.Series([f"person-{value}" for value in ids])) + person.insert(1, "person_id", ids) + for entity in US_SCHEMA.group_entities: + person[US_SCHEMA.membership_column(entity)] = ids + person["nullable_input"] = np.resize( + np.asarray([True, None, False], dtype=object), + row_count, + ) + person["is_incapable_of_self_care"] = True + person["tax_unit_role_input"] = "DEPENDENT" + person[support_channel_column("person")] = channels + person[support_source_id_column("person")] = source_ids + person[support_clone_index_column("person")] = clone_indices + + household_weights = np.asarray( + [1.0] + + [float(draw.target) for draw in positive_draws] + + ([0.0] if include_zero_weight_clone else []), + dtype=np.float64, + ) + mutable = channels == stacked_spine_module.ACS_STACKED_SUPPORT_CHANNEL + reconciled_person, reconciliation = ( + immigration_runtime.reconcile_us_immigration_humanitarian_transfer( + person, + weights=household_weights, + mutable_rows=mutable, + seed=0, + time_period=2024, + controls=controls, + ) + ) + + tables = {"person": reconciled_person} + for entity in US_SCHEMA.group_entities: + table = pd.DataFrame({US_SCHEMA.id_column(entity): ids}) + table.insert( + 0, + f"{entity}_source_label", + pd.Series([f"{entity}-{value}" for value in ids], dtype=object), + ) + table[support_channel_column(entity)] = channels + table[support_source_id_column(entity)] = source_ids + table[support_clone_index_column(entity)] = clone_indices + tables[entity] = table + + household = tables["household"] + household["puma"] = "0600101" + household["congressional_district_geoid"] = np.full( + row_count, + 601, + dtype=np.int64, + ) + household["county_fips"] = "06001" + + for index, spec in enumerate( + post_transfer_calibration_runtime.POST_TRANSFER_CALIBRATION_SPECS.values() + ): + tables[spec.entity][spec.target] = ( + 10.0 + index + np.arange(row_count, dtype=np.float64) + ) + + assembly_tables = { + entity: table.loc[table[support_clone_index_column(entity)].eq(0)] + for entity, table in tables.items() + } + metadata = spine_assembly_manifest( + assembly_tables, + channels=( + BASE_ASEC_SUPPORT_CHANNEL, + stacked_spine_module.ACS_STACKED_SUPPORT_CHANNEL, + ), + ) + frame = Frame( + tables, + US_SCHEMA, + { + "household": Weights( + household_weights, + WeightKind.IMPORTANCE, + ) + }, + metadata=metadata, + ) + return frame, reconciliation + + def _semantic_string_columns(table: pd.DataFrame) -> tuple[str, ...]: return tuple( column @@ -646,24 +810,54 @@ def _write_ready_pool( tmp_path: Path, *, stacked: bool = False, + include_zero_weight_clone: bool = False, sample_fraction: float = 1.0, ) -> Path: run_id = "fixture-publication" pool_path = tmp_path / "pool.h5" diagnostics_path = tmp_path / "pool.agreement.json" manifest_path = tmp_path / "pool.manifest.json" + gate_names = ( + ( + "us_stacked_completeness", + "us_by_origin_battery", + "immigration_composition", + ) + if stacked + else ("us_spine_agreement",) + ) agreement_gate = { "passed": True, "gates": { - "us_spine_agreement": { + name: { "passed": True, "failures": [], "details": {"fixture": True}, } + for name in gate_names }, } + if include_zero_weight_clone and not stacked: + raise ValueError("Only the stacked H5 fixture may include a clone row.") schema_version = US_MULTISPINE_POOL_MANIFEST_SCHEMA_VERSION if stacked else 4 - pool_frame = _pool_frame_with_object_strings_on_every_entity(stacked=stacked) + if stacked: + pool_frame, immigration_reconciliation = ( + _stacked_pool_frame_with_live_immigration( + include_zero_weight_clone=include_zero_weight_clone, + ) + ) + dag = _canonical_stacked_late_dag_receipt( + pool_frame, + immigration_reconciliation=immigration_reconciliation, + ) + provenance_counts = spine_provenance_counts( + pool_frame, + boundary="stacked H5 fixture provenance counts", + ) + else: + pool_frame = _pool_frame_with_object_strings_on_every_entity() + dag = None + provenance_counts = {"household": {"rows": pool_frame.n("household")}} geography_assignment = ( _fixture_geography_assignment(pool_frame.table("household")) if stacked @@ -741,7 +935,7 @@ def _write_ready_pool( }, }, "agreement_gate": agreement_gate, - "provenance_counts": {"household": {"rows": 3}}, + "provenance_counts": provenance_counts, "pool_h5": { "path": str(pool_path.resolve()), "sha256": _sha256(pool_path), @@ -756,7 +950,7 @@ def _write_ready_pool( }, } if stacked: - dag = _canonical_stacked_late_dag_receipt() + assert dag is not None assert geography_assignment is not None sampling, stack_manifest = _fixture_stacked_sampling(sample_fraction) transition_authority = ( @@ -832,15 +1026,14 @@ def _write_gate_failed_pool(tmp_path: Path) -> Path: manifest = json.loads(manifest_path.read_text(encoding="utf-8")) diagnostics_path = Path(manifest["agreement_diagnostics"]["path"]) diagnostics = json.loads(diagnostics_path.read_text(encoding="utf-8")) - failed_gate = { + # The scoring loader authenticates the canonical terminal gate set, so the + # failed fixture must carry the real gates with one of them flipped. + failed_gate = json.loads(json.dumps(manifest["terminal_gates"])) + failed_gate["passed"] = False + failed_gate["gates"]["us_stacked_completeness"] = { "passed": False, - "gates": { - "us_spine_agreement": { - "passed": False, - "failures": ["fixture terminal failure"], - "details": {"fixture": False}, - } - }, + "failures": ["fixture terminal failure"], + "details": {"fixture": False}, } manifest.update( { @@ -864,28 +1057,44 @@ def _write_gate_failed_pool(tmp_path: Path) -> Path: def _canonical_late_calibration_owner_receipt( + frame: Frame, spec: post_transfer_calibration_runtime.PostTransferCalibrationSpec, ) -> dict[str, object]: - values = np.asarray( - [10.0, 20.0, 30.0, 40.0, 50.0, 100.0, 200.0, 300.0, 400.0, 500.0] + table = frame.table(spec.entity) + channel = table[support_channel_column(spec.entity)].astype(str) + clone_index = pd.to_numeric( + table[support_clone_index_column(spec.entity)], + errors="raise", + ) + reference = (channel.eq(BASE_ASEC_SUPPORT_CHANNEL) & clone_index.eq(0)).to_numpy( + dtype=bool ) - weights = np.asarray([2.0, 3.0, 5.0, 5.0, 5.0, 4.0, 4.0, 4.0, 4.0, 4.0]) - entity_ids = np.arange(1, len(values) + 1) - reference = np.asarray([True] * 5 + [False] * 5) - recipient = ~reference + recipient = ( + channel.eq(stacked_spine_module.ACS_STACKED_SUPPORT_CHANNEL) & clone_index.eq(0) + ).to_numpy(dtype=bool) + weights = np.asarray( + frame.resolve_weights(spec.entity).values, + dtype=np.float64, + ) + entity_ids = table[frame.schema.entity_id_column(spec.entity)].to_numpy(copy=False) constrained = spec.special_constraint != "none" - result = post_transfer_calibration_runtime.calibrate_post_transfer_values( - values, - weights, - entity_ids, - spec=spec, + application = post_transfer_calibration_runtime.apply_post_transfer_calibration( + frame, + entity=spec.entity, + family=spec.family, + target=spec.target, reference_rows=reference, recipient_rows=recipient, mutable_rows=recipient, allowed_carrier_rows=recipient if constrained else None, addition_candidate_rows=recipient if constrained else None, ) - calibration = result.receipt + result_values = application.frame.table(spec.entity)[spec.target].to_numpy( + dtype=np.float64, + copy=True, + ) + table[spec.target] = result_values + calibration = application.receipt scope = calibration["scope"] constraint: dict[str, object] = {"constraint": spec.special_constraint} if spec.special_constraint == "adult_care_qualifying_one_per_tax_unit": @@ -926,13 +1135,13 @@ def _canonical_late_calibration_owner_receipt( ), "reference_output_values_sha256": ( stacked_spine_module._post_transfer_float64_sha256( - result.values[reference], + result_values[reference], boundary="synthetic reference calibration output", ) ), "recipient_output_values_sha256": ( stacked_spine_module._post_transfer_float64_sha256( - result.values[recipient], + result_values[recipient], boundary="synthetic recipient calibration output", ) ), @@ -958,7 +1167,7 @@ def _canonical_late_calibration_owner_receipt( def _canonical_pregnancy_structural_receipt() -> dict[str, object]: - policy = acs_transfer_module.acs_transfer_execution_contract_identity( + policy = acs_transfer_runtime.acs_transfer_execution_contract_identity( targets=("is_pregnant",), derive_schedule_d=False, )["structural_target_policies"]["is_pregnant"] @@ -986,11 +1195,10 @@ def _canonical_pregnancy_structural_receipt() -> dict[str, object]: } -@cache -def _cached_canonical_stacked_late_dag_receipt( - _fit_jobs: str | None, - _predict_workers: str | None, - _cpu_count: int | None, +def _canonical_stacked_late_dag_receipt( + frame: Frame, + *, + immigration_reconciliation: dict[str, object], ) -> dict[str, object]: """Build a signed fixture receipt over the live canonical contracts.""" @@ -1041,17 +1249,80 @@ def _cached_canonical_stacked_late_dag_receipt( "sha256" ] ) + immigration_reconciliation = stacked_spine_module._json_ready( + immigration_reconciliation + ) + immigration_recipient_rows = int(immigration_reconciliation["mutable_rows"]) + immigration_producer_rows = int(immigration_reconciliation["immutable_rows"]) group_receipts: dict[str, object] = {} for group in stacked_spine_module.CANONICAL_US_LATE_TRANSFER_GROUPS: + is_immigration_group = ( + group.entity == "person" and group.family == "source_operator_immigration" + ) group_targets = { f"{group.entity}/{group.family}/{target}": { - "authorized_null_rows": 0, - "imputed_rows": 0, + **( + { + "producer_roles": ["asec_source"], + "producer_rows": immigration_producer_rows, + } + if is_immigration_group + else {} + ), + "authorized_null_rows": ( + immigration_recipient_rows if is_immigration_group else 0 + ), + "imputed_rows": ( + immigration_recipient_rows if is_immigration_group else 0 + ), "unmodeled_rows": 0, "residual_null_rows": 0, } for target in group.targets } + if is_immigration_group: + required_predictors, _optional_predictors = ( + stacked_spine_module._acs_pattern_predictor_authority( + entity=group.entity, + family_targets=group.targets, + ) + ) + pattern = acs_transfer_runtime.AcsTransferPattern( + name=acs_transfer_runtime._pattern_name(0, ()), + observed_optional_predictors=(), + predictors=required_predictors, + seed=0, + weight_kind="design", + donor_rows=immigration_producer_rows, + recipient_rows=immigration_recipient_rows, + target_regimes=tuple( + (model_target, "positive_only") + for model_target in acs_transfer_runtime._model_target_names( + group.targets + ) + ), + ) + for target in group.targets: + key = f"{group.entity}/{group.family}/{target}" + record = acs_transfer_runtime.AcsImputedInput( + column=target, + entity=group.entity, + family=group.family, + donor_spine="synthetic_stacked_h5_fixture", + donor_channel="asec", + predictors=pattern.predictors, + seed=pattern.seed, + weight_kind=pattern.weight_kind, + patterns=(pattern,), + imputed_recipient_rows=immigration_recipient_rows, + reconciliation=immigration_reconciliation, + ) + group_targets[key]["qrf_pattern_evidence"] = ( + stacked_spine_module._acs_imputed_pattern_evidence(record) + ) + group_targets[key]["post_transfer_reconciliation"] = dict( + immigration_reconciliation + ) pregnancy_key = f"{group.entity}/{group.family}/is_pregnant" if pregnancy_key in group_targets: group_targets[pregnancy_key]["structural_policy"] = ( @@ -1060,7 +1331,7 @@ def _cached_canonical_stacked_late_dag_receipt( calibrated_keys = sorted(set(group_targets) & set(late_specs)) for key in calibrated_keys: group_targets[key]["post_transfer_calibration"] = ( - _canonical_late_calibration_owner_receipt(late_specs[key]) + _canonical_late_calibration_owner_receipt(frame, late_specs[key]) ) group_receipts[group.name] = { "producer": group.name, @@ -1309,20 +1580,290 @@ def _cached_canonical_stacked_late_dag_receipt( return receipt -def _canonical_stacked_late_dag_receipt() -> dict[str, object]: - return deepcopy( - _cached_canonical_stacked_late_dag_receipt( - worker_identity_module.os.environ.get("POPULACE_FIT_N_JOBS"), - worker_identity_module.os.environ.get("POPULACE_FIT_PREDICT_WORKERS"), - worker_identity_module.os.cpu_count(), +def _strip_immigration_transfer_evidence(transfer: dict[str, object]) -> None: + """Restore the pre-schema-10 shape of the immigration transfer targets.""" + + branch_only = ( + "qrf_pattern_evidence", + "post_transfer_reconciliation", + "producer_roles", + "producer_rows", + ) + + def historical(target_key: str, receipt: dict[str, object]) -> None: + entity, family, target = target_key.split("/", 2) + if not stacked_spine_module._is_immigration_transfer_target( + entity=entity, family=family, target=target + ): + return + for key in branch_only: + receipt.pop(key, None) + receipt["authorized_null_rows"] = 0 + receipt["imputed_rows"] = 0 + + for group in transfer["groups"].values(): + for target_key, receipt in group["targets"].items(): + historical(target_key, receipt) + for target_key, receipt in transfer["targets"].items(): + historical(target_key, receipt) + + +_IMMIGRATION_SOURCE_PRODUCER = "source:with_us_immigration_inputs" +_IMMIGRATION_TRANSFER_PRODUCER = "transfer:person/source_operator_immigration" + +#: One wage-earning CPS ASEC person record: the raw fields the immigration +#: source producer reads that the persisted fixture stack does not carry. +#: WSAL_VAL and SEMP_VAL are the schema-16 requirement #767 replaced with +#: A_LFSR, which only today's contract reads; both contracts read the rest. +_ASEC_PERSON_RECORD = { + "WSAL_VAL": 52_000.0, + "SEMP_VAL": 0.0, + "A_LFSR": 1.0, + "A_HSCOL": 0.0, + "A_MARITL": 7.0, + "A_MJOCC": 2.0, + "A_SPOUSE": 0.0, + "CAID": 2.0, + "CHAMPVA": 2.0, + "IHSFLG": 2.0, + "MCARE": 2.0, + "MIL": 2.0, + "PEAFEVER": 2.0, + "PEIO1COW": 4.0, + "PEN_SC1": 0.0, + "PEN_SC2": 0.0, + "RESNSS1": 0.0, + "RESNSS2": 0.0, + "SPM_CAPHOUSESUB": 0.0, + "SSI_YN": 2.0, + "SS_YN": 2.0, +} + + +def _historically_changed_late_producers() -> tuple[str, ...]: + """Name every producer whose schema-16 contract differs from today's.""" + + legacy = late_producer_registry_module.legacy_us_late_producer_contracts() + live = stacked_spine_module.CANONICAL_US_LATE_PRODUCER_REGISTRY + return tuple( + name for name in legacy.schedule.order if legacy.registry[name] != live[name] + ) + + +def _late_dag_input_frame(manifest: Mapping[str, object]) -> Frame: + """The fixture stack plus what the immigration producers' contracts read. + + This is the frame ``_write_ready_pool(stacked=True)`` hands the fixture + DAG builder, plus exactly the inputs the immigration source and transfer + contracts (schema-16 and current) declare that the fixture stack does not + carry: the raw ASEC person record on the ASEC-origin rows (the ACS spine + has no CPS fields, so its rows stay null), ``is_female``, the person and + household spine source ids, ``state_fips``, ACS ``TYPEHUGQ``, and the + stack and PUF-attachment manifests in frame metadata. The transfer's + optional predictors stay absent, so its row seals their tolerated + absences exactly as the builder does. One frame serves every sealed row, + so a row's input and output evidence are computed on the same frame. + """ + + frame, _reconciliation = _stacked_pool_frame_with_live_immigration() + assert ( + spine_provenance_counts(frame, boundary="late DAG input fixture") + == manifest["provenance_counts"] + ), "the rebuilt fixture stack is not the one this pool persisted" + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + person = tables["person"] + asec_person = ( + person[support_channel_column("person")] + .astype(str) + .eq(BASE_ASEC_SUPPORT_CHANNEL) + .to_numpy() + ) + for column, value in _ASEC_PERSON_RECORD.items(): + person[column] = np.where(asec_person, value, np.nan) + person["is_female"] = (np.arange(len(person)) % 2).astype(np.float64) + person["person_spine_source_id"] = person[ + support_source_id_column("person") + ].to_numpy(dtype=np.float64) + household = tables["household"] + acs_household = ( + household[support_channel_column("household")] + .astype(str) + .eq(stacked_spine_module.ACS_STACKED_SUPPORT_CHANNEL) + .to_numpy() + ) + # Every fixture household is in PUMA 0600101 / county 06001. + household["state_fips"] = 6.0 + household["household_spine_source_id"] = household[ + support_source_id_column("household") + ].to_numpy(dtype=np.float64) + household["TYPEHUGQ"] = np.where(acs_household, 1.0, np.nan) + weights = {entity: frame.weights_for(entity) for entity in frame.weighted_entities} + _sampling, stack_manifest = _fixture_stacked_sampling(1.0) + # The attachment step records its manifest under the controls the + # fixture's primary-PUF resource receipts declare (fraction 1, seed 578). + attached = puf_support_runtime._attach_clone_arm_to_seeded_sample( + frame, + fraction=1.0, + seed=578, + ) + return Frame( + tables, + frame.schema, + weights, + frame.strata, + mass_log=frame.mass_log, + metadata={ + **frame.metadata, + stacked_spine_module.STACKED_SPINE_MANIFEST_KEY: stack_manifest, + puf_support_runtime.PUF_CLONE_ATTACHMENT_MANIFEST_KEY: attached.metadata[ + puf_support_runtime.PUF_CLONE_ATTACHMENT_MANIFEST_KEY + ], + }, + ) + + +def _seal_execution_rows( + dag: dict[str, object], + contracts: Mapping[str, ProducerContract], + frame: Frame, +) -> None: + """Seal the named rows' surfaces exactly as ``run_stacked_late_producer_dag``. + + Readiness counts, tolerated-absence receipts, content-bound declared-input + evidence, and output evidence all come from the production builders, + computed on ``frame`` against the given contract and the row's own sealed + available-input receipts and callback receipt. Callers re-chain digests. + """ + + for row in dag["execution"]: + contract = contracts.get(row["producer"]) + if contract is None: + continue + available = row["available_input_receipts"] + unfilled_rows, invalid_rows = stacked_spine_module._late_input_readiness_rows( + frame, + contract, + available_input_receipts=available, + ) + row["declared_inputs"] = stacked_spine_module._late_declared_input_evidence( + frame, + contract, + available_input_receipts=available, + unfilled_rows=unfilled_rows, + invalid_rows=invalid_rows, ) + row["declared_absence_receipts"] = { + receipt_id: dict(receipt) + for receipt_id, receipt in ( + stacked_spine_module._late_declared_absence_receipts( + contract, + unfilled_rows, + invalid_rows=invalid_rows, + ).items() + ) + } + row["output_surface"] = [ + stacked_spine_module._late_output_column_evidence( + frame, + output=output, + producer_receipt=row["producer_receipt"], + ) + for output in contract.outputs + ] + + +def _rechain_late_dag(dag: dict[str, object]) -> None: + """Re-derive every row's surface digests, the chain, and the receipt digest. + + The chain starts from the genesis digest of the receipt's own version, so + this serves both a current receipt and a version-3 (schema-9) one. + """ + + previous_sha256 = stacked_spine_module._late_execution_genesis_sha256( + producer_schedule_sha256=dag["producer_schedule"]["payload_sha256"], + input_frame_sha256=dag["input_frame_sha256"], + receipt_schema_version=dag["version"], + ) + for row in dag["execution"]: + row["input_surface_sha256"] = _json_sha256(row["declared_inputs"]) + row["output_surface_sha256"] = _json_sha256(row["output_surface"]) + row["producer_receipt_sha256"] = _json_sha256(row["producer_receipt"]) + row["previous_execution_sha256"] = previous_sha256 + row.pop("sha256", None) + row["sha256"] = stacked_spine_module._canonical_sha256(row) + previous_sha256 = row["sha256"] + dag["execution_chain_sha256"] = previous_sha256 + dag.pop("sha256", None) + dag["sha256"] = stacked_spine_module._canonical_sha256(dag) + + +def _reseal_current_pool_rows( + manifest_path: Path, + contracts: Mapping[str, ProducerContract], +) -> None: + """Re-seal a current pool's named rows and re-chain its late DAG.""" + + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + dag = manifest["stage_receipts"]["impute"]["stacked_late_producer_dag"] + _seal_execution_rows(dag, contracts, _late_dag_input_frame(manifest)) + _rechain_late_dag(dag) + manifest["late_producer_transition_authority_sha256"] = ( + stacked_spine_module._late_producer_transition_authority_receipt(dag)["sha256"] ) + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + + +def _immigration_row_contracts( + *, historical: Collection[str] +) -> dict[str, ProducerContract]: + """Every historically changed producer's contract, legacy only if named.""" + + legacy = late_producer_registry_module.legacy_us_late_producer_contracts() + live = stacked_spine_module.CANONICAL_US_LATE_PRODUCER_REGISTRY + changed = _historically_changed_late_producers() + unknown = set(historical) - set(changed) + if unknown: + raise ValueError(f"{sorted(unknown)} have no distinct historical contract.") + return { + name: legacy.registry[name] if name in historical else live[name] + for name in changed + } + + +def _late_row(manifest_path: Path, producer: str) -> dict[str, object]: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + dag = manifest["stage_receipts"]["impute"]["stacked_late_producer_dag"] + return next(row for row in dag["execution"] if row["producer"] == producer) + + +def _declared_input_columns(row: Mapping[str, object]) -> set[str]: + return { + column["column"] + for declared in row["declared_inputs"] + for alternative in declared["evidence"]["alternatives"] + for column in alternative + } def _rewrite_as_legacy_relocated_worker_pool( manifest_path: Path, + *, + historical_contract: bool = True, + current_surface_producers: Collection[str] = (), ) -> tuple[dict[str, object], dict[str, object]]: - """Re-sign the tiny fixture with the frozen schema-9 worker binding.""" + """Re-sign the tiny fixture with the frozen schema-9 worker binding. + + With ``historical_contract`` (the default), every producer whose + schema-16 contract differs from today's has its execution row sealed by + the production row builder against that historical contract, as a real + schema-9 pool's row was. ``current_surface_producers`` seals the named + ones against today's contract instead: the stale surface the loader must + refuse. + + ``historical_contract=False`` keeps today's schedule, operator order, and + immigration evidence and only lowers the version numbers: the forgery a + real schema-9 pool can never be, which the loader must refuse. + """ manifest = json.loads(manifest_path.read_text(encoding="utf-8")) dag = manifest["stage_receipts"]["impute"]["stacked_late_producer_dag"] @@ -1406,24 +1947,57 @@ def _rewrite_as_legacy_relocated_worker_pool( available ) - schedule = dag["producer_schedule"] - schedule["schema_version"] = 16 - schedule["execution_receipt_contract"]["version"] = 3 - schedule["execution_receipt_contract"]["transition_authority"]["version"] = 1 - schedule_payload = { - key: value - for key, value in schedule.items() - if key - not in { - "payload_sha256", - "producer_count", - "source_producer_count", - "transfer_group_count", - "transfer_target_count", - "status", - } - } - schedule["payload_sha256"] = _json_sha256(schedule_payload) + # A schema-9 pool sealed the historical schedule, not a version-downgraded + # copy of today's: install the frozen content real pools carry, and give + # the immigration targets the ordinary transfer receipts they had before + # the paired reconciliation and QRF evidence existed. + if historical_contract: + schedule = json.loads( + json.dumps( + dict( + late_producer_registry_module.legacy_us_late_producer_schedule_receipt() + ) + ) + ) + dag["producer_schedule"] = schedule + _strip_immigration_transfer_evidence(dag["post_puf_transfer"]) + for row in dag["execution"]: + group_receipt = dag["post_puf_transfer"]["groups"].get(row["producer"]) + if group_receipt is not None: + row["producer_receipt"] = json.loads(json.dumps(group_receipt)) + # Seal every row whose schema-16 contract differs from today's with + # the production readiness and evidence builders, run against that + # historical contract on the fixture's own late-DAG input frame, so + # the rows carry the surfaces a schema-9 build sealed. + _seal_execution_rows( + dag, + _immigration_row_contracts( + historical=set(_historically_changed_late_producers()) + - set(current_surface_producers) + ), + _late_dag_input_frame(manifest), + ) + manifest["operator_order"] = list(h5_io._SCHEMA9_STACKED_POOL_OPERATOR_ORDER) + else: + schedule = dag["producer_schedule"] + schedule["schema_version"] = 16 + schedule["execution_receipt_contract"]["version"] = 3 + schedule["execution_receipt_contract"]["transition_authority"]["version"] = 1 + schedule["payload_sha256"] = _json_sha256( + { + key: value + for key, value in schedule.items() + if key + not in { + "payload_sha256", + "producer_count", + "source_producer_count", + "transfer_group_count", + "transfer_target_count", + "status", + } + } + ) dag["post_puf_transfer"]["producer_schedule"] = json.loads(json.dumps(schedule)) dag["post_puf_transfer"]["authority"] = ( stacked_spine_module._legacy_stacked_authority_receipt() @@ -1860,6 +2434,56 @@ def test_ready_stacked_pool_loader_binds_terminal_gate_aliases( frame, manifest, _ = load_simulation_ready_us_multispine_pool(manifest_path) + positive_draws = tuple( + draw + for draw in immigration_runtime.us_immigration_controls().humanitarian + if draw.target > 0 + ) + expected_rows = 1 + len(positive_draws) + expected_weights = np.asarray( + [1.0, *(float(draw.target) for draw in positive_draws)], + dtype=np.float64, + ) + assert all(frame.n(entity) == expected_rows for entity in US_SCHEMA.entities) + np.testing.assert_array_equal( + frame.weights_for("household").values, + expected_weights, + ) + person = frame.table("person") + assert person.loc[0, "PRCITSHP"] == 1 + assert pd.isna(person.loc[0, "CIT"]) + assert person.loc[0, "immigration_status_str"] == "CITIZEN" + assert person.loc[1:, "CIT"].eq(5).all() + assert person.loc[1:, ["POBP", "YOEP"]].notna().all().all() + for entity in US_SCHEMA.group_entities: + np.testing.assert_array_equal( + person[US_SCHEMA.membership_column(entity)], + frame.table(entity)[US_SCHEMA.id_column(entity)], + ) + for draw in positive_draws: + emitted = immigration_runtime.us_immigration_humanitarian_draw_mask( + frame, + draw, + ) + assert int(emitted.sum()) == 1 + assert float(frame.resolve_weights("person").values[emitted].sum()) == float( + draw.target + ) + for entity in US_SCHEMA.entities: + assert manifest["provenance_counts"][entity] == { + "rows": expected_rows, + "by_source_channel": { + BASE_ASEC_SUPPORT_CHANNEL: 1, + stacked_spine_module.ACS_STACKED_SUPPORT_CHANNEL: len(positive_draws), + }, + "by_clone_index": {"0": expected_rows}, + "by_source_channel_and_clone_index": { + BASE_ASEC_SUPPORT_CHANNEL: {"0": 1}, + stacked_spine_module.ACS_STACKED_SUPPORT_CHANNEL: { + "0": len(positive_draws) + }, + }, + } assert manifest["terminal_gates"] == manifest["agreement_gate"] assert manifest["schema_version"] == US_MULTISPINE_POOL_MANIFEST_SCHEMA_VERSION assert ( @@ -2007,6 +2631,61 @@ def test_ready_stacked_pool_loader_rejects_boolean_sampling_aliases( load_simulation_ready_us_multispine_pool(manifest_path) +def test_ready_stacked_pool_loader_replays_immigration_status_against_h5( + tmp_path: Path, +) -> None: + """An ordinary H5 re-hash cannot forge the sealed live immigration proof.""" + + pytest.importorskip("tables") + manifest_path = _write_ready_pool(tmp_path, stacked=True) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + pool_path = Path(manifest["pool_h5"]["path"]) + with pd.HDFStore(pool_path, mode="a") as store: + person = h5_io.read_frame_table(store, "person") + parole_rows = person["immigration_status_str"].eq("PAROLED_ONE_YEAR") + assert parole_rows.any() + person.loc[parole_rows.idxmax(), "immigration_status_str"] = ( + "LEGAL_PERMANENT_RESIDENT" + ) + h5_io.put_frame_table( + store, + "person", + person, + preferred_format="fixed", + ) + + # Re-sign only the ordinary byte-level artifact envelope. The immutable + # late-transition receipt still describes the untampered live population. + manifest["pool_h5"]["sha256"] = _sha256(pool_path) + manifest["pool_h5"]["size_bytes"] = pool_path.stat().st_size + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + + with pytest.raises( + ValueError, + match="differs from the live ASEC/ACS weighted population", + ): + load_simulation_ready_us_multispine_pool(manifest_path) + + +def test_ready_stacked_pool_loader_requires_immigration_terminal_gate( + tmp_path: Path, +) -> None: + pytest.importorskip("tables") + manifest_path = _write_ready_pool(tmp_path, stacked=True) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + diagnostics_path = Path(manifest["agreement_diagnostics"]["path"]) + diagnostics = json.loads(diagnostics_path.read_text(encoding="utf-8")) + for payload in (manifest, diagnostics): + payload["terminal_gates"]["gates"].pop("immigration_composition") + payload["agreement_gate"]["gates"].pop("immigration_composition") + diagnostics_path.write_text(json.dumps(diagnostics), encoding="utf-8") + manifest["agreement_diagnostics"]["sha256"] = _sha256(diagnostics_path) + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + + with pytest.raises(ValueError, match="canonical terminal gate set"): + load_simulation_ready_us_multispine_pool(manifest_path) + + def test_ready_stacked_pool_loader_binds_h5_cd_vintage_attrs( tmp_path: Path, ) -> None: @@ -2075,7 +2754,11 @@ def test_ready_stacked_pool_loader_rejects_divergent_clone_geography( tmp_path: Path, ) -> None: pytest.importorskip("tables") - manifest_path = _write_ready_pool(tmp_path, stacked=True) + manifest_path = _write_ready_pool( + tmp_path, + stacked=True, + include_zero_weight_clone=True, + ) manifest = json.loads(manifest_path.read_text(encoding="utf-8")) pool_path = Path(manifest["pool_h5"]["path"]) clone_column = support_clone_index_column("household") @@ -2117,7 +2800,11 @@ def test_scoring_pool_loader_authenticates_failed_stacked_terminal_receipt( ) ) - assert frame.n("household") == 3 + expected_rows = 1 + sum( + draw.target > 0 + for draw in immigration_runtime.us_immigration_controls().humanitarian + ) + assert frame.n("household") == expected_rows assert loaded_manifest["status"] == "gate_failed" assert loaded_manifest["simulation_ready"] is False assert loaded_manifest["terminal_gates"]["passed"] is False @@ -2148,7 +2835,7 @@ def test_scoring_pool_loader_authenticates_failed_stacked_terminal_receipt( "failure_count": 1, "failures": [ { - "gate": "us_spine_agreement", + "gate": "us_stacked_completeness", "message": "fixture terminal failure", } ], @@ -2181,7 +2868,12 @@ def test_current_worker_authentication_ignores_audit_alias_relocation( manifest_path ) - assert frame.n("household") == 3 + positive_draws = [ + draw + for draw in immigration_runtime.us_immigration_controls().humanitarian + if draw.target > 0 + ] + assert frame.n("household") == 1 + len(positive_draws) assert authenticated.sha256 == pool_sha256 assert ( manifest["worker_execution_authentication"]["audit_aliases"] @@ -2216,6 +2908,27 @@ def test_scoring_loader_accepts_legacy_worker_alias_relocation_only( recorded_worker, semantic_identity = _rewrite_as_legacy_relocated_worker_pool( manifest_path ) + # The sealed rows are the schema-16 ones, not today's: the immigration + # source still reads WSAL_VAL/SEMP_VAL (#767 replaced them with A_LFSR), + # and the immigration transfer lacks the seven requirements #767 added. + legacy = late_producer_registry_module.legacy_us_late_producer_contracts() + live = stacked_spine_module.CANONICAL_US_LATE_PRODUCER_REGISTRY + source_row = _late_row(manifest_path, _IMMIGRATION_SOURCE_PRODUCER) + transfer_row = _late_row(manifest_path, _IMMIGRATION_TRANSFER_PRODUCER) + assert ( + len(source_row["declared_inputs"]) + == 57 + == len(legacy.registry[_IMMIGRATION_SOURCE_PRODUCER].inputs) + ) + assert ( + len(transfer_row["declared_inputs"]) + == 93 + == len(legacy.registry[_IMMIGRATION_TRANSFER_PRODUCER].inputs) + ) + assert len(live[_IMMIGRATION_SOURCE_PRODUCER].inputs) == 56 + assert len(live[_IMMIGRATION_TRANSFER_PRODUCER].inputs) == 100 + assert {"WSAL_VAL", "SEMP_VAL"} <= _declared_input_columns(source_row) + assert "A_LFSR" not in _declared_input_columns(source_row) manifest_sha256 = _sha256(manifest_path) attestation_path = _write_legacy_worker_attestation( manifest_path, @@ -2244,7 +2957,12 @@ def test_scoring_loader_accepts_legacy_worker_alias_relocation_only( "compatibility_attestation_sha256": _sha256(attestation_path), "purpose": "scoring_only", } - assert frame.n("household") == 3 + positive_draws = [ + draw + for draw in immigration_runtime.us_immigration_controls().humanitarian + if draw.target > 0 + ] + assert frame.n("household") == 1 + len(positive_draws) assert manifest["status"] == "gate_failed" assert manifest["simulation_ready"] is False assert authenticated.sha256 == manifest["pool_h5"]["sha256"] @@ -2272,6 +2990,105 @@ def test_scoring_loader_accepts_legacy_worker_alias_relocation_only( ) +def test_scoring_loader_refuses_legacy_rows_sealed_with_todays_surfaces( + tmp_path: Path, +) -> None: + """An attested schema-9 pool must carry the rows schema 16 sealed.""" + + pytest.importorskip("tables") + manifest_path = _write_gate_failed_pool(tmp_path) + recorded_worker, semantic_identity = _rewrite_as_legacy_relocated_worker_pool( + manifest_path, + current_surface_producers=(_IMMIGRATION_SOURCE_PRODUCER,), + ) + source_row = _late_row(manifest_path, _IMMIGRATION_SOURCE_PRODUCER) + assert len(source_row["declared_inputs"]) == 56 + attestation_path = _write_legacy_worker_attestation( + manifest_path, + recorded_worker=recorded_worker, + semantic_identity=semantic_identity, + ) + + with pytest.raises(ValueError, match="exact 57-input readiness surface"): + load_authenticated_us_multispine_pool_for_scoring( + manifest_path, + expected_manifest_sha256=_sha256(manifest_path), + worker_identity_attestation=attestation_path, + ) + + +def test_current_loader_refuses_rows_sealed_with_historical_surfaces( + tmp_path: Path, +) -> None: + """History authenticates only under an attestation, never on the current path.""" + + pytest.importorskip("tables") + (tmp_path / "control").mkdir() + control_path = _write_gate_failed_pool(tmp_path / "control") + # Control: re-sealing with today's contracts reproduces a loadable pool, + # so the refusal below comes from the historical surfaces, not the helper. + _reseal_current_pool_rows(control_path, _immigration_row_contracts(historical=())) + load_authenticated_us_multispine_pool_for_scoring(control_path) + + (tmp_path / "historical").mkdir() + manifest_path = _write_gate_failed_pool(tmp_path / "historical") + _reseal_current_pool_rows( + manifest_path, + _immigration_row_contracts(historical=_historically_changed_late_producers()), + ) + source_row = _late_row(manifest_path, _IMMIGRATION_SOURCE_PRODUCER) + assert len(source_row["declared_inputs"]) == 57 + + with pytest.raises(ValueError, match="exact 56-input readiness surface"): + load_authenticated_us_multispine_pool_for_scoring(manifest_path) + + +def test_scoring_loader_refuses_version_downgraded_current_contract( + tmp_path: Path, +) -> None: + """Lowered version numbers do not make today's contract a schema-9 pool.""" + + pytest.importorskip("tables") + manifest_path = _write_gate_failed_pool(tmp_path) + recorded_worker, semantic_identity = _rewrite_as_legacy_relocated_worker_pool( + manifest_path, + historical_contract=False, + ) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + dag = manifest["stage_receipts"]["impute"]["stacked_late_producer_dag"] + assert dag["producer_schedule"]["payload_sha256"] != ( + late_producer_registry_module.LEGACY_SCHEMA16_LATE_PRODUCER_SCHEDULE_PAYLOAD_SHA256 + ) + attestation_path = _write_legacy_worker_attestation( + manifest_path, + recorded_worker=recorded_worker, + semantic_identity=semantic_identity, + ) + + with pytest.raises(ValueError, match="canonical late-DAG operator order"): + load_authenticated_us_multispine_pool_for_scoring( + manifest_path, + expected_manifest_sha256=_sha256(manifest_path), + worker_identity_attestation=attestation_path, + ) + + # Borrowing the historical operator order does not rescue it: the sealed + # schedule is still today's content under yesterday's version numbers. + manifest["operator_order"] = list(h5_io._SCHEMA9_STACKED_POOL_OPERATOR_ORDER) + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + attestation_path = _write_legacy_worker_attestation( + manifest_path, + recorded_worker=recorded_worker, + semantic_identity=semantic_identity, + ) + with pytest.raises(ValueError, match="DAG schedule is not canonical"): + load_authenticated_us_multispine_pool_for_scoring( + manifest_path, + expected_manifest_sha256=_sha256(manifest_path), + worker_identity_attestation=attestation_path, + ) + + @pytest.mark.usefixtures("live_worker_identity") def test_scoring_loader_requires_complete_schema_nine_stacked_envelope( tmp_path: Path, @@ -2591,7 +3408,14 @@ def test_denied_gate_failed_pool_is_available_only_for_scoring( frame, loaded_manifest, _ = load_authenticated_us_multispine_pool_for_scoring( manifest_path ) - assert frame.n("household") == 3 + # The stacked fixture carries one base row plus one row per positive + # canonical humanitarian draw (see _stacked_pool_frame_with_live_immigration). + positive_draws = [ + draw + for draw in immigration_runtime.us_immigration_controls().humanitarian + if draw.target > 0 + ] + assert frame.n("household") == 1 + len(positive_draws) assert loaded_manifest["status"] == "gate_failed" refused_loaders = ( diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/engine_free/us/test_us_multispine_pool_tool.py index 5403902555..199aebdf82 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_multispine_pool_tool.py @@ -776,9 +776,15 @@ def test_stacked_resume_rejects_noncanonical_post_puf_transfer_receipt( pool_tool: ModuleType, tmp_path: Path, ) -> None: + asec_source, acs_source = _humanitarian_weighted_source_frames( + asec_count=10, + acs_count=120, + sample_fraction=0.10, + sample_seed=578, + ) stack = pool_tool.assemble_stacked_spine( - _many_household_source_frame(), - _many_household_source_frame(measured_offset=1_000.0), + asec_source, + acs_source, sample_fraction=0.10, sample_seed=578, ) @@ -1007,7 +1013,7 @@ def deterministic_fixture_h5( manifest = pool_tool._read_json_object(outputs.manifest) diagnostics = pool_tool._read_json_object(outputs.agreement_diagnostics) assert pool_tool.POOL_MANIFEST_SCHEMA_VERSION == 10 - assert pool_tool.POOL_STAGE_CHECKPOINT_MATERIALIZER_VERSION == 7 + assert pool_tool.POOL_STAGE_CHECKPOINT_MATERIALIZER_VERSION == 8 assert manifest["schema_version"] == 4 assert diagnostics["schema_version"] == 4 assert "materializer_version" not in manifest["pool_h5"] @@ -2326,7 +2332,7 @@ def test_pool_checkpoint_store_round_trips_nullable_boolean_families( manifest = pool_tool._read_json_object( cold_store.checkpoint_manifest_path(stage) ) - assert manifest["materializer_version"] == 7 + assert manifest["materializer_version"] == 8 loaded = pool_tool.load_frame_checkpoint(path).frame if stage == "assembled": assert "fixture_declared_boolean" not in loaded.person @@ -2347,11 +2353,11 @@ def test_pool_checkpoint_store_round_trips_nullable_boolean_families( assert resumed.frame.person["fixture_declared_boolean"].isna().sum() == 1 -def test_simulated_v7_checkpoint_accepts_both_string_encodings_without_rewrite( +def test_simulated_v8_checkpoint_accepts_both_string_encodings_without_rewrite( pool_tool: ModuleType, tmp_path: Path, ) -> None: - """V7 authenticates both physical string encodings as one logical frame.""" + """V8 authenticates both physical string encodings as one logical frame.""" pytest.importorskip("h5py") checkpoint_root = tmp_path / "checkpoints" @@ -2363,7 +2369,7 @@ def test_simulated_v7_checkpoint_accepts_both_string_encodings_without_rewrite( loaded = pool_tool.load_frame_checkpoint(checkpoint_path) canonical_v2_bytes = checkpoint_path.read_bytes() canonical_identity = loaded.metadata["identity"] - assert loaded.metadata["materializer_version"] == 7 + assert loaded.metadata["materializer_version"] == 8 assert any( column["dtype"] == str(CANONICAL_STRING_DTYPE) for columns in loaded.metadata["frame_schema"]["entities"].values() @@ -2394,7 +2400,7 @@ def test_simulated_v7_checkpoint_accepts_both_string_encodings_without_rewrite( banked_v2_bytes = checkpoint_path.read_bytes() assert banked_v2_bytes != canonical_v2_bytes assert legacy_metadata["identity"] == canonical_identity - assert legacy_metadata["materializer_version"] == 7 + assert legacy_metadata["materializer_version"] == 8 assert any( column["dtype"] == "object" for columns in legacy_metadata["frame_schema"]["entities"].values() @@ -2769,7 +2775,7 @@ def test_tail_support_contract_identity_mutation_rebuilds_pool_checkpoints( assert changed_store.load_deepest() is None -@pytest.mark.parametrize("legacy_version", (1, 2, 3, 4, 5, 6)) +@pytest.mark.parametrize("legacy_version", (1, 2, 3, 4, 5, 6, 7)) def test_legacy_pool_materializer_artifacts_fail_closed_with_named_receipts( pool_tool: ModuleType, monkeypatch: pytest.MonkeyPatch, @@ -2802,9 +2808,9 @@ def test_legacy_pool_materializer_artifacts_fail_closed_with_named_receipts( assert manifest["identity"]["materializer_version"] == legacy_version capsys.readouterr() - assert pool_tool.POOL_STAGE_CHECKPOINT_MATERIALIZER_VERSION == 7 + assert pool_tool.POOL_STAGE_CHECKPOINT_MATERIALIZER_VERSION == 8 current_store = _checkpoint_fixture_store(pool_tool, checkpoint_root) - assert current_store.base_identity["materializer_version"] == 7 + assert current_store.base_identity["materializer_version"] == 8 assert current_store.load_deepest() is None output = capsys.readouterr().out diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_plan.py b/packages/microcosm-build/tests/engine_free/us/test_us_plan.py index 841fd212bc..5eb3093d2e 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_plan.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_plan.py @@ -1301,8 +1301,8 @@ class TestBaseStageSourceClosure: #: Raw columns restored from pinned official sidecars because the frozen #: census_cps inputs never carried them (LKWEEKS only for income year - #: 2022; ED_VAL and PAW_TYP for every pooled year). - SIDECAR_RESTORED_COLUMNS = frozenset({"LKWEEKS", "ED_VAL", "PAW_TYP"}) + #: 2022; ED_VAL, A_LFSR, and PAW_TYP for every pooled year). + SIDECAR_RESTORED_COLUMNS = frozenset({"LKWEEKS", "ED_VAL", "A_LFSR", "PAW_TYP"}) #: Census person columns asec_census_person_columns restores into every #: pooled vintage, by exact PERIDNUM identity, that NO frozen input carries diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_post_export_scoring.py b/packages/microcosm-build/tests/engine_free/us/test_us_post_export_scoring.py index 3c23ead7be..c93a0e35e5 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_post_export_scoring.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_post_export_scoring.py @@ -75,7 +75,7 @@ def test_batched_engines_partition_release_and_order_their_work( validation = consumers["reform_validation"].record() demographics = consumers["demographics"].record() probes = len(us_release_reform_coverage_probes()) - assert smoke["reform_systems"] == smoke["reform_passes"] == probes == 43 + assert smoke["reform_systems"] == smoke["reform_passes"] == probes == 47 assert validation["reform_systems"] == validation["reform_passes"] > 0 assert demographics["reform_systems"] == demographics["reform_passes"] == 0 assert all(engine.reform is None for engine in constructions) @@ -1540,7 +1540,7 @@ def test_guard_sweep_records_and_scores_shipped_reform_passes( ) -> None: sweep = _load_sweep_module() plans, requests, counts = sweep._record_requests(builder) - assert counts["smoke_probes"] == len(us_release_reform_coverage_probes()) == 43 + assert counts["smoke_probes"] == len(us_release_reform_coverage_probes()) == 47 specs = reform_validation_module.load_default_reform_specs(period=builder.PERIOD) assert counts["validation_specs"] == sum(not spec.in_sample for spec in specs) == 52 assert ( @@ -1548,8 +1548,11 @@ def test_guard_sweep_records_and_scores_shipped_reform_passes( == sum(spec.in_sample for spec in specs) == 12 ) + # The microcosm#767 H.R.1 probes add aca_ptc 2026 and 2027 and + # medicaid_enrolled 2027; their snap 2026 key is shared with the Texas + # vehicle probe. assert counts["baseline_keys"] == { - "reform_coverage_smoke": 20, + "reform_coverage_smoke": 23, "reform_validation": 75, "demographics": 1, } @@ -1584,7 +1587,7 @@ def test_guard_sweep_records_and_scores_shipped_reform_passes( assert ( len(requests) == counts["smoke_probes"] + counts["validation_reform_passes"] - == 109 + == 113 ) frame, log = _nested_frame(), _EngineLog() diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_puf_support_base_builder.py b/packages/microcosm-build/tests/engine_free/us/test_us_puf_support_base_builder.py index 31a612a3d7..26e3fe41d1 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_puf_support_base_builder.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_puf_support_base_builder.py @@ -239,6 +239,7 @@ def _education_source() -> pd.DataFrame: "A_LINENO": [1, 2, 1], "PERIDNUM": [f"{value:022d}" for value in (1, 2, 3)], "ED_VAL": [0.0, 500.0, 1_000.0], + "A_LFSR": [1, 3, 7], } ) source.attrs["source_audit"] = {2022: {"rows": 3}} @@ -842,11 +843,13 @@ def test_raw_stage_copy_adds_only_exact_source_mappings( assert frame_identity(source) == before assert "LKWEEKS" not in source.table("person") assert "ED_VAL" not in source.table("person") + assert "A_LFSR" not in source.table("person") assert "PAW_TYP" not in source.table("person") assert raw.table("person")["LKWEEKS"].tolist() == [7.0, -1.0, 12.0] assert raw.table("person")["ED_VAL"].tolist() == [0.0, 500.0, 1_000.0] + assert raw.table("person")["A_LFSR"].tolist() == [1, 3, 7] assert raw.table("person")["PAW_TYP"].tolist() == [0, 1, 2] - assert set(mappings) == {"ED_VAL", "LKWEEKS", "PAW_TYP"} + assert set(mappings) == {"A_LFSR", "ED_VAL", "LKWEEKS", "PAW_TYP"} assert all( mapping["operation"] == "exact_source_join" and mapping["join_keys"] == ["source_year", "PERIDNUM"] @@ -855,6 +858,49 @@ def test_raw_stage_copy_adds_only_exact_source_mappings( builder.assert_operator_free_source_frame(raw, label="raw-stage fixture") +def test_direct_builder_attaches_measured_labor_force_status_without_mutation() -> None: + builder = _load_support_builder_module() + source = _raw_asec_frame() + before = frame_identity(source) + + attached = builder._with_asec_labor_force_status_source( + source, + _education_source(), + ) + + assert frame_identity(source) == before + assert "A_LFSR" not in source.table("person") + assert attached.table("person")["A_LFSR"].tolist() == [1, 3, 7] + assert np.array_equal( + attached.weights_for("household").values, + source.weights_for("household").values, + ) + + +def test_direct_immigration_path_uses_labor_force_status_ephemerally( + monkeypatch: pytest.MonkeyPatch, +) -> None: + builder = _load_support_builder_module() + source = _raw_asec_frame() + observed: list[list[int]] = [] + + def immigration(frame: Frame, *, seed: int, time_period: int) -> Frame: + observed.append(frame.table("person")["A_LFSR"].tolist()) + return frame + + monkeypatch.setattr(builder, "with_us_immigration_inputs", immigration) + result = builder._with_us_immigration_inputs_from_asec_source( + source, + _education_source(), + seed=7, + time_period=2024, + ) + + assert observed == [[1, 3, 7]] + assert "A_LFSR" not in source.table("person") + assert "A_LFSR" not in result.table("person") + + @pytest.mark.parametrize("execution", ["all", "pre_clone_enrichment"]) def test_spm_stage_receives_explicit_asec_source_paths( monkeypatch: pytest.MonkeyPatch, @@ -1004,10 +1050,14 @@ def add_age(frame: Frame, **_kwargs) -> Frame: "with_us_retirement_contribution_inputs", "with_us_retirement_distribution_inputs", "with_us_esi_premium_inputs", - "with_us_immigration_inputs", ) for name in identity_transforms: monkeypatch.setattr(builder, name, lambda value, **_kwargs: value) + monkeypatch.setattr( + builder, + "_with_us_immigration_inputs_from_asec_source", + lambda value, _source, **_kwargs: value, + ) passing_gate = SimpleNamespace(passed=True, failures=(), details={}) for name in ( "us_relationship_inputs_signal_gate", @@ -1032,6 +1082,7 @@ def add_age(frame: Frame, **_kwargs) -> Frame: assert source_checkpoint.path.read_bytes() == baseline_source.path.read_bytes() assert "ED_VAL" not in source_checkpoint.frame.table("person") assert "LKWEEKS" not in source_checkpoint.frame.table("person") + assert "A_LFSR" not in source_checkpoint.frame.table("person") assert "PAW_TYP" not in source_checkpoint.frame.table("person") raw_path = args.checkpoint_dir / builder.ASEC_RAW_STAGE_CHECKPOINT_FILENAME @@ -1039,6 +1090,7 @@ def add_age(frame: Frame, **_kwargs) -> Frame: assert raw_metadata["stage"] == "raw_source_mapping" assert raw.table("person")["LKWEEKS"].tolist() == [7.0, -1.0, 12.0] assert raw.table("person")["ED_VAL"].tolist() == [0.0, 500.0, 1_000.0] + assert raw.table("person")["A_LFSR"].tolist() == [1, 3, 7] assert raw.table("person")["PAW_TYP"].tolist() == [0, 1, 2] assert "age" not in raw.table("person") assert [path.name for path in args.checkpoint_dir.glob("*.frame.h5")] == [ @@ -1053,6 +1105,7 @@ def add_age(frame: Frame, **_kwargs) -> Frame: assert enriched.table("person")["age"].tolist() == [31, 29, 50] assert "ED_VAL" not in enriched.table("person") assert "LKWEEKS" not in enriched.table("person") + assert "A_LFSR" not in enriched.table("person") assert "PAW_TYP" not in enriched.table("person") assert sorted(path.name for path in args.checkpoint_dir.glob("*.frame.h5")) == [ "000_source_construction.frame.h5", @@ -1090,6 +1143,7 @@ def test_completed_source_stage_repairs_raw_auxiliary_without_rewriting_legacy( assert context_path.read_bytes() == expected_context repaired, _metadata = builder.load_asec_raw_stage_checkpoint(raw_path) assert repaired.table("person")["ED_VAL"].tolist() == [0.0, 500.0, 1_000.0] + assert repaired.table("person")["A_LFSR"].tolist() == [1, 3, 7] assert repaired.table("person")["PAW_TYP"].tolist() == [0, 1, 2] @@ -1186,10 +1240,14 @@ def test_source_and_preclone_stages_round_trip_design_weight_kind( "with_us_retirement_contribution_inputs", "with_us_retirement_distribution_inputs", "with_us_esi_premium_inputs", - "with_us_immigration_inputs", ) for name in identity_transforms: monkeypatch.setattr(builder, name, lambda value, **_kwargs: value) + monkeypatch.setattr( + builder, + "_with_us_immigration_inputs_from_asec_source", + lambda value, _source, **_kwargs: value, + ) passing_gate = SimpleNamespace(passed=True, failures=(), details={}) for name in ( "us_relationship_inputs_signal_gate", @@ -2478,8 +2536,8 @@ def fake_retirement_distributions( ) monkeypatch.setattr( builder, - "with_us_immigration_inputs", - lambda frame, *, seed, time_period: frame, + "_with_us_immigration_inputs_from_asec_source", + lambda frame, _source, *, seed, time_period: frame, ) monkeypatch.setattr( builder, diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_spec_bundle.py b/packages/microcosm-build/tests/engine_free/us/test_us_spec_bundle.py index dc816db364..61aa105bbd 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_spec_bundle.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_spec_bundle.py @@ -142,11 +142,11 @@ def test_constant_derived_domain_counts_are_complete( assert len(compiled_schedule["waves"]) == 6 assert ( compiled_schedule["schedule_sha256"] - == "f1fb754e6ade602be394837bc0d7d091f2fba4b4b83fb00f59a0258fe56eb5da" + == "46cea046d2fb8223d32772bd6a7807c52791640943949154982dcc52d93fa9eb" ) assert ( compiled_schedule["payload_sha256"] - == "76a79c9d05cfde1ec1c812de530b5080e6bc212f9c9e693eba628711d6f553e1" + == "3f1daff0b5840d70d818e256515b2cf5d0a28bf4ad9f932deece3a56964047e0" ) assert len(take_up["programs"]) == 17 @@ -258,7 +258,7 @@ def test_legacy_seed_vintage_and_publication_grammars_are_pinned( "identity_generation": 1, "seed_protocol": LEGACY_V1_PROTOCOL.id, } - assert len(LEGACY_V1_PROTOCOL.sites) == 53 + assert len(LEGACY_V1_PROTOCOL.sites) == 66 assert len(LEGACY_V1_PROTOCOL.streams) == 14 assert LEGACY_V1_PROTOCOL.site("survey_sample_asec").default == 578 assert LEGACY_V1_PROTOCOL.site("puf_live_aggregate_disaggregation").default == 0 diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_stacked_spine.py b/packages/microcosm-build/tests/engine_free/us/test_us_stacked_spine.py index b4171135ce..e3804f8123 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_stacked_spine.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_stacked_spine.py @@ -1779,6 +1779,24 @@ def test_gap_fill_plan_covers_declared_families_exactly() -> None: assert early_keys | late_keys == full_keys +def test_immigration_pattern_authority_requires_harmonized_evidence() -> None: + required, optional = stacked_spine_module._acs_pattern_predictor_authority( + entity="person", + family_targets=("ssn_card_type", "immigration_status_str"), + ) + immigration = ( + "__acs_transfer_is_us_citizen", + "__acs_transfer_birth_country_code", + "__acs_transfer_arrival_year", + ) + + assert required == ( + *acs_transfer_module.ACS_PERSON_TRANSFER_PREDICTORS, + *immigration, + ) + assert not set(immigration).intersection(optional) + + def test_gap_fill_validator_accepts_canonical_calibration_evidence() -> None: stacked_spine_module.validate_stacked_gap_fill_receipt( _canonical_gap_fill_calibration_receipt(), @@ -2656,6 +2674,59 @@ def test_late_readiness_rejects_object_typed_nonfinite_numeric_input() -> None: ) +def test_immigration_arrival_readiness_accepts_native_acs_yoep_blanks() -> None: + producer = next( + group.name + for group in stacked_spine_module.CANONICAL_US_LATE_TRANSFER_GROUPS + if group.family == "source_operator_immigration" + ) + full_contract = stacked_spine_module.CANONICAL_US_LATE_PRODUCER_REGISTRY[producer] + requirement = next( + item + for item in full_contract.inputs + if item.column == "@effective:immigration_acs_arrival" + ) + contract = replace(full_contract, inputs=(requirement,), outputs=()) + frame = _post_puf_transfer_fixture() + person = frame.table("person").copy() + acs_rows = person[support_channel_column("person")].astype(str).eq("acs") + assert acs_rows.any() + person["CIT"] = np.where(acs_rows, 1.0, np.nan) + person["YOEP"] = np.nan + tables = {entity: frame.table(entity) for entity in frame.entities} + tables["person"] = person + native_blank = Frame( + tables, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + unfilled, invalid = stacked_spine_module._late_input_readiness_rows( + native_blank, + contract, + ) + + assert requirement.required_scope == "acs_source" + assert requirement.alternatives == ( + (ProducerInputColumn("person", "YOEP", "column_present"),), + ) + assert unfilled == {requirement: 0} + assert invalid == {requirement: 0} + assert ( + stacked_spine_module.run_producer_when_ready( + contract, + lambda: "ran", + unfilled_rows=unfilled, + invalid_rows=invalid, + absence_receipts={}, + ) + == "ran" + ) + + def test_typehugq_accepts_exact_1688_asec_structural_null_rows() -> None: frame, contract, requirement = _typehugq_cross_origin_readiness_fixture() household = frame.table("household") @@ -5421,6 +5492,32 @@ def test_real_late_executor_follows_canonical_order_and_finalizes_sources_once( stacked_spine_module.US_LATE_PRODUCER_TRANSITION_AUTHORITY_KEY ]["sha256"] ) + immigration_group = result.receipt["post_puf_transfer"]["groups"][ + "transfer:person/source_operator_immigration" + ] + immigration_targets = immigration_group["targets"] + pair_receipts = [ + immigration_targets[f"person/source_operator_immigration/{target}"] + for target in ("ssn_card_type", "immigration_status_str") + ] + assert ( + pair_receipts[0]["post_transfer_reconciliation"] + == pair_receipts[1]["post_transfer_reconciliation"] + ) + assert pair_receipts[0]["post_transfer_reconciliation"]["kind"] == ( + "deterministic_humanitarian_residual_target" + ) + expected_required, _optional = ( + stacked_spine_module._acs_pattern_predictor_authority( + entity="person", + family_targets=("ssn_card_type", "immigration_status_str"), + ) + ) + assert all( + tuple(receipt["qrf_pattern_evidence"]["patterns"][0]["predictors"]) + == expected_required + for receipt in pair_receipts + ) stacked_spine_module.validate_stacked_late_producer_receipt( result.receipt, boundary="executor regression", @@ -5429,6 +5526,474 @@ def test_real_late_executor_follows_canonical_order_and_finalizes_sources_once( ) +def test_immigration_group_receipt_rejects_reordered_harmonized_predictors( + monkeypatch: pytest.MonkeyPatch, +) -> None: + result, _events, _finalizer_calls = _run_real_late_executor_fixture(monkeypatch) + transfer = deepcopy(dict(result.receipt["post_puf_transfer"])) + target = transfer["groups"]["transfer:person/source_operator_immigration"][ + "targets" + ]["person/source_operator_immigration/immigration_status_str"] + evidence = target["qrf_pattern_evidence"] + predictors = evidence["patterns"][0]["predictors"] + predictors[-1], predictors[-2] = predictors[-2], predictors[-1] + unsigned = dict(evidence) + unsigned.pop("sha256") + evidence["sha256"] = stacked_spine_module._canonical_sha256(unsigned) + + with pytest.raises(ValueError, match="predictor order is outside canonical"): + stacked_spine_module.validate_stacked_post_puf_transfer_receipt( + transfer, + boundary="reordered immigration predictor regression", + ) + + +@pytest.mark.parametrize( + ("mutation", "message"), + ( + ("remove", "immigration post-transfer reconciliation is absent"), + ("diverge", "paired immigration reconciliation evidence is incomplete"), + ("inflated_tolerance", "reconciliation tolerance is non-canonical"), + ("zero_target_immutable", "mass against an explicit-zero target"), + ), +) +def test_immigration_group_receipt_rejects_missing_or_divergent_reconciliation( + monkeypatch: pytest.MonkeyPatch, + mutation: str, + message: str, +) -> None: + result, _events, _finalizer_calls = _run_real_late_executor_fixture(monkeypatch) + transfer = deepcopy(dict(result.receipt["post_puf_transfer"])) + targets = transfer["groups"]["transfer:person/source_operator_immigration"][ + "targets" + ] + status = targets["person/source_operator_immigration/immigration_status_str"] + reconciliation = status["post_transfer_reconciliation"] + if mutation == "remove": + status.pop("post_transfer_reconciliation") + elif mutation == "diverge": + reconciliation["seed"] += 1 + elif mutation == "inflated_tolerance": + reconciliation["floating_tolerance"] *= 1_000_000 + else: + zero_draw = next( + draw for draw in reconciliation["draws"].values() if draw["target"] == 0 + ) + zero_draw.update( + { + "immutable_population": 1.0, + "achieved_population": 1.0, + "absolute_error": 1.0, + "immutable_overshoot": 1.0, + } + ) + + with pytest.raises(ValueError, match=message): + stacked_spine_module.validate_stacked_post_puf_transfer_receipt( + transfer, + boundary=f"{mutation} immigration reconciliation regression", + ) + + +def test_immigration_reconciliation_replays_receipt_and_downstream_live_frame( + monkeypatch: pytest.MonkeyPatch, +) -> None: + result, _events, _finalizer_calls = _run_real_late_executor_fixture(monkeypatch) + transfer = deepcopy(dict(result.receipt["post_puf_transfer"])) + immigration_group = transfer["groups"][ + "transfer:person/source_operator_immigration" + ] + immigration_targets = immigration_group["targets"] + source_receipt = immigration_targets[ + "person/source_operator_immigration/ssn_card_type" + ]["post_transfer_reconciliation"] + fabricated_reconciliation = deepcopy(source_receipt) + draw = next( + item + for item in fabricated_reconciliation["draws"].values() + if item["target"] > 0 + ) + draw["eligible_recipient_population"] += 1.0 + draw["selected_recipient_population"] += 1.0 + draw["achieved_population"] += 1.0 + draw["residual_selection_error"] = 1.0 + draw["absolute_error"] = 1.0 + for target in ("ssn_card_type", "immigration_status_str"): + key = f"person/source_operator_immigration/{target}" + immigration_targets[key]["post_transfer_reconciliation"] = deepcopy( + fabricated_reconciliation + ) + transfer["targets"][key] = deepcopy(immigration_targets[key]) + + # The forged values remain internally self-consistent; only live replay can + # distinguish them from the actually emitted weighted population. + stacked_spine_module.validate_stacked_post_puf_transfer_receipt( + transfer, + boundary="fabricated immigration mass receipt-only control", + ) + with pytest.raises( + ValueError, + match="differs from the live ASEC/ACS weighted population", + ): + stacked_spine_module.validate_stacked_post_puf_transfer_receipt( + transfer, + boundary="fabricated immigration mass live replay", + frame=result.frame, + ) + + tables = {entity: result.frame.table(entity) for entity in result.frame.entities} + person = tables["person"].copy() + first_positive_draw = next( + draw + for draw in immigration_module.us_immigration_controls().humanitarian + if draw.target > 0 + ) + emitted = immigration_module.us_immigration_humanitarian_draw_mask( + result.frame, + first_positive_draw, + time_period=2024, + ) + person.loc[person.index[np.flatnonzero(emitted)[0]], "immigration_status_str"] = ( + "LEGAL_PERMANENT_RESIDENT" + ) + tables["person"] = person + drifted = Frame( + tables, + result.frame.schema, + { + entity: result.frame.weights_for(entity) + for entity in result.frame.weighted_entities + }, + result.frame.strata, + mass_log=result.frame.mass_log, + metadata=result.frame.metadata, + ) + with pytest.raises( + ValueError, + match="differs from the live ASEC/ACS weighted population", + ): + stacked_spine_module.validate_stacked_late_producer_transition_authority( + drifted, + result.receipt, + boundary="downstream immigration label live replay", + expected_transition_authority_sha256=(result.transition_authority_sha256), + ) + + +def _ukrainian_parole_stack() -> tuple[ + Frame, pd.DataFrame, np.ndarray, np.ndarray, float +]: + """One ASEC citizen household stacked on two equal-weight ACS U4U candidates.""" + + asec = _source_frame( + household_ids=[11], + weights=[2.0], + stratum="asec_2024", + ) + acs = _source_frame( + household_ids=[101, 102], + weights=[1.0, 1.0], + extra_household_columns={"TYPEHUGQ": 1}, + stratum="acs_2024_1yr", + ) + base = assemble_stacked_spine( + asec, + acs, + acs_sample_fraction=1.0, + acs_sample_seed=578, + ).frame + person = base.table("person").copy() + channel = person[support_channel_column("person")].astype(str) + immutable = channel.eq("asec").to_numpy(dtype=bool) + mutable = channel.eq("acs").to_numpy(dtype=bool) + person["PRCITSHP"] = np.where(immutable, 1.0, np.nan) + person["PENATVTY"] = np.where(immutable, 57.0, np.nan) + person["PEINUSYR"] = np.where(immutable, 0.0, np.nan) + person["CIT"] = np.where(mutable, 5.0, np.nan) + person["POBP"] = np.where(mutable, 164.0, np.nan) + person["YOEP"] = np.where(mutable, 2022.0, np.nan) + person["source_year"] = 2024 + person["source_household_id"] = person["person_household_id"] + person["source_person_id"] = person["person_id"] + person["ssn_card_type"] = np.where( + immutable, + "CITIZEN", + "OTHER_NON_CITIZEN", + ) + person["immigration_status_str"] = np.where( + immutable, + "CITIZEN", + "LEGAL_PERMANENT_RESIDENT", + ) + weights = np.asarray(base.resolve_weights("person").values, dtype=np.float64) + mutable_weight = float(weights[mutable][0]) + assert weights[mutable].tolist() == [mutable_weight, mutable_weight] + return base, person, weights, mutable, mutable_weight + + +def _ukrainian_parole_controls(target: float) -> immigration_module.ImmigrationControls: + return immigration_module.ImmigrationControls( + undocumented=immigration_module.UndocumentedControls( + workers=1.0, + students=1.0, + population_anchor=1.0, + sources={ + "undocumented_workers": "https://example.com/workers", + "undocumented_students": "https://example.com/students", + "undocumented_population_anchor": "https://example.com/population", + }, + ), + humanitarian=( + immigration_module.HumanitarianDraw( + category="paroled_one_year", + origin="ukraine", + status="PAROLED_ONE_YEAR", + target=target, + source="https://example.com/u4u", + ), + ), + ) + + +def _with_person(frame: Frame, replacement: pd.DataFrame) -> Frame: + tables = {entity: frame.table(entity) for entity in frame.entities} + tables["person"] = replacement + return Frame( + tables, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +def test_immigration_live_replay_rejects_equal_weight_selection_identity_swap( + monkeypatch: pytest.MonkeyPatch, +) -> None: + base, person, weights, mutable, mutable_weight = _ukrainian_parole_stack() + immutable = ~mutable + controls = _ukrainian_parole_controls(mutable_weight) + reconciled_person, reconciliation = ( + immigration_module.reconcile_us_immigration_humanitarian_transfer( + person, + weights=weights, + mutable_rows=mutable, + seed=0, + time_period=2024, + controls=controls, + ) + ) + + reconciled = _with_person(base, reconciled_person) + target_receipt = { + "producer_rows": int(immutable.sum()), + "authorized_null_rows": int(mutable.sum()), + "imputed_rows": int(mutable.sum()), + "unmodeled_rows": 0, + "residual_null_rows": 0, + "post_transfer_reconciliation": reconciliation, + } + monkeypatch.setattr( + stacked_spine_module, + "us_immigration_controls", + lambda: controls, + ) + stacked_spine_module._validate_immigration_post_transfer_reconciliation( + target_receipt, + boundary="canonical equal-weight selection", + frame=reconciled, + ) + + swapped_person = reconciled_person.copy() + selected_index = swapped_person.index[ + mutable + & swapped_person["immigration_status_str"].eq("PAROLED_ONE_YEAR").to_numpy() + ][0] + unselected_index = swapped_person.index[ + mutable + & swapped_person["immigration_status_str"] + .eq("LEGAL_PERMANENT_RESIDENT") + .to_numpy() + ][0] + swapped_person.loc[selected_index, "immigration_status_str"] = ( + "LEGAL_PERMANENT_RESIDENT" + ) + swapped_person.loc[unselected_index, "immigration_status_str"] = "PAROLED_ONE_YEAR" + swapped = _with_person(reconciled, swapped_person) + assert float( + swapped.resolve_weights("person") + .values[ + swapped_person["immigration_status_str"] + .eq("PAROLED_ONE_YEAR") + .to_numpy(dtype=bool) + ] + .sum() + ) == pytest.approx(mutable_weight) + + with pytest.raises( + ValueError, + match="selected mutable-row identities differ from the canonical seeded selection", + ): + stacked_spine_module._validate_immigration_post_transfer_reconciliation( + target_receipt, + boundary="equal-weight selection identity swap", + frame=swapped, + ) + + +def _exhausted_ukrainian_parole( + monkeypatch: pytest.MonkeyPatch, + target_multiple: float, +) -> tuple[Frame, dict[str, object], float]: + """Reconcile the two-candidate stack to ``target_multiple`` candidate weights.""" + + base, person, weights, mutable, mutable_weight = _ukrainian_parole_stack() + controls = _ukrainian_parole_controls(target_multiple * mutable_weight) + reconciled_person, reconciliation = ( + immigration_module.reconcile_us_immigration_humanitarian_transfer( + person, + weights=weights, + mutable_rows=mutable, + seed=0, + time_period=2024, + controls=controls, + ) + ) + monkeypatch.setattr( + stacked_spine_module, + "us_immigration_controls", + lambda: controls, + ) + target_receipt = { + "producer_rows": int((~mutable).sum()), + "authorized_null_rows": int(mutable.sum()), + "imputed_rows": int(mutable.sum()), + "unmodeled_rows": 0, + "residual_null_rows": 0, + "post_transfer_reconciliation": reconciliation, + } + return _with_person(base, reconciled_person), target_receipt, mutable_weight + + +def test_immigration_reconciliation_takes_every_candidate_of_an_exhausted_draw( + monkeypatch: pytest.MonkeyPatch, +) -> None: + frame, target_receipt, weight = _exhausted_ukrainian_parole(monkeypatch, 2.5) + draw = target_receipt["post_transfer_reconciliation"]["draws"][ + "paroled_one_year:ukraine" + ] + assert draw["candidates_exhausted"] is True + assert draw["within_residual_discrete_weight_bound"] is False + assert draw["within_discrete_weight_bound"] is False + assert draw["eligible_recipient_population"] == pytest.approx(2 * weight) + assert draw["selected_recipient_population"] == pytest.approx(2 * weight) + assert draw["selection_threshold_tie_population"] == 0.0 + assert draw["candidate_shortfall_population"] == pytest.approx(0.5 * weight) + person = frame.table("person") + channel = person[support_channel_column("person")].astype(str) + assert person.loc[channel.eq("acs"), "immigration_status_str"].tolist() == [ + "PAROLED_ONE_YEAR", + "PAROLED_ONE_YEAR", + ] + for live in (frame, None): + stacked_spine_module._validate_immigration_post_transfer_reconciliation( + target_receipt, + boundary="exhausted draw", + frame=live, + ) + + +def _forged(target_receipt: dict[str, object], **fields: object) -> dict[str, object]: + forged = deepcopy(target_receipt) + forged["post_transfer_reconciliation"]["draws"]["paroled_one_year:ukraine"].update( + fields + ) + return forged + + +def test_immigration_reconciliation_rejects_forged_candidate_shortfalls( + monkeypatch: pytest.MonkeyPatch, +) -> None: + frame, target_receipt, weight = _exhausted_ukrainian_parole(monkeypatch, 2.5) + validate = stacked_spine_module._validate_immigration_post_transfer_reconciliation + + concealed = _forged( + target_receipt, + candidates_exhausted=False, + within_residual_discrete_weight_bound=True, + within_discrete_weight_bound=True, + ) + with pytest.raises(ValueError, match="conceals a recipient candidate shortfall"): + validate(concealed, boundary="concealed shortfall", frame=None) + + covered = _forged(target_receipt, eligible_recipient_population=2.5 * weight) + with pytest.raises( + ValueError, + match="claims a candidate shortfall its eligible population covers", + ): + validate(covered, boundary="covered shortfall", frame=None) + + inconsistent = _forged(target_receipt, within_discrete_weight_bound=True) + with pytest.raises(ValueError, match="values are invalid"): + validate(inconsistent, boundary="inconsistent bound flag", frame=None) + + unselected = _forged(target_receipt, selected_recipient_population=weight) + with pytest.raises(ValueError, match="leaves candidates unselected"): + validate(unselected, boundary="unselected candidate", frame=None) + + person = frame.table("person").copy() + channel = person[support_channel_column("person")].astype(str) + dropped = person.index[channel.eq("acs").to_numpy()][0] + person.loc[dropped, "immigration_status_str"] = "LEGAL_PERMANENT_RESIDENT" + with pytest.raises( + ValueError, + match="differs from the live ASEC/ACS weighted population", + ): + validate( + target_receipt, + boundary="live candidate left unselected", + frame=_with_person(frame, person), + ) + + +def test_immigration_reconciliation_rejects_exhaustion_below_the_band_floor( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Two candidates reach 0.2 of a ten-candidate target, under the + # composition gate's floor of 0.35: the reconciliation refuses it. + with pytest.raises( + ValueError, + match=r"leaves it below 0\.35 of its target", + ): + _exhausted_ukrainian_parole(monkeypatch, 10.0) + + # A receipt that claims such a draw is refused too. + _frame, target_receipt, weight = _exhausted_ukrainian_parole(monkeypatch, 2.5) + monkeypatch.setattr( + stacked_spine_module, + "us_immigration_controls", + lambda: _ukrainian_parole_controls(10 * weight), + ) + below_floor = _forged( + target_receipt, + target=10 * weight, + residual_target=10 * weight, + residual_selection_error=8 * weight, + absolute_error=8 * weight, + candidate_shortfall_population=8 * weight, + ) + with pytest.raises( + ValueError, + match="falls below the floor of the composition gate's band", + ): + stacked_spine_module._validate_immigration_post_transfer_reconciliation( + below_floor, + boundary="exhausted below the band floor", + frame=None, + ) + + def test_late_executor_signature_rejects_qrf_regime_evidence_tampering( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/packages/microcosm-build/tests/engine_scenario/us/test_us_hr1_immigration_channels.py b/packages/microcosm-build/tests/engine_scenario/us/test_us_hr1_immigration_channels.py new file mode 100644 index 0000000000..19b3cf3c3b --- /dev/null +++ b/packages/microcosm-build/tests/engine_scenario/us/test_us_hr1_immigration_channels.py @@ -0,0 +1,178 @@ +"""The four H.R.1 immigration probes bind through the engine (microcosm #767). + +The release-blocking reform-coverage smoke gate scores the pinned probe +reforms on the full export; these tests prove the same channels move, +nonzero and sign-correct, on synthetic households whose +``immigration_status_str`` carries the humanitarian values the source stage +now imputes. A probe whose channel dies here would score zero on every +release, so this is the PR-CI early warning for the engine side of the +silent-zero failure — without needing restricted microdata. + +Each situation pins the take-up flag its program reads (the same +data-seeded surface the release carries) and, for the ACA channels, an +``slcsp`` override so no rating-area geography is required. +""" + +from __future__ import annotations + +import pytest +from policyengine_us import Simulation + +from microcosm.build.us_runtime.reform_coverage_smoke import _build_reform +from microcosm.build.us_runtime.release_input_coverage import ( + us_release_reform_coverage_probes, +) + +_HR1_PROBE_IDS = ( + "hr1_medicaid_humanitarian_eligibility_restoration", + "hr1_aca_below_fpl_exception_restoration", + "hr1_aca_lawful_presence_restoration", + "hr1_snap_humanitarian_eligibility_restoration", +) + + +@pytest.fixture(scope="module") +def hr1_probes(): + probes = { + probe.id: probe + for probe in us_release_reform_coverage_probes() + if probe.id.startswith("hr1_") + } + assert set(probes) == set(_HR1_PROBE_IDS) + return probes + + +def _household( + status: str, + income: float, + year: int, + *, + person: dict | None = None, + tax_unit: dict | None = None, + spm_unit: dict | None = None, +) -> dict: + people = { + "parent": { + "age": {year: 35}, + "employment_income": {year: income}, + "immigration_status_str": {year: status}, + **(person or {}), + }, + "child": { + "age": {year: 8}, + "immigration_status_str": {year: status}, + **(person or {}), + }, + } + return { + "people": people, + "families": {"f": {"members": ["parent", "child"]}}, + "marital_units": {"m": {"members": ["parent"]}}, + "tax_units": {"t": {"members": ["parent", "child"], **(tax_unit or {})}}, + "spm_units": {"s": {"members": ["parent", "child"], **(spm_unit or {})}}, + "households": { + "h": {"members": ["parent", "child"], "state_name": {year: "TX"}} + }, + } + + +def _effect(probe, situation: dict, year: int) -> tuple[float, float]: + baseline = Simulation(situation=situation) + reformed = Simulation(situation=situation, reform=_build_reform(probe)) + base = float(baseline.calculate(probe.budget_measure, year).sum()) + reform = float(reformed.calculate(probe.budget_measure, year).sum()) + return base, reform + + +def test_medicaid_restoration_reenrolls_refugee_family(hr1_probes) -> None: + # At 2027 law refugees are outside the §71109-narrowed qualified list; + # restoring the pre-H.R.1 list must enroll the anchored taker. The probe + # counts enrollees (medicaid_enrolled), so the totals here are people. + probe = hr1_probes["hr1_medicaid_humanitarian_eligibility_restoration"] + assert (probe.budget_measure, probe.measure_unit) == ( + "medicaid_enrolled", + "persons", + ) + situation = _household( + "REFUGEE", + 25_000.0, + 2027, + person={"takes_up_medicaid_if_eligible": {2027: True}}, + ) + base, reform = _effect(probe, situation, 2027) + assert base == 0.0 + assert reform > 0.0 + + +def test_below_fpl_exception_restoration_pays_ptc_to_tps_unit(hr1_probes) -> None: + # A below-poverty TPS unit is lawfully present for the ACA at 2026 law + # yet Medicaid-ineligible by status — exactly the §71302 population. + probe = hr1_probes["hr1_aca_below_fpl_exception_restoration"] + situation = _household( + "TPS", + 12_000.0, + 2026, + tax_unit={ + "takes_up_aca_if_eligible": {2026: True}, + "slcsp": {2026: 8_000.0}, + }, + ) + base, reform = _effect(probe, situation, 2026) + assert base == 0.0 + assert reform > 0.0 + + +def test_lawful_presence_restoration_requalifies_asylee_unit(hr1_probes) -> None: + # At 2027 law §71301 adds asylees to the ACA ineligible list; restoring + # the pre-H.R.1 list (DACA/UNDOCUMENTED only) must re-qualify the unit. + probe = hr1_probes["hr1_aca_lawful_presence_restoration"] + situation = _household( + "ASYLEE", + 35_000.0, + 2027, + tax_unit={ + "takes_up_aca_if_eligible": {2027: True}, + "slcsp": {2027: 8_000.0}, + }, + ) + base, reform = _effect(probe, situation, 2027) + assert base == 0.0 + assert reform > 0.0 + + +def test_snap_restoration_reincludes_refugee_members(hr1_probes) -> None: + # At 2026 law §10108 makes refugees excluded SNAP members; restoring the + # pre-H.R.1 list must raise the household allotment. + probe = hr1_probes["hr1_snap_humanitarian_eligibility_restoration"] + situation = _household( + "REFUGEE", + 20_000.0, + 2026, + spm_unit={"takes_up_snap_if_eligible": {2026: True}}, + ) + base, reform = _effect(probe, situation, 2026) + assert reform > base + + +def test_probe_magnitude_floors_sit_far_below_plausible_scale(hr1_probes) -> None: + # The floors are release-scale guards: far above numerical noise, far + # below the plausible aggregate effect of the imputed stocks (about + # 1.6M weighted persons across the humanitarian categories). The + # Medicaid floor counts people; the others are dollars. + floors = { + "hr1_medicaid_humanitarian_eligibility_restoration": 10_000.0, + "hr1_aca_below_fpl_exception_restoration": 5_000_000.0, + "hr1_aca_lawful_presence_restoration": 10_000_000.0, + "hr1_snap_humanitarian_eligibility_restoration": 25_000_000.0, + } + for probe_id, floor in floors.items(): + probe = hr1_probes[probe_id] + assert probe.min_abs_effect == floor + assert probe.measure_unit == ( + "persons" + if probe_id == "hr1_medicaid_humanitarian_eligibility_restoration" + else "usd" + ) + assert probe.expected_sign == "positive" + assert probe.binding_inputs == ("immigration_status_str",) + assert probe.issue == "PolicyEngine/microcosm#767" diff --git a/packages/microcosm-build/tests/engine_scenario/us/test_us_reform_coverage_probe_engine_drift.py b/packages/microcosm-build/tests/engine_scenario/us/test_us_reform_coverage_probe_engine_drift.py index bd08dca497..3ea29deea5 100644 --- a/packages/microcosm-build/tests/engine_scenario/us/test_us_reform_coverage_probe_engine_drift.py +++ b/packages/microcosm-build/tests/engine_scenario/us/test_us_reform_coverage_probe_engine_drift.py @@ -13,8 +13,10 @@ These tests pin the invariants that would have caught both before a release: each list-valued probe differs from the engine's own baseline list by exactly -the one item it means to change, and each source/deduction probe moves only -its own leaf on a household that also carries the neighbouring channels. +the items it declares (the one item a source/deduction probe means to change, +or the pre-H.R.1 status set an H.R.1 restoration probe restores), and each +source/deduction probe moves only its own leaf on a household that also +carries the neighbouring channels. """ import pytest @@ -39,6 +41,35 @@ "niit_passive_pass_through_exclusion": "passive_partnership_s_corp_income", } _ADDED_ITEM = {"domestic_production_ald_reactivation": "domestic_production_ald"} +# The H.R.1 restoration probes (microcosm#767) restore a pre-H.R.1 status list, +# so each changes a declared set, not one item: SS71109 (Medicaid) and SS10108 +# (SNAP) dropped these humanitarian statuses from the eligible lists, and +# SS71301 (ACA) added the humanitarian statuses to the ineligible list. +_PRE_HR1_ELIGIBLE = frozenset( + { + "REFUGEE", + "ASYLEE", + "DEPORTATION_WITHHELD", + "PAROLED_ONE_YEAR", + "CONDITIONAL_ENTRANT", + } +) +_RESTORED_ITEMS: dict[str, tuple[frozenset[str], frozenset[str]]] = { + "hr1_medicaid_humanitarian_eligibility_restoration": ( + frozenset(), + _PRE_HR1_ELIGIBLE, + ), + "hr1_snap_humanitarian_eligibility_restoration": ( + frozenset(), + _PRE_HR1_ELIGIBLE, + ), + "hr1_aca_lawful_presence_restoration": ( + frozenset( + {"REFUGEE", "ASYLEE", "DEPORTATION_WITHHELD", "PAROLED_ONE_YEAR", "TPS"} + ), + frozenset(), + ), +} _SNAP_SOURCE_PROBES = { "child_support_received_snap_exclusion": "child_support_received", @@ -90,7 +121,7 @@ def test_every_list_valued_probe_declares_its_one_item() -> None: # Guards the parametrization below against silently collecting nothing, and # makes a new list-valued probe declare the item it means to change. ids = {probe_id for probe_id, *_ in _list_valued_changes()} - assert ids == set(_REMOVED_ITEM) | set(_ADDED_ITEM) + assert ids == set(_REMOVED_ITEM) | set(_ADDED_ITEM) | set(_RESTORED_ITEMS) assert set(_SNAP_SOURCE_PROBES) <= set(_REMOVED_ITEM) @@ -107,11 +138,12 @@ def test_list_probe_differs_from_engine_baseline_by_exactly_one_item( assert len(value) == len(set(value)), f"{probe_id}: duplicate items in {path}" removed = set(baseline) - set(value) added = set(value) - set(baseline) - expected = ( - (set(), {_ADDED_ITEM[probe_id]}) - if probe_id in _ADDED_ITEM - else ({_REMOVED_ITEM[probe_id]}, set()) - ) + if probe_id in _RESTORED_ITEMS: + expected = tuple(set(items) for items in _RESTORED_ITEMS[probe_id]) + elif probe_id in _ADDED_ITEM: + expected = (set(), {_ADDED_ITEM[probe_id]}) + else: + expected = ({_REMOVED_ITEM[probe_id]}, set()) assert (removed, added) == expected, ( f"{probe_id}: {path} must equal the installed engine's baseline with " f"only {expected} changed (removed, added); got removed " diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py index 5f77431853..fa730ab99e 100644 --- a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py @@ -9,6 +9,7 @@ [ ("success", 0, "iterating"), ("red", 1, "failed"), + ("immigration_red", 1, "failed"), ("error", None, "failed"), ], ) @@ -71,6 +72,7 @@ def test_stacked_tool_entrypoint_fixture_e2e_emits_one_logbook_row_at_every_term "simulate", "completeness", "battery", + "immigration", "publish", ] # Exported rows must never embed host-absolute paths; pytest tmp @@ -89,6 +91,28 @@ def test_stacked_tool_entrypoint_fixture_e2e_emits_one_logbook_row_at_every_term manifest = json.loads( (tmp_path / "stacked-pool.manifest.json").read_text(encoding="utf-8") ) + if terminal == "immigration_red": + assert manifest["status"] == "gate_failed" + assert manifest["simulation_ready"] is False + assert manifest["terminal_gates"]["passed"] is False + assert ( + manifest["terminal_gates"]["gates"]["fixture_battery"]["passed"] is True + ) + assert manifest["terminal_gates"]["gates"]["fixture_immigration"] == { + "passed": False, + "failures": ["fixture immigration terminal failure"], + "details": {}, + } + assert row.gate_verdicts["fixture_battery"]["verdict"] == "passed" + assert row.gate_verdicts["fixture_immigration"]["verdict"] == "failed" + immigration_receipt = json.loads( + _receipt_file_from_reference( + row.gate_verdicts["fixture_immigration"]["receipt"] + ).read_text(encoding="utf-8") + ) + assert immigration_receipt["terminal_gates"]["gates"][ + "fixture_immigration" + ]["failures"] == ["fixture immigration terminal failure"] assert manifest["schema_version"] == pool_tool.POOL_MANIFEST_SCHEMA_VERSION assert manifest["pipeline"] == "us-stacked-pool" assert manifest["pool_h5"]["materializer_version"] == ( @@ -157,13 +181,14 @@ def test_stacked_tool_entrypoint_fixture_e2e_emits_one_logbook_row_at_every_term "materialize_multispine_agreement_outputs", "stacked_completeness_gate", "by_origin_battery", + "us_immigration_composition_gate", ] assert manifest["sampling"] == { **manifest["sampling"], "sample_fraction": 0.01, "fraction_token": "f001", "sample_seed": 578, - "realized_households": {"asec": 1, "acs": 1}, + "realized_households": {"asec": 1, "acs": 12}, } @@ -353,7 +378,9 @@ def capture_equality(expected: object, actual: object) -> None: # The opt-in cross-fitted gate calibration in microcosm.fit.qrf moves it. # microcosm#1073's scf_wealth education-debt declaration moves it # through the bundle's source stages. - "spec_sha256": "4ccbd01720871e59ad8e3c00e1c1296231facef91e83c561076ee499e0a138b2", + # microcosm#767's humanitarian immigration controls, its attested + # immigration kernel and its frozen legacy schedule move it. + "spec_sha256": "783ff9a423a3de4db335c5886e37139ac94b0ae619528433e08a273c17bb111f", }, } @@ -651,6 +678,7 @@ def fail_publication(*_args, **_kwargs) -> None: assert set(row.gate_verdicts) == { "fixture_completeness", "fixture_battery", + "fixture_immigration", "pipeline_error", } terminal_path = _receipt_file_from_reference( @@ -916,6 +944,7 @@ def identity() -> dict[str, object]: "materialize_multispine_agreement_outputs", "stacked_completeness_gate", "by_origin_battery", + "us_immigration_composition_gate", ] assert pool_code["late_producer_schedule"] == pool_tool._json_ready( pool_tool.us_late_producer_schedule_receipt() @@ -1169,7 +1198,7 @@ def changed_source_stage_binding( assert "checkpoint base identity is stale" in capsys.readouterr().out -def test_pool_envelope_v7_preserves_stacked_bank_identity_but_rejects_v6( +def test_pool_envelope_v8_preserves_stacked_bank_identity_but_rejects_v7( pool_tool: ModuleType, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, @@ -1202,7 +1231,7 @@ def identity() -> dict[str, object]: legacy.setattr( pool_tool, "POOL_STAGE_CHECKPOINT_MATERIALIZER_VERSION", - 6, + 7, ) assert identity() == current_identity legacy_store = pool_tool._PoolStageCheckpointStore( @@ -1223,7 +1252,7 @@ def identity() -> dict[str, object]: ) capsys.readouterr() - assert pool_tool.POOL_STAGE_CHECKPOINT_MATERIALIZER_VERSION == 7 + assert pool_tool.POOL_STAGE_CHECKPOINT_MATERIALIZER_VERSION == 8 assert identity() == current_identity current_store = pool_tool._PoolStageCheckpointStore( checkpoint_root, @@ -1374,12 +1403,14 @@ def test_stacked_entrypoint_resumes_each_checkpoint_boundary( "simulate", "completeness", "battery", + "immigration", "publish", ] assert simulated_resume_order == [ "build_stacked_pool", "completeness", "battery", + "immigration", "publish", ] assert transferred_resume_order == [ @@ -1390,6 +1421,7 @@ def test_stacked_entrypoint_resumes_each_checkpoint_boundary( "simulate", "completeness", "battery", + "immigration", "publish", ] assert assembled_resume_order == [ @@ -1404,6 +1436,7 @@ def test_stacked_entrypoint_resumes_each_checkpoint_boundary( "simulate", "completeness", "battery", + "immigration", "publish", ] final_rows = [ diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_us_post_export_scoring.py b/packages/microcosm-build/tests/engine_workflow/us/test_us_post_export_scoring.py index 74f767dc36..62b4d758b8 100644 --- a/packages/microcosm-build/tests/engine_workflow/us/test_us_post_export_scoring.py +++ b/packages/microcosm-build/tests/engine_workflow/us/test_us_post_export_scoring.py @@ -276,9 +276,15 @@ def test_shipped_baseline_plans_match_whole_file_policyengine_us( sweep = _load_sweep_module() path = _write_engine_h5(builder, tmp_path, _GUARD_HOUSEHOLDS) plans, _, counts = sweep._record_requests(builder) - # microcosm#1251's passive partnership/S-corp probe adds one smoke baseline key. + # The four H.R.1 humanitarian-eligibility probes (#767) add three new + # baseline keys: aca_ptc 2026 and 2027 and medicaid_enrolled 2027 (their + # snap 2026 key is shared with the Texas vehicle probe). The Medicaid + # probe counts enrollees because Medicaid dollars divide by a file-wide + # state denominator, which the household-batched scorer refuses + # (microcosm#956). microcosm#1251's passive partnership/S-corp probe + # adds one smoke baseline key. assert counts["baseline_keys"] == { - "reform_coverage_smoke": 20, + "reform_coverage_smoke": 23, "reform_validation": 75, "demographics": 1, } diff --git a/test_support/microcosm_build/spec_engine_inventory_coverage.py b/test_support/microcosm_build/spec_engine_inventory_coverage.py index 5f4dea19f1..a84186c7a5 100644 --- a/test_support/microcosm_build/spec_engine_inventory_coverage.py +++ b/test_support/microcosm_build/spec_engine_inventory_coverage.py @@ -119,14 +119,14 @@ def binding() -> dict[str, object]: "primary_targets": 66, "producer_authored_outputs": 92, "producer_compiled_outputs": 230, - "producer_inputs": 2_747, + "producer_inputs": 2_753, "producer_nodes": 38, "producer_virtual_resources": 75, "release_rungs": 5, "resolved_references": 337, - "seed_owner_bindings": 112, + "seed_owner_bindings": 125, "seed_owner_rows": 54, - "seed_sites": 53, + "seed_sites": 66, "seed_streams": 14, "source_operators": 16, "source_stages": 38, diff --git a/test_support/microcosm_build/us_multispine_pool.py b/test_support/microcosm_build/us_multispine_pool.py index fae6114fb9..c04fe3e0ac 100644 --- a/test_support/microcosm_build/us_multispine_pool.py +++ b/test_support/microcosm_build/us_multispine_pool.py @@ -726,6 +726,7 @@ def _producer_dtype_source_frame() -> Frame: person["PRCITSHP"] = [1, 5, 1, 5] person["PEINUSYR"] = [0, 24, 0, 24] person["PENATVTY"] = [57, 303, 57, 303] + person["A_LFSR"] = [1, 0, 1, 0] person["A_SPOUSE"] = 0 person["CAID"] = 2 person["IHSFLG"] = 2 @@ -762,6 +763,9 @@ def _producer_dtype_acs_source_frame() -> Frame: person["acs_social_security_income"] = [0.0, 12_000.0] person["acs_retirement_income"] = [0.0, 8_000.0] person["acs_interest_dividend_rental_income"] = [100.0, 2_000.0] + person["CIT"] = [1, 5] + person["POBP"] = [6, 373] + person["YOEP"] = [np.nan, 2022] household = tables["household"] household["state_fips"] = [6, 36] household["tenure_type"] = ["RENTED", "OWNED_WITH_MORTGAGE"] diff --git a/test_support/microcosm_build/us_multispine_pool_tool.py b/test_support/microcosm_build/us_multispine_pool_tool.py index 3d87c63629..09c4cefa62 100644 --- a/test_support/microcosm_build/us_multispine_pool_tool.py +++ b/test_support/microcosm_build/us_multispine_pool_tool.py @@ -23,6 +23,7 @@ import pytest import microcosm.build.us_runtime.acs_transfer as acs_transfer_module +import microcosm.build.us_runtime.immigration as immigration_module import microcosm.build.us_runtime.multispine_pool as multispine_pool_module import microcosm.build.us_runtime.post_transfer_calibration as post_transfer_calibration_runtime import microcosm.build.us_runtime.stacked_spine as stacked_spine_module @@ -193,6 +194,86 @@ def _many_household_source_frame( ) +def _with_household_design_weights(frame: Frame, values: np.ndarray) -> Frame: + """Return a source fixture with replacement household design weights.""" + + weights = np.asarray(values, dtype=np.float64) + assert weights.shape == (len(frame.table("household")),) + return Frame( + {entity: frame.table(entity).copy(deep=True) for entity in frame.entities}, + frame.schema, + {"household": Weights(weights, frame.weights_for("household").kind)}, + frame.strata.copy(deep=True), + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +def _humanitarian_weighted_source_frames( + *, + asec_count: int, + acs_count: int, + sample_fraction: float, + sample_seed: int, + state_fips: str | None = None, + puma: str | None = None, +) -> tuple[Frame, Frame]: + """Shape sampled survey weights to the live humanitarian controls. + + The stacked assembly normalizes each sampled arm back to its source mass, + then splits the ASEC anchor mass equally across ASEC and ACS. Setting the + ASEC source mass to twice the humanitarian target total and the selected + ACS weights to the individual targets therefore makes both assembled arms + exact without changing the frozen assembly manifest later in the fixture + pipeline. + """ + + controls = immigration_module.us_immigration_controls() + positive_draws = tuple(draw for draw in controls.humanitarian if draw.target > 0) + target_weights = np.asarray( + [draw.target for draw in positive_draws], + dtype=np.float64, + ) + target_mass = float(target_weights.sum()) + asec = _many_household_source_frame( + count=asec_count, + state_fips=state_fips, + ) + acs = _many_household_source_frame( + count=acs_count, + measured_offset=1_000.0, + state_fips=state_fips, + puma=puma, + ) + sampled_acs, _receipt = stacked_spine_module.sample_acs_households( + acs, + fraction=sample_fraction, + seed=sample_seed, + ) + selected_ids = sampled_acs.table("household")["household_id"].to_numpy( + dtype=np.int64 + ) + assert len(selected_ids) == len(target_weights) + + asec = _with_household_design_weights( + asec, + np.full(asec_count, 2.0 * target_mass / asec_count, dtype=np.float64), + ) + acs_household_ids = acs.table("household")["household_id"].to_numpy(dtype=np.int64) + acs_weights = np.ones(acs_count, dtype=np.float64) + id_to_position = { + int(household_id): position + for position, household_id in enumerate(acs_household_ids) + } + for household_id, target_weight in zip( + selected_ids, + target_weights, + strict=True, + ): + acs_weights[id_to_position[int(household_id)]] = target_weight + return asec, _with_household_design_weights(acs, acs_weights) + + def _replace_person( frame: Frame, person: pd.DataFrame, @@ -1230,6 +1311,283 @@ def _canonical_late_calibration_owner_receipt( return owner +_HUMANITARIAN_ACS_EVIDENCE = { + "paroled_one_year:afghanistan": (200, 2021), + "paroled_one_year:ukraine": (164, 2022), + "paroled_one_year:nicaragua": (315, 2023), + "paroled_one_year:venezuela": (373, 2022), + "refugee": (412, 2022), + "asylee": (207, 2016), + "deportation_withheld": (501, 2015), + "tps:venezuela": (373, 2015), + "tps:el_salvador": (312, 2001), + "tps:honduras": (314, 1998), + "tps:nicaragua": (315, 1998), + "tps:nepal": (229, 2015), + "tps:other_designated": (205, 2015), +} + + +def _with_reconciled_immigration_fixture( + frame: Frame, +) -> tuple[Frame, dict[str, object]]: + """Build and reconcile one live ACS household per positive manifest draw.""" + + controls = immigration_module.us_immigration_controls() + positive_draws = tuple(draw for draw in controls.humanitarian if draw.target > 0) + expected_labels = {draw.label for draw in controls.humanitarian} + assert set(_HUMANITARIAN_ACS_EVIDENCE) == expected_labels + row_count = 1 + len(positive_draws) + ids = np.arange(1, row_count + 1, dtype=np.int64) + expected_channels = {"asec": 1, "acs": len(positive_draws)} + preserve_live_lineage = all( + len(frame.table(entity)) == row_count + and support_channel_column(entity) in frame.table(entity) + and frame.table(entity)[support_channel_column(entity)] + .astype(str) + .value_counts() + .to_dict() + == expected_channels + for entity in frame.entities + ) + + tables: dict[str, pd.DataFrame] = {} + for entity in frame.entities: + source = frame.table(entity) + assert not source.empty + if preserve_live_lineage: + table = source.copy(deep=True) + else: + table = pd.concat([source.iloc[[0]]] * row_count, ignore_index=True) + table[f"{entity}_id"] = ids + table[support_channel_column(entity)] = np.asarray( + ["asec", *(["acs"] * len(positive_draws))], + dtype=object, + ) + table[support_clone_index_column(entity)] = np.zeros( + row_count, + dtype=np.int64, + ) + tables[entity] = table + + person = tables[frame.schema.person_entity] + if not preserve_live_lineage: + for entity in frame.schema.group_entities: + person[f"person_{entity}_id"] = ids + person["person_id"] = ids + person_channels = person[support_channel_column(frame.schema.person_entity)].astype( + str + ) + asec_rows = person_channels.eq("asec").to_numpy(dtype=bool) + acs_rows = person_channels.eq("acs").to_numpy(dtype=bool) + assert int(asec_rows.sum()) == 1 + assert int(acs_rows.sum()) == len(positive_draws) + # Keep legacy TPS arrivals outside the DACA age-at-entry cohort so each + # row remains eligible for the single humanitarian draw it represents. + person["A_AGE"] = np.full(row_count, 60.0) + if "age" in person: + person["age"] = np.full(row_count, 60.0) + if "source_year" in person and not preserve_live_lineage: + person["source_year"] = np.full(row_count, 2024, dtype=np.int64) + if "source_household_id" in person and not preserve_live_lineage: + person["source_household_id"] = ids + if "source_person_id" in person and not preserve_live_lineage: + person["source_person_id"] = ids + if "PERIDNUM" in person and not preserve_live_lineage: + person["PERIDNUM"] = pd.Series(ids.astype(str), dtype="string") + + acs_evidence = [_HUMANITARIAN_ACS_EVIDENCE[draw.label] for draw in positive_draws] + for column in ("PRCITSHP", "PENATVTY", "PEINUSYR", "CIT", "POBP", "YOEP"): + person[column] = np.full(row_count, np.nan, dtype=np.float64) + person.loc[asec_rows, ["PRCITSHP", "PENATVTY", "PEINUSYR"]] = ( + 1.0, + 57.0, + 0.0, + ) + person.loc[acs_rows, "CIT"] = 5.0 + person.loc[acs_rows, "POBP"] = [item[0] for item in acs_evidence] + person.loc[acs_rows, "YOEP"] = [item[1] for item in acs_evidence] + person["ssn_card_type"] = pd.Series( + np.where(asec_rows, "CITIZEN", "OTHER_NON_CITIZEN"), + index=person.index, + dtype="string", + ) + person["immigration_status_str"] = pd.Series( + np.where(asec_rows, "CITIZEN", "LEGAL_PERMANENT_RESIDENT"), + index=person.index, + dtype="string", + ) + + household = tables["household"] + existing_household_weights = pd.Series( + np.asarray(frame.weights_for("household").values, dtype=np.float64), + index=frame.table("household")["household_id"].to_numpy(dtype=np.int64), + ) + asec_household_id = int(person.loc[asec_rows, "person_household_id"].iloc[0]) + household_id_to_weight = { + asec_household_id: ( + float(existing_household_weights.loc[asec_household_id]) + if preserve_live_lineage + else 1.0 + ), + **{ + int(household_id): float(draw.target) + for household_id, draw in zip( + person.loc[acs_rows, "person_household_id"], + positive_draws, + strict=True, + ) + }, + } + household_weights = household["household_id"].map(household_id_to_weight) + assert household_weights.notna().all() + assert tuple(frame.weighted_entities) == ("household",) + weights = { + "household": Weights( + household_weights.to_numpy(dtype=np.float64), + frame.weights_for("household").kind, + ) + } + source_strata = np.asarray(frame.strata, dtype=object) + strata = ( + frame.strata + if preserve_live_lineage + else pd.Series(np.resize(source_strata, row_count), dtype=object) + ) + metadata = dict(frame.metadata) + assembly_key = stacked_spine_module.SPINE_ASSEMBLY_MANIFEST_KEY + assembly = metadata.get(assembly_key) + if not preserve_live_lineage and isinstance(assembly, Mapping): + assembly = copy.deepcopy(dict(assembly)) + declared_channels = tuple(assembly["channels"]) + assembly["native_row_counts"] = { + entity: { + channel: int(table[support_channel_column(entity)].eq(channel).sum()) + for channel in declared_channels + } + for entity, table in tables.items() + } + metadata[assembly_key] = assembly + stacked_key = stacked_spine_module.STACKED_SPINE_MANIFEST_KEY + stacked = metadata.get(stacked_key) + if not preserve_live_lineage and isinstance(stacked, Mapping): + stacked = copy.deepcopy(dict(stacked)) + household_channels = household[support_channel_column("household")].astype(str) + live_weights = household_weights.to_numpy(dtype=np.float64) + live_masses = { + channel: float( + live_weights[household_channels.eq(channel).to_numpy()].sum() + ) + for channel in ("asec", "acs") + } + total_mass = float(live_weights.sum()) + shares = { + channel: live_masses[channel] / total_mass for channel in ("asec", "acs") + } + sample_receipts = { + channel: dict(sample) + for channel, sample in stacked["survey_samples"].items() + } + mass_anchor = str(stacked["mass_anchor_channel"]) + normalized_masses = { + channel: total_mass if channel == mass_anchor else live_masses[channel] + for channel in ("asec", "acs") + } + for channel, sample in sample_receipts.items(): + sampled_mass = float(sample["sampled_household_mass"]) + normalized_mass = normalized_masses[channel] + sample.update( + { + "incoming_household_mass": normalized_mass, + "normalization_factor": normalized_mass / sampled_mass, + "normalized_household_mass": normalized_mass, + } + ) + stacked["survey_samples"] = sample_receipts + stacked["household_mass_shares"] = shares + harmonization = { + channel: dict(arm) + for channel, arm in stacked["weight_harmonization"].items() + } + for channel, arm in harmonization.items(): + incoming_mass = normalized_masses[channel] + allocated_mass = live_masses[channel] + arm.update( + { + "share": shares[channel], + "incoming_mass": incoming_mass, + "allocated_mass": allocated_mass, + "declared_allocation": shares[channel] * total_mass, + "scale_factor": allocated_mass / incoming_mass, + } + ) + stacked["weight_harmonization"] = harmonization + metadata[stacked_key] = stacked + reconciled_frame = Frame( + tables, + frame.schema, + weights, + strata, + mass_log=frame.mass_log, + metadata=metadata, + ) + reconciled_person, receipt = ( + immigration_module.reconcile_us_immigration_humanitarian_transfer( + reconciled_frame.table(frame.schema.person_entity), + weights=np.asarray( + reconciled_frame.resolve_weights(frame.schema.person_entity).values, + dtype=np.float64, + ), + mutable_rows=acs_rows, + seed=0, + ) + ) + return _replace_person(reconciled_frame, reconciled_person), receipt + + +def _immigration_fixture_qrf_evidence( + *, + target: str, + family_targets: tuple[str, ...], + donor_rows: int, + recipient_rows: int, +) -> dict[str, object]: + """Return canonical joint-codec QRF evidence for one paired output leaf.""" + + required_predictors, _optional_predictors = ( + stacked_spine_module._acs_pattern_predictor_authority( + entity="person", + family_targets=family_targets, + ) + ) + pattern = acs_transfer_module.AcsTransferPattern( + name=acs_transfer_module._pattern_name(0, ()), + observed_optional_predictors=(), + predictors=required_predictors, + seed=0, + weight_kind=WeightKind.DESIGN.value, + donor_rows=donor_rows, + recipient_rows=recipient_rows, + target_regimes=tuple( + (model_target, "positive_only") + for model_target in acs_transfer_module._model_target_names(family_targets) + ), + ) + record = acs_transfer_module.AcsImputedInput( + column=target, + entity="person", + family="source_operator_immigration", + donor_spine="synthetic_pool_tool_fixture", + donor_channel="asec", + predictors=required_predictors, + seed=0, + weight_kind=WeightKind.DESIGN.value, + patterns=(pattern,), + imputed_recipient_rows=recipient_rows, + ) + return stacked_spine_module._acs_imputed_pattern_evidence(record) + + def _canonical_pregnancy_structural_receipt() -> dict[str, object]: policy = acs_transfer_module.acs_transfer_execution_contract_identity( targets=("is_pregnant",), @@ -1264,7 +1622,12 @@ def _canonical_late_transfer_receipt( *, authority: Mapping[str, object] | None = None, frame: Frame | None = None, + immigration_reconciliation: Mapping[str, object] | None = None, ) -> dict[str, object]: + if immigration_reconciliation is None: + _fixture_frame, immigration_reconciliation = ( + _with_reconciled_immigration_fixture(_source_frame()) + ) canonical_family = { (entity, target): family for entity, families in ( @@ -1286,12 +1649,36 @@ def _canonical_late_transfer_receipt( ] ) for group in pool_tool.CANONICAL_US_LATE_TRANSFER_GROUPS: + is_immigration_group = ( + group.entity == "person" and group.family == "source_operator_immigration" + ) + immigration_mutable_rows = int(immigration_reconciliation["mutable_rows"]) + immigration_immutable_rows = int(immigration_reconciliation["immutable_rows"]) group_targets = { f"{group.entity}/{group.family}/{target}": { - "authorized_null_rows": 0, - "imputed_rows": 0, + "authorized_null_rows": immigration_mutable_rows + if is_immigration_group + else 0, + "imputed_rows": immigration_mutable_rows if is_immigration_group else 0, "unmodeled_rows": 0, "residual_null_rows": 0, + **( + { + "qrf_pattern_evidence": ( + _immigration_fixture_qrf_evidence( + target=target, + family_targets=group.targets, + donor_rows=immigration_immutable_rows, + recipient_rows=immigration_mutable_rows, + ) + ), + "post_transfer_reconciliation": copy.deepcopy( + immigration_reconciliation + ), + } + if is_immigration_group + else {} + ), } for target in group.targets } @@ -1354,6 +1741,7 @@ def _canonical_late_dag_receipt( authority: Mapping[str, object] | None = None, output_frame_sha256: str = "f" * 64, frame: Frame | None = None, + immigration_reconciliation: Mapping[str, object] | None = None, ) -> dict[str, object]: schedule = stacked_spine_module.CANONICAL_US_LATE_PRODUCER_SCHEDULE schedule_receipt = pool_tool._json_ready( @@ -1397,6 +1785,7 @@ def _canonical_late_dag_receipt( pool_tool, authority=authority, frame=frame, + immigration_reconciliation=immigration_reconciliation, ) input_frame_sha256 = "e" * 64 previous_sha256 = stacked_spine_module._late_execution_genesis_sha256( @@ -1618,8 +2007,9 @@ def _authorized_late_impute_fixture( *, authority: Mapping[str, object] | None = None, ) -> tuple[Frame, dict[str, object], str]: - """Bind one structurally signed synthetic DAG proof to a live fixture frame.""" + """Bind one structurally signed fixture DAG proof to its live frame.""" + frame, immigration_reconciliation = _with_reconciled_immigration_fixture(frame) tables = {entity: frame.table(entity).copy(deep=True) for entity in frame.entities} for entity, table in tables.items(): if support_channel_column(entity) not in table: @@ -1664,6 +2054,7 @@ def _authorized_late_impute_fixture( authority=authority, output_frame_sha256=stacked_spine_module._late_frame_content_sha256(frame), frame=frame, + immigration_reconciliation=immigration_reconciliation, ) authorized, transition_authority_sha256 = ( stacked_spine_module._bind_late_producer_transition_authority(frame, dag) @@ -1694,15 +2085,17 @@ def _install_stacked_entrypoint_stubs( verified = _verified_inputs_fixture(pool_tool, tmp_path / "pins") source_manifest = pool_tool.load_acs_source_manifest() puf_donor = pd.DataFrame({"fixture": np.arange(7)}) + asec_source, acs_source = _humanitarian_weighted_source_frames( + asec_count=100, + acs_count=1_200, + sample_fraction=0.01, + sample_seed=578, + state_fips="06" if real_geography_assignment else None, + puma="0600100" if real_geography_assignment else None, + ) loaded = pool_tool._LoadedInputs( - asec=_many_household_source_frame( - state_fips="06" if real_geography_assignment else None, - ), - acs=_many_household_source_frame( - measured_offset=1_000.0, - state_fips="06" if real_geography_assignment else None, - puma="0600100" if real_geography_assignment else None, - ), + asec=asec_source, + acs=acs_source, acs_rent_donor=pd.DataFrame({"fixture": [1.0]}), puf_donor=puf_donor, asec_raw_stage_checkpoint={"artifact": "fixture-raw-stage"}, @@ -1843,8 +2236,8 @@ def gap_fill(frame: Frame, **kwargs): ) assert counts == { ("person", "strike_benefits"): { - "authorized_null_rows": 1, - "recipient_rows": 1, + "authorized_null_rows": 12, + "recipient_rows": 12, "donor_rows": 1, } } @@ -1955,9 +2348,12 @@ def late_producer_dag(frame: Frame, **kwargs: object): "family": group.family, "ordered_targets": list(group.targets), } + late_base, immigration_reconciliation = _with_reconciled_immigration_fixture( + primary_puf_result.frame + ) late_tables = { - entity: primary_puf_result.frame.table(entity).copy(deep=True) - for entity in primary_puf_result.frame.entities + entity: late_base.table(entity).copy(deep=True) + for entity in late_base.entities } for ( spec @@ -1985,22 +2381,17 @@ def late_producer_dag(frame: Frame, **kwargs: object): index=late_person.index, dtype="string", ) - late_tables.update( - { - name: primary_puf_result.frame.link(name) - for name in primary_puf_result.frame.links - } - ) + late_tables.update({name: late_base.link(name) for name in late_base.links}) late_frame = Frame( late_tables, - primary_puf_result.frame.schema, + late_base.schema, { - entity: primary_puf_result.frame.weights_for(entity) - for entity in primary_puf_result.frame.weighted_entities + entity: late_base.weights_for(entity) + for entity in late_base.weighted_entities }, - primary_puf_result.frame.strata, - mass_log=primary_puf_result.frame.mass_log, - metadata=primary_puf_result.frame.metadata, + late_base.strata, + mass_log=late_base.mass_log, + metadata=late_base.metadata, ) dag_receipt = _canonical_late_dag_receipt( pool_tool, @@ -2009,6 +2400,7 @@ def late_producer_dag(frame: Frame, **kwargs: object): late_frame ), frame=late_frame, + immigration_reconciliation=immigration_reconciliation, ) authorized_frame, transition_authority_sha256 = ( stacked_spine_module._bind_late_producer_transition_authority( @@ -2100,6 +2492,19 @@ def battery( monkeypatch.setattr(pool_tool, "stacked_completeness_gate", completeness) monkeypatch.setattr(pool_tool, "by_origin_battery", battery) + + def immigration(_frame: Frame, *, time_period: int) -> GateResult: + assert time_period == pool_tool.POOL_TIME_PERIOD + order.append("immigration") + if terminal == "immigration_red": + return GateResult( + name="fixture_immigration", + passed=False, + failures=("fixture immigration terminal failure",), + ) + return GateResult(name="fixture_immigration", passed=True) + + monkeypatch.setattr(pool_tool, "us_immigration_composition_gate", immigration) real_publish = pool_tool._write_stacked_outputs def publish(*args, **kwargs): diff --git a/test_support/microcosm_build/us_spec_bundle.py b/test_support/microcosm_build/us_spec_bundle.py index 06dbcce94d..50c3698ad5 100644 --- a/test_support/microcosm_build/us_spec_bundle.py +++ b/test_support/microcosm_build/us_spec_bundle.py @@ -195,7 +195,7 @@ def _load_generator_module(): LEGACY_COMPATIBILITY_SHA256 = { "source_stages.json": ( - "959214c437bfa59d43e32279b82f29a0987649cd7579ff87567c70f084eae66c" + "67587983a455bfffde688b024e7d4bd03a9d1d1f3b79291881558e60bb20311a" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" diff --git a/test_support/microcosm_build/us_stacked_spine.py b/test_support/microcosm_build/us_stacked_spine.py index 6228e5e493..6fb3085b8f 100644 --- a/test_support/microcosm_build/us_stacked_spine.py +++ b/test_support/microcosm_build/us_stacked_spine.py @@ -31,6 +31,7 @@ import microcosm.build.us_runtime.acs_income_universe as universe_module import microcosm.build.us_runtime.acs_transfer as acs_transfer_module +import microcosm.build.us_runtime.immigration as immigration_module import microcosm.build.us_runtime.multispine_pool as multispine_pool_module import microcosm.build.us_runtime.post_transfer_calibration as post_transfer_calibration_runtime import microcosm.build.us_runtime.puf_capital_gains_tail as tail_module @@ -1128,6 +1129,112 @@ def _late_universe_entry_fixture() -> Frame: return initial +def _humanitarian_late_executor_entry_fixture() -> Frame: + """Build one exact weighted ACS carrier for every positive manifest draw.""" + + controls = immigration_module.us_immigration_controls() + positive_draws = tuple(draw for draw in controls.humanitarian if draw.target > 0) + profiles = { + "paroled_one_year:afghanistan": (200, 2021), + "paroled_one_year:ukraine": (164, 2022), + "paroled_one_year:nicaragua": (315, 2023), + "paroled_one_year:venezuela": (373, 2022), + "refugee": (412, 2022), + "asylee": (207, 2016), + "tps:venezuela": (373, 2021), + "tps:el_salvador": (312, 2001), + "tps:honduras": (314, 1998), + "tps:nicaragua": (315, 1998), + "tps:nepal": (229, 2015), + "tps:other_designated": (224, 2024), + } + assert {draw.label for draw in positive_draws} == set(profiles) + positive_target = float(sum(draw.target for draw in positive_draws)) + + # Stacked assembly assigns half of the ASEC anchor mass to each source. + # Make the ACS incoming total equal that allocation so every live ACS + # household retains its manifest target as its final resolved weight. + asec = _source_frame( + household_ids=[11], + weights=[2.0 * positive_target], + stratum="asec_2024", + ) + asec_person = asec.table("person") + asec_person["is_female"] = False + asec_person["is_household_head"] = True + asec_person["employment_income_before_lsr"] = 10_000.0 + asec_person["self_employment_income_before_lsr"] = 0.0 + asec_person["pre_subsidy_rent"] = 12_000.0 + asec_person["unemployment_compensation"] = 1.0 + asec_person["is_disabled"] = False + for column, value in ( + ("taxable_interest_income", 100.0), + ("tax_exempt_interest_income", 0.0), + ("qualified_dividend_income", 50.0), + ("non_qualified_dividend_income", 25.0), + ("rental_income", 0.0), + ("estate_income", 0.0), + ): + asec_person[column] = value + asec_person["PRCITSHP"] = 1.0 + asec_person["PENATVTY"] = 57.0 + asec_person["PEINUSYR"] = 0.0 + asec_person["source_year"] = 2024 + asec_person["source_household_id"] = asec_person["person_household_id"] + asec_person["source_person_id"] = asec_person["person_id"] + asec.table("household")["tenure_type"] = "RENTED" + + acs = _source_frame( + household_ids=list(range(101, 101 + len(positive_draws))), + weights=[float(draw.target) for draw in positive_draws], + stratum="acs_2024_1yr", + ) + acs_person = acs.table("person") + acs_person["is_female"] = False + acs_person["is_household_head"] = True + acs_person["employment_income_before_lsr"] = 10_000.0 + acs_person["WAGP"] = 10_000.0 + acs_person["self_employment_income_before_lsr"] = 0.0 + acs_person["SEMP"] = 0.0 + acs_person["acs_interest_dividend_rental_income"] = 0.0 + acs_person["CIT"] = 5.0 + acs_person["POBP"] = [profiles[draw.label][0] for draw in positive_draws] + acs_person["YOEP"] = [profiles[draw.label][1] for draw in positive_draws] + acs_person["age"] = [12.0, *([60.0] * (len(positive_draws) - 1))] + acs_person["source_year"] = 2024 + acs_person["source_household_id"] = acs_person["person_household_id"] + acs_person["source_person_id"] = acs_person["person_id"] + acs.table("household")["TYPEHUGQ"] = 1 + acs.table("household")["tenure_type"] = "RENTED" + + registry = stacked_spine_module.CANONICAL_US_LATE_PRODUCER_REGISTRY + primary_contract = registry[stacked_spine_module.US_LATE_PRIMARY_PUF_STAGE] + initial = _fill_late_contract_surface( + assemble_stacked_spine( + asec, + acs, + acs_sample_fraction=1.0, + acs_sample_seed=578, + ).frame, + contracts=(primary_contract,), + include_outputs=False, + ) + initial_person = initial.table("person") + structural_row = initial_person.index[ + initial_person[support_channel_column("person")].eq("acs") + ][0] + initial_person.loc[ + structural_row, + [ + "WAGP", + "SEMP", + "employment_income_before_lsr", + "self_employment_income_before_lsr", + ], + ] = np.nan + return initial + + def _run_real_late_executor_fixture( monkeypatch: pytest.MonkeyPatch, *, @@ -1136,7 +1243,7 @@ def _run_real_late_executor_fixture( asec_earnings_delta: float = 0.0, ) -> tuple[stacked_spine_module.StackedLateProducerResult, tuple[str, ...], int]: registry = stacked_spine_module.CANONICAL_US_LATE_PRODUCER_REGISTRY - initial = _late_universe_entry_fixture() + initial = _humanitarian_late_executor_entry_fixture() initial_person = initial.table("person") if asec_earnings_delta: asec_row = initial_person.index[ @@ -1202,6 +1309,33 @@ def primary(frame: Frame): include_outputs=True, ) completed_person = completed.table("person") + asec_rows = completed_person[support_channel_column("person")].eq("asec") + completed_person.loc[asec_rows, ["CIT", "POBP", "YOEP"]] = np.nan + completed_person.loc[ + ~asec_rows, + ["PRCITSHP", "PENATVTY", "PEINUSYR"], + ] = np.nan + completed_person["A_AGE"] = completed_person["age"] + completed_person["ssn_card_type"] = np.where( + asec_rows, + "CITIZEN", + "NONE", + ) + completed_person["immigration_status_str"] = np.where( + asec_rows, + "CITIZEN", + "UNDOCUMENTED", + ) + indicator_documented = ~asec_rows & pd.to_numeric( + completed_person["POBP"], + errors="raise", + ).isin((207, 412)) + completed_person.loc[indicator_documented, "ssn_card_type"] = ( + "OTHER_NON_CITIZEN" + ) + completed_person.loc[indicator_documented, "immigration_status_str"] = ( + "LEGAL_PERMANENT_RESIDENT" + ) completed_person["unemployment_compensation"] = np.ones( len(completed_person), dtype=np.float64, @@ -1311,10 +1445,55 @@ def transfer( for spec in post_transfer_calibration_runtime.POST_TRANSFER_CALIBRATION_SPECS.values() if spec.stage == "late_transfer" } + is_immigration_group = ( + group.entity == "person" and group.family == "source_operator_immigration" + ) + immigration_reconciliation: Mapping[str, object] | None = None + imputed_recipient_rows = 1 + if is_immigration_group: + mutable_rows = ( + frame.table("person")[support_channel_column("person")] + .astype(str) + .eq("acs") + .to_numpy(dtype=bool) + ) + reconciled_person, immigration_reconciliation = ( + immigration_module.reconcile_us_immigration_humanitarian_transfer( + frame.table("person"), + weights=np.asarray( + frame.resolve_weights("person").values, + dtype=np.float64, + ), + mutable_rows=mutable_rows, + seed=0, + time_period=2024, + ) + ) + tables = {entity: frame.table(entity) for entity in frame.entities} + tables["person"] = reconciled_person + frame = Frame( + tables, + frame.schema, + { + entity: frame.weights_for(entity) + for entity in frame.weighted_entities + }, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + imputed_recipient_rows = int(mutable_rows.sum()) evidence_targets = tuple( - target - for target in group.targets - if f"{group.entity}/{group.family}/{target}" in late_specs + dict.fromkeys( + ( + *( + target + for target in group.targets + if f"{group.entity}/{group.family}/{target}" in late_specs + ), + *(group.targets if is_immigration_group else ()), + ) + ) ) model_targets = acs_transfer_module._model_target_names(evidence_targets) pattern = AcsTransferPattern( @@ -1324,7 +1503,7 @@ def transfer( seed=0, weight_kind="design", donor_rows=1, - recipient_rows=1, + recipient_rows=imputed_recipient_rows, target_regimes=tuple((target, "positive_only") for target in model_targets), ) plain_pattern = replace(pattern, target_regimes=()) @@ -1339,7 +1518,8 @@ def transfer( seed=pattern.seed, weight_kind=pattern.weight_kind, patterns=(pattern if target in evidence_targets else plain_pattern,), - imputed_recipient_rows=1, + imputed_recipient_rows=imputed_recipient_rows, + reconciliation=immigration_reconciliation, ) for target in group.targets ) @@ -1361,8 +1541,8 @@ def transfer( ): key = f"{group.entity}/{group.family}/{target}" target_receipt: dict[str, object] = { - "authorized_null_rows": 1, - "imputed_rows": 1, + "authorized_null_rows": imputed_recipient_rows, + "imputed_rows": imputed_recipient_rows, "unmodeled_rows": 0, "residual_null_rows": 0, } @@ -1392,10 +1572,14 @@ def transfer( "final_clone_disagreement_source_persons": 0, "status": "verified", } - if key in late_specs: + if key in late_specs or is_immigration_group: target_receipt["qrf_pattern_evidence"] = ( stacked_spine_module._acs_imputed_pattern_evidence(record) ) + if is_immigration_group: + target_receipt["post_transfer_reconciliation"] = dict( + record.reconciliation or {} + ) target_receipts[key] = target_receipt calibrated_keys = sorted(set(target_receipts) & set(late_specs)) for key in calibrated_keys: diff --git a/tools/_legacy/build_us_acs_multispine_base.py b/tools/_legacy/build_us_acs_multispine_base.py index 2436608173..1155f21c2e 100644 --- a/tools/_legacy/build_us_acs_multispine_base.py +++ b/tools/_legacy/build_us_acs_multispine_base.py @@ -552,7 +552,7 @@ def _with_local_immigration( raise SystemExit( f"ACS local immigration stage failed ({ACS_LOCAL_IMMIGRATION_ISSUE}): {exc}" ) from exc - gate = acs_local_immigration_signal_gate(frame) + gate = acs_local_immigration_signal_gate(frame, time_period=period) if not gate.passed: raise SystemExit( "Local staging immigration gate failed: " + "; ".join(gate.failures) diff --git a/tools/build_us_acs_local_release.py b/tools/build_us_acs_local_release.py index 21f8a44c50..f6bb375f21 100644 --- a/tools/build_us_acs_local_release.py +++ b/tools/build_us_acs_local_release.py @@ -4426,9 +4426,9 @@ def _require_local_immigration(staging_summary: dict) -> dict: the H5. A summary without both is a pre-#1020 staging run, whose ACS persons would all be default-filled citizens with a valid SSN and whose entry clocks would all be the engine default of 5 years. A receipt whose - worker and student targets are not the donor residual + targets are not the donor residuals (:func:`acs_local_immigration_receipt_failures`) predates the #1052 - review and is refused too. + review or the microcosm#767 humanitarian residuals and is refused too. """ receipt = staging_summary.get("acs_local_immigration") @@ -4451,7 +4451,7 @@ def _require_local_immigration(staging_summary: dict) -> dict: if failures: raise SystemExit( "The staging summary's ACS local immigration receipt does not " - "target the donor residual (microcosm#1052 review): " + "target the donor residuals (microcosm#1052 review, microcosm#767): " + "; ".join(failures) + ". Re-run staging (tools/build_us_acs_multispine_base.py) with " "the current builder." @@ -4842,8 +4842,9 @@ def do_finalize(args) -> None: # microcosm#1020: refuse immigration labels or entry clocks that are # missing, all-CITIZEN or all at the engine default on either spine, # labels that contradict measured ACS CIT citizenship, and a file outside - # the non-citizen and undocumented-anchor bands. - immigration_gate = acs_local_immigration_signal_gate(frame) + # the non-citizen, undocumented-anchor and humanitarian-stock bands + # (microcosm#767). + immigration_gate = acs_local_immigration_signal_gate(frame, time_period=PERIOD) # microcosm#1021: refuse ACS disability flags that are missing, constant, # or not the native ACS difficulty items on the packaged bytes, or a # staging receipt that shows them imputed. weeks_worked is graded at diff --git a/tools/build_us_fiscal_refresh_release.py b/tools/build_us_fiscal_refresh_release.py index 3b3a4575ed..32f97686b0 100644 --- a/tools/build_us_fiscal_refresh_release.py +++ b/tools/build_us_fiscal_refresh_release.py @@ -44,7 +44,7 @@ import time import tomllib import uuid -from collections.abc import Callable, Iterable, Mapping, Sequence +from collections.abc import Callable, Iterable, Mapping, MutableMapping, Sequence from contextlib import contextmanager, nullcontext from datetime import UTC, datetime from pathlib import Path @@ -286,6 +286,10 @@ require_authenticated_us_multispine_pool_h5, us_multispine_pool_release_receipt, ) +from microcosm.build.us_runtime.immigration import ( + us_immigration_controls, + us_immigration_humanitarian_draw_mask, +) from microcosm.build.us_runtime.input_mass import us_input_mass_totals from microcosm.build.us_runtime.l0_refit_export import ( attach_l0_refit_entity_weights, @@ -615,14 +619,18 @@ class PoolReleaseIdentityMismatchError(ValueError): # 11: target-frame checkpoint columns now preserve nullable booleans as # canonical bool values plus an explicit uint8 null mask. Older checkpoints # cannot attest this lossless physical representation. -# 12: the identity now carries staged_frame_sha256, a digest of the staged -# frame handed to the materializer (microcosm#1018), so a staging change that +# 12: the identity carries staged_frame_sha256, a digest of the staged frame +# handed to the materializer (microcosm#1018), so a staging change that # forgets this bump can no longer reuse stale target columns. Checkpoints # written without the digest cannot attest their staged frame. # 13: the base simulation runs over household batches and its compilation # carries a batching receipt (microcosm#956), so checkpoints written by the # whole-pool base pass miss too. -TARGET_FRAME_CHECKPOINT_MATERIALIZER_VERSION = 13 +# 14: source-aware humanitarian-stock targets materialize from the paired +# immigration labels and native ASEC/ACS origin/arrival evidence +# (microcosm#767); earlier checkpoints cannot attest those per-draw +# calibration columns or mask semantics. +TARGET_FRAME_CHECKPOINT_MATERIALIZER_VERSION = 14 DEFAULT_MAXIMUM_MICROSIM_BATCH_SIZE = 5_000 DEFAULT_L0_REFIT_LAMBDA_SHARE = 0.8 DEFAULT_US_FISCAL_CALIBRATION_EPOCHS = 1_500 @@ -2667,6 +2675,96 @@ def _selection_mass_protection_specs( return tuple(specs) +def _humanitarian_immigration_stock_specs( + *, + time_period: int = PERIOD, +) -> tuple[TargetSpec, ...]: + """Return one positive calibration target per manifest humanitarian draw. + + These are builder-owned preservation targets, not Ledger fiscal facts, so + the caller adds them only after the fiscal target-parity gate has checked + the feed registry. Values and citations come directly from the same + immigration source-stage manifest that assigns the person labels. + """ + + specs: list[TargetSpec] = [] + for draw in us_immigration_controls().humanitarian: + if draw.target <= 0: + continue + measure = f"humanitarian_immigration_stock.{draw.label.replace(':', '.')}" + metadata = { + "materializer": "humanitarian_immigration_stock", + "measure_mode": "indicator_sum", + "target_role": "humanitarian_immigration_stock", + "humanitarian_draw": draw.label, + "humanitarian_category": draw.category, + "humanitarian_status": draw.status, + "issue": "PolicyEngine/microcosm#767", + } + if draw.origin is not None: + metadata["humanitarian_origin"] = draw.origin + specs.append( + TargetSpec( + name=measure, + entity="household", + value=draw.target, + measure=measure, + period=time_period, + source=draw.source, + family="humanitarian_immigration_stock", + metadata=metadata, + ) + ) + return tuple(specs) + + +def _materialize_humanitarian_immigration_stock_targets( + *, + frame: Frame, + household: MutableMapping[str, np.ndarray], + target_specs: Sequence[TargetSpec], + time_period: int = PERIOD, +) -> None: + """Collapse source-aware humanitarian person indicators to households. + + Every mask is person-local (emitted status plus that person's own source + evidence), so the base pass may run this per household batch. + """ + + humanitarian_specs = [ + spec + for spec in target_specs + if spec.metadata.get("materializer") == "humanitarian_immigration_stock" + ] + if not humanitarian_specs: + return + controls = us_immigration_controls() + draws = {draw.label: draw for draw in controls.humanitarian} + if len(draws) != len(controls.humanitarian): + raise RuntimeError("Humanitarian immigration controls have duplicate draws.") + for spec in humanitarian_specs: + draw_label = spec.metadata.get("humanitarian_draw") + if draw_label not in draws: + raise RuntimeError( + f"Humanitarian target {spec.name!r} refers to unknown manifest " + f"draw {draw_label!r}." + ) + mask = np.asarray( + us_immigration_humanitarian_draw_mask( + frame, + draws[draw_label], + time_period=time_period, + ), + dtype=bool, + ) + if mask.shape != (frame.n("person"),): + raise RuntimeError( + f"Humanitarian draw {draw_label!r} returned mask shape " + f"{mask.shape}; expected {(frame.n('person'),)}." + ) + household[spec.measure] = _collapse_person(frame, mask.astype(np.float64)) + + # The byte framing of _staged_frame_sha256. It is the first field hashed, so a # framing change must rename it and digests under two codecs never compare # equal. @@ -6457,6 +6555,12 @@ def _base_simulation_household_columns( variable="state_income_tax", tax_unit_positions=tax_unit_positions, ) + _materialize_humanitarian_immigration_stock_targets( + frame=frame, + household=columns, + target_specs=target_specs, + time_period=PERIOD, + ) # Federal Student Aid portfolio targets (microcosm#1074) read the SCF # education-debt stage's stored person balances, not an engine variable: # the pinned engine does not define them yet, and a stored input is what @@ -13261,7 +13365,10 @@ def _main(argv: Sequence[str] | None = None) -> int | None: for failure in target_parity_gate.failures ) ) - target_specs = target_registry.specs + target_specs = ( + *target_registry.specs, + *_humanitarian_immigration_stock_specs(time_period=PERIOD), + ) active_target_registry = TargetRegistry(target_specs, country="us") # SSI take-up wiring resolves as soon as the registry exists (fail-fast, # microcosm#507/#508): the band targets come from the same ledger-fed @@ -13865,7 +13972,7 @@ def _main(argv: Sequence[str] | None = None) -> int | None: seed=args.seed, time_period=PERIOD, ) - immigration_gate = us_immigration_composition_gate(base_frame) + immigration_gate = us_immigration_composition_gate(base_frame, time_period=PERIOD) if not immigration_gate.passed: if telemetry is not None: telemetry.stage( @@ -15496,6 +15603,13 @@ def _main(argv: Sequence[str] | None = None) -> int | None: ) else: export_frame = _with_l0_refit_weights(base_frame, result) + # Calibration can redistribute person stocks even though the persisted + # labels remain structurally valid. Re-evaluate the composition contract + # at delivered weights and use this final verdict in every terminal gate + # and release artifact below. + final_immigration_gate = us_immigration_composition_gate( + export_frame, time_period=PERIOD + ) result = _without_calibrated_frames( result, export_frame=export_frame if ladder_outcome is not None else None, @@ -15813,7 +15927,7 @@ def _main(argv: Sequence[str] | None = None) -> int | None: health_input_gate, base_population_gate, incumbent_diagnostics, - immigration_gate, + final_immigration_gate, enforced_input_mass_reference_gate, degenerate_input_gate, ecps_parity_gate=enforced_ecps_parity_gate, @@ -15876,7 +15990,7 @@ def _main(argv: Sequence[str] | None = None) -> int | None: target_profile_gate=target_profile_gate, health_input_gate=health_input_gate, base_population_gate=base_population_gate, - immigration_gate=immigration_gate, + immigration_gate=final_immigration_gate, input_mass_reference_gate=input_mass_reference_gate, hours_worked_gate=hours_worked_gate, snap_take_up_gate=snap_take_up_gate, @@ -16724,7 +16838,7 @@ def _main(argv: Sequence[str] | None = None) -> int | None: health_input_gate=health_input_gate, base_population_gate=base_population_gate, incumbent_diagnostics=incumbent_diagnostics, - immigration_gate=immigration_gate, + immigration_gate=final_immigration_gate, input_mass_reference_gate=enforced_input_mass_reference_gate, degenerate_input_gate=degenerate_input_gate, ecps_parity_gate=enforced_ecps_parity_gate, diff --git a/tools/build_us_multispine_pool.py b/tools/build_us_multispine_pool.py index 68abde6c13..bd5c75e7ff 100644 --- a/tools/build_us_multispine_pool.py +++ b/tools/build_us_multispine_pool.py @@ -6,9 +6,10 @@ ``stack -> geography -> gap-fill -> PUF pass + tail -> late DAG -> derive -> seed -> simulate -> gates``. Both survey arms use one composition-preserving ``--sample-fraction``; PUF -donors always remain full. The terminal completeness gate plus by-origin -battery replace two-spine agreement. ``--legacy-two-spine`` retains the -retiring pipeline byte-for-byte for reproducibility. +donors always remain full. The terminal completeness, by-origin, and +humanitarian-immigration gates replace two-spine agreement. +``--legacy-two-spine`` retains the retiring pipeline byte-for-byte for +reproducibility. Every input is local and explicitly SHA-pinned; this tool never downloads data. It writes a nullable input-only H5 plus a manifest and terminal gate @@ -135,6 +136,7 @@ ACS_2022_RENT_ARTIFACT_SHA256, load_acs_2022_rent_donor, ) +from microcosm.build.us_runtime.immigration import us_immigration_composition_gate from microcosm.build.us_runtime.multispine_pool import ( POOL_CHECKPOINT_STAGE_ORDER, POOL_DERIVE_OPERATOR_ORDER, @@ -291,6 +293,9 @@ # 7: Stacked primary-PUF output universes are explicit. Earlier envelopes can # contain nulls outside the PUF clone for an output declared over the whole # pool and therefore cannot resume safely even when their bank is reusable. +# 8: ACS native citizenship/origin/arrival inputs, pooled immigration control +# scaling, and hard post-transfer humanitarian reconciliation change the +# assembled and transferred outputs. Earlier checkpoints cannot resume. # # Bump this version whenever any producer above changes a stage output without # changing one of the explicit identity fields below. In particular, adding, @@ -305,7 +310,7 @@ # normalizes that logical view in memory. Moving between those encodings does # not change a producer's scalar output and therefore does not advance this # ledger; changing string values or the canonical logical dtype policy does. -POOL_STAGE_CHECKPOINT_MATERIALIZER_VERSION = 7 +POOL_STAGE_CHECKPOINT_MATERIALIZER_VERSION = 8 _PRIMARY_QRF_N_ESTIMATORS = 100 _ACS_TRANSFER_N_ESTIMATORS = 100 @@ -336,7 +341,10 @@ _STACKED_CHECKPOINT_IDENTITY_ARTIFACT_KIND = ( "populace_us_stacked_pool_checkpoint_identity" ) -# Version 13 binds the portable authenticated primary-QRF worker identity. +# Version 13 binds native ACS humanitarian eligibility, pooled immigration +# control scaling, reconciliation, the final composition gate, and the +# portable authenticated primary-QRF worker identity. Earlier stacked +# checkpoints predate the source-aware status surface. # Version 12 binds the post-assembly household geography assignment authority, # target vintage, algorithm, operator order, and seed. Earlier checkpoints # predate the congressional-district support required by release preflight. @@ -399,14 +407,14 @@ class PoolBuildOutputs: @dataclass(frozen=True) class StackedPoolBuildResult: - """Input-only stacked pool and its two fresh terminal gate verdicts.""" + """Input-only stacked pool and its fresh terminal gate verdicts.""" frame: Frame stack_receipt: Mapping[str, object] assembly_receipt: Mapping[str, object] provenance_counts: Mapping[str, Mapping[str, object]] stage_receipts: Mapping[str, Mapping[str, object]] - terminal_gates: tuple[GateResult, GateResult] + terminal_gates: tuple[GateResult, ...] release_id: str qbi_transition_authority_sha256: str | None = None late_producer_transition_authority_sha256: str | None = None @@ -4055,9 +4063,10 @@ def primary_puf_producer(primary_input: Frame): simulation_frame, tail_manifest=tail_manifest, ) + immigration = us_immigration_composition_gate(current, time_period=POOL_TIME_PERIOD) # Manifest conversion is itself the final canonical-authority check and # deliberately happens before publication or readiness is asserted. - GateReport((completeness, battery)).to_manifest() + GateReport((completeness, battery, immigration)).to_manifest() mark_phase("terminal_gates") counts = spine_provenance_counts( current, @@ -4079,7 +4088,7 @@ def primary_puf_producer(primary_input: Frame): assembly_receipt=dict(assembly_receipt), provenance_counts=counts, stage_receipts=receipts, - terminal_gates=(completeness, battery), + terminal_gates=(completeness, battery, immigration), release_id=release_id, qbi_transition_authority_sha256=qbi_transition_authority_sha256, late_producer_transition_authority_sha256=( diff --git a/tools/build_us_puf_support_base.py b/tools/build_us_puf_support_base.py index 93ab88ad90..2556d978ab 100644 --- a/tools/build_us_puf_support_base.py +++ b/tools/build_us_puf_support_base.py @@ -63,6 +63,7 @@ ASEC_2023_WEEKS_UNEMPLOYED_SOURCE_YEAR, ASEC_2023_WEEKS_UNEMPLOYED_ZIP_URL, ASEC_EDUCATION_ASSISTANCE_ARCHIVES, + ASEC_LABOR_FORCE_STATUS_COLUMN, ASEC_RAW_STAGE_ARTIFACT_KIND, ASEC_RAW_STAGE_CHECKPOINT_FILENAME, ASEC_RAW_STAGE_OPERATOR_STATUS, @@ -78,6 +79,7 @@ PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS, PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS, PUF_TAX_DETAIL_SUPPORT_CHANNEL, + US_IMMIGRATION_OUTPUT_COLUMNS, US_PUF_SUPPORT_FIT_NAME, US_SOURCE_MANIFEST, US_SUPPORT_SPINE_SPEC, @@ -92,6 +94,7 @@ fetch_asec_2023_weeks_unemployed_source, fill_asec_2022_weeks_unemployed_source, fill_asec_education_assistance_source, + fill_asec_labor_force_status_source, fill_asec_public_assistance_type_source, finalize_puf_e01000_reconciliation, impute_us_housing_assistance_to_puf_support, @@ -367,7 +370,8 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: "maps to the survey-year YYYY+1 archive). Source construction " "restores from it the reviewed Census person columns the " "--asec-h5 input lacks (microcosm #720); the raw-stage mapping " - "restores ED_VAL and PAW_TYP; the SPM independence role stage " + "restores ED_VAL and PAW_TYP; the immigration stage reads A_LFSR " + "from it; the SPM independence role stage " "derives the role. Years without a mapping are fetched from the " "official Census archive and verified against the same pins." ), @@ -1104,6 +1108,80 @@ def _write_policyengine_dataset( ) +def _with_asec_labor_force_status_source( + frame: Frame, + source: pd.DataFrame, +) -> Frame: + """Attach measured A_LFSR without mutating the source-construction frame.""" + + tables = {entity: frame.table(entity).copy(deep=True) for entity in frame.entities} + tables["person"] = fill_asec_labor_force_status_source(tables["person"], source) + return Frame( + tables, + frame.schema, + { + entity: Weights( + frame.weights_for(entity).values.copy(), + frame.weights_for(entity).kind, + ) + for entity in frame.weighted_entities + }, + frame.strata.copy(deep=True), + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +def _with_us_immigration_inputs_from_asec_source( + frame: Frame, + source: pd.DataFrame, + *, + seed: int, + time_period: int, +) -> Frame: + """Run immigration with measured A_LFSR as an ephemeral raw input.""" + + person = frame.table("person") + if set(US_IMMIGRATION_OUTPUT_COLUMNS).issubset(person.columns): + return with_us_immigration_inputs( + frame, + seed=seed, + time_period=time_period, + ) + carried_labor_force_status = ASEC_LABOR_FORCE_STATUS_COLUMN in person + staged = ( + frame + if carried_labor_force_status + else _with_asec_labor_force_status_source(frame, source) + ) + result = with_us_immigration_inputs( + staged, + seed=seed, + time_period=time_period, + ) + if carried_labor_force_status: + return result + + tables = { + entity: result.table(entity).copy(deep=True) for entity in result.entities + } + tables["person"] = tables["person"].drop(columns=[ASEC_LABOR_FORCE_STATUS_COLUMN]) + return Frame( + tables, + result.schema, + { + entity: Weights( + result.weights_for(entity).values.copy(), + result.weights_for(entity).kind, + ) + for entity in result.weighted_entities + }, + result.strata.copy(deep=True), + mass_log=result.mass_log, + metadata=result.metadata, + ) + + def _run_all( args: argparse.Namespace, *, @@ -1284,8 +1362,9 @@ def _run_all( "ESI premium signal gate failed before support cloning:\n " + "\n ".join(esi_premiums_gate.failures) ) - base = with_us_immigration_inputs( + base = _with_us_immigration_inputs_from_asec_source( base, + education_assistance_source, seed=args.seed, time_period=args.target_year, ) @@ -2201,6 +2280,7 @@ def _asec_raw_source_mapping_frame( weeks_source, ) person = fill_asec_education_assistance_source(person, education_source) + person = fill_asec_labor_force_status_source(person, education_source) person = fill_asec_public_assistance_type_source( person, public_assistance_type_source, @@ -2245,6 +2325,14 @@ def _asec_raw_source_mapping_frame( "operation": "exact_source_join", "source_pins": education_pins, }, + "A_LFSR": { + "audit": dict(education_source.attrs.get("source_audit", {})), + "column": "A_LFSR", + "entity": "person", + "join_keys": ["source_year", "PERIDNUM"], + "operation": "exact_source_join", + "source_pins": education_pins, + }, "LKWEEKS": { "audit": dict(weeks_source.attrs.get("source_audit", {})), "column": "LKWEEKS", @@ -2261,8 +2349,8 @@ def _asec_raw_source_mapping_frame( } ], }, - # PAW_TYP lives in the same pinned survey-year person members as - # ED_VAL, so the mapping reuses those archive pins (microcosm#591). + # PAW_TYP and A_LFSR live in the same pinned survey-year person members + # as ED_VAL, so their mappings reuse those archive pins. "PAW_TYP": { "audit": dict(public_assistance_type_source.attrs.get("source_audit", {})), "column": "PAW_TYP", @@ -2285,6 +2373,10 @@ def _pre_clone_enrichment_stage( _asec_education_source_paths(args), income_years=_pooled_income_years(args), ) + education_assistance_source = load_asec_education_assistance_sources( + _asec_education_source_paths(args), + income_years=_pooled_income_years(args), + ) base = derive_us_cps_carried_inputs( raw_base, public_assistance_type_source=public_assistance_type_source, @@ -2422,8 +2514,9 @@ def _pre_clone_enrichment_stage( us_esi_premiums_signal_gate(base), "ESI premium signal gate failed before support cloning", ) - base = with_us_immigration_inputs( + base = _with_us_immigration_inputs_from_asec_source( base, + education_assistance_source, seed=args.seed, time_period=args.target_year, ) diff --git a/tools/build_us_release_input_coverage_manifest.py b/tools/build_us_release_input_coverage_manifest.py index 20c6cc4f23..326a094f5a 100644 --- a/tools/build_us_release_input_coverage_manifest.py +++ b/tools/build_us_release_input_coverage_manifest.py @@ -1597,6 +1597,153 @@ ), "issue": "PolicyEngine/microcosm#312", }, + { + "id": "hr1_medicaid_humanitarian_eligibility_restoration", + "name": "H.R.1 SS71109 Medicaid humanitarian-status restoration", + "parameter_changes": { + "gov.hhs.medicaid.eligibility.eligible_immigration_statuses": { + "2026-10-01.2100-12-31": [ + "CITIZEN", + "LEGAL_PERMANENT_RESIDENT", + "REFUGEE", + "ASYLEE", + "DEPORTATION_WITHHELD", + "CUBAN_HAITIAN_ENTRANT", + "CONDITIONAL_ENTRANT", + "PAROLED_ONE_YEAR", + ] + } + }, + "budget_measure": "medicaid_enrolled", + "measure_unit": "persons", + "period": 2027, + "effect_direction": "reform_minus_baseline", + "expected_sign": "positive", + "binding_inputs": ["immigration_status_str"], + "min_abs_effect": 10_000.0, + "reason": ( + "H.R.1 SS71109 narrows the Medicaid qualified-immigrant list to " + "citizens, LPRs, and Cuban/Haitian entrants from 2026-10-01. The " + "probe restores the pre-narrowing list at 2027 law, adding " + "REFUGEE, ASYLEE, DEPORTATION_WITHHELD, PAROLED_ONE_YEAR and " + "CONDITIONAL_ENTRANT; the source stage draws no conditional " + "entrants, so the reform binds only through the first four values " + "of immigration_status_str. The probe counts weighted people in " + "medicaid_enrolled rather than summing medicaid dollars, because " + "PolicyEngine-US prices each enrollee through " + "medicaid_slcsp_state_denominator, an aggregate over the whole " + "file that the household-batched post-export scorer refuses " + "(PolicyEngine/microcosm#956). On the default-pool stage receipt " + "(docs/evidence/us-immigration-statuses-767), 0.391 million of " + "the 0.714 million weighted people in the restored statuses " + "report Medicaid at interview, and the Medicaid take-up stage " + "sets takes_up_medicaid_if_eligible for every reporter. The " + "reform only widens the status list, so every reporter whom the " + "narrowed list alone kept out enrolls. The 10,000-person floor is " + "2.6% of the reporters: a structural zero fails it, and the effect " + "clears it unless Medicaid's other rules keep out more than 97% of " + "the reporters. A ~0 count " + "means the humanitarian statuses regressed to zero records " + "(microcosm #767's silent-zero failure) or the engine channel " + "broke." + ), + "issue": "PolicyEngine/microcosm#767", + }, + { + "id": "hr1_aca_below_fpl_exception_restoration", + "name": "H.R.1 SS71302 ACA below-FPL immigrant exception restoration", + "parameter_changes": { + "gov.aca.below_fpl_immigration_exception_in_effect": { + "2026-01-01.2026-12-31": True + } + }, + "budget_measure": "aca_ptc", + "period": 2026, + "effect_direction": "reform_minus_baseline", + "expected_sign": "positive", + "binding_inputs": ["immigration_status_str"], + "min_abs_effect": 5_000_000.0, + "reason": ( + "H.R.1 SS71302 repeals the 26 USC 36B(c)(1)(B) below-FPL " + "lawfully-present exception from 2026. Restoring it for 2026 " + "binds, at 2026 law, only through tax units containing a " + "non-citizen who is ACA-lawfully-present yet " + "Medicaid-ineligible by status - on this file exactly the TPS " + "population the source stage draws to the CRS RS20844 per-country " + "stocks (0.860 million weighted persons on the default-pool stage " + "receipt, docs/evidence/us-immigration-statuses-767; the other " + "humanitarian classes remain Medicaid-qualified at 2026 annual " + "parameters). " + "Below-poverty TPS units with marketplace take-up must regain " + "premium tax credits above the floor; ~$0 means the TPS records " + "vanished or the exception channel broke." + ), + "issue": "PolicyEngine/microcosm#767", + }, + { + "id": "hr1_aca_lawful_presence_restoration", + "name": "H.R.1 SS71301 ACA lawful-presence list restoration", + "parameter_changes": { + "gov.aca.ineligible_immigration_statuses": { + "2027-01-01.2100-12-31": ["DACA", "UNDOCUMENTED"] + } + }, + "budget_measure": "aca_ptc", + "period": 2027, + "effect_direction": "reform_minus_baseline", + "expected_sign": "positive", + "binding_inputs": ["immigration_status_str"], + "min_abs_effect": 10_000_000.0, + "reason": ( + "H.R.1 SS71301 adds TPS/REFUGEE/ASYLEE/DEPORTATION_WITHHELD/" + "PAROLED_ONE_YEAR to the ACA ineligible-status list from " + "2027-01-01. The probe restores the pre-H.R.1 list (DACA and " + "UNDOCUMENTED only) at 2027, re-qualifying every humanitarian " + "status the source stage draws (1.574 million weighted persons " + "across parole, refugee, asylee and TPS on the default-pool stage " + "receipt). Units among them with " + "marketplace take-up must regain premium tax credits above the " + "floor; ~$0 means the humanitarian statuses regressed or the " + "lawful-presence channel broke." + ), + "issue": "PolicyEngine/microcosm#767", + }, + { + "id": "hr1_snap_humanitarian_eligibility_restoration", + "name": "H.R.1 SS10108 SNAP humanitarian-status restoration", + "parameter_changes": { + "gov.usda.snap.eligibility.eligible_immigration_statuses": { + "2025-07-01.2100-12-31": [ + "CITIZEN", + "LEGAL_PERMANENT_RESIDENT", + "REFUGEE", + "ASYLEE", + "DEPORTATION_WITHHELD", + "CUBAN_HAITIAN_ENTRANT", + "CONDITIONAL_ENTRANT", + "PAROLED_ONE_YEAR", + ] + } + }, + "budget_measure": "snap", + "period": 2026, + "effect_direction": "reform_minus_baseline", + "expected_sign": "positive", + "binding_inputs": ["immigration_status_str"], + "min_abs_effect": 25_000_000.0, + "reason": ( + "H.R.1 SS10108 narrows SNAP alien eligibility to citizens, LPRs, " + "Cuban/Haitian entrants, and COFA citizens from 2025-07-01 " + "(modeled 2025-07). Restoring the pre-H.R.1 list at 2026 " + "re-includes the REFUGEE/ASYLEE/DEPORTATION_WITHHELD/" + "PAROLED_ONE_YEAR members the source stage draws to cited " + "DHS/OHSS stocks; is_snap_excluded_member stops excluding them, " + "so SNAP allotments for their households must rise above the " + "floor. ~$0 means the humanitarian statuses regressed to zero " + "records or the SNAP immigration channel broke." + ), + "issue": "PolicyEngine/microcosm#767", + }, ] diff --git a/tools/generate_us_bundle_from_constants.py b/tools/generate_us_bundle_from_constants.py index c8b0dc9443..c715197388 100644 --- a/tools/generate_us_bundle_from_constants.py +++ b/tools/generate_us_bundle_from_constants.py @@ -122,7 +122,7 @@ def _domain_builders() -> dict[str, Callable[[], object]]: # frozen files, is the forward YAML -> legacy-payload path. FROZEN_LEGACY_RESOURCE_SHA256 = { "source_stages.json": ( - "959214c437bfa59d43e32279b82f29a0987649cd7579ff87567c70f084eae66c" + "67587983a455bfffde688b024e7d4bd03a9d1d1f3b79291881558e60bb20311a" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" @@ -142,6 +142,7 @@ def _domain_builders() -> dict[str, Callable[[], object]]: "ecps_parity_reference.json", "federal_eitc_by_state.json", "fiscal_target_references.json", + "legacy_schema16_late_producer_schedule.json", "obbba_reforms.json", "puf_aggregate_record_disaggregation.json", "release_input_coverage_manifest.json", diff --git a/tools/spec_engine_coverage.py b/tools/spec_engine_coverage.py index 8e6c05a0c8..46f72d3436 100644 --- a/tools/spec_engine_coverage.py +++ b/tools/spec_engine_coverage.py @@ -41,7 +41,7 @@ REPORT_SCHEMA_VERSION = 3 EXPECTED_POINTER_INVENTORY_SHA256 = ( - "f6832e8d2892af22e41afd0b0bea0cba354bbff8b84bebcc652e765a3ae7df92" + "439d805c20da730deaa1aec21b94b635d2667075dda4aec542815d4a84e8a5b3" ) DEFAULT_REPORT_PATH = ( Path(__file__).resolve().parents[1] diff --git a/tools/us_bundle_generation/identity_contracts.py b/tools/us_bundle_generation/identity_contracts.py index 06b2bf04d8..8173dabf36 100644 --- a/tools/us_bundle_generation/identity_contracts.py +++ b/tools/us_bundle_generation/identity_contracts.py @@ -102,7 +102,7 @@ def _stages_with_operation( def build_seed_site_bindings( source_document: Mapping[str, Any], ) -> list[dict[str, object]]: - """Bind all 53 legacy-v1 sites to their concrete execution owners.""" + """Bind all 66 legacy-v1 sites to their concrete execution owners.""" source_ids = _source_stage_ids(source_document) transfer_nodes = tuple(group.name for group in CANONICAL_US_LATE_TRANSFER_GROUPS) @@ -173,6 +173,24 @@ def build_seed_site_bindings( "snap_discretionary_exemption_assignment": source( "snap_abawd_discretionary_exemption" ), + **{ + f"immigration_humanitarian_{label}_assignment": source("immigration_status") + for label in ( + "paroled_one_year_afghanistan", + "paroled_one_year_ukraine", + "paroled_one_year_nicaragua", + "paroled_one_year_venezuela", + "refugee", + "asylee", + "deportation_withheld", + "tps_venezuela", + "tps_el_salvador", + "tps_honduras", + "tps_nicaragua", + "tps_nepal", + "tps_other_designated", + ) + }, "immigration_ead_workers_assignment": source("immigration_status"), "immigration_ead_students_assignment": source("immigration_status"), "ssi_take_up_assignment": source("ssi_take_up"), @@ -203,9 +221,9 @@ def build_seed_site_bindings( } protocol_site_ids = tuple(site.id for site in LEGACY_V1_PROTOCOL.sites) - if len(protocol_site_ids) != 53 or set(owners) != set(protocol_site_ids): + if len(protocol_site_ids) != 66 or set(owners) != set(protocol_site_ids): raise RuntimeError( - "legacy-v1 seed owner ledger must cover exactly 53 protocol sites; " + "legacy-v1 seed owner ledger must cover exactly 66 protocol sites; " f"missing={sorted(set(protocol_site_ids) - owners.keys())!r}, " f"extra={sorted(owners.keys() - set(protocol_site_ids))!r}" ) diff --git a/tools/us_bundle_generation/imputation.py b/tools/us_bundle_generation/imputation.py index d0f69f76ce..3cb3c2c872 100644 --- a/tools/us_bundle_generation/imputation.py +++ b/tools/us_bundle_generation/imputation.py @@ -884,6 +884,15 @@ def _normalise_transfer_execution( ) ) adult_care.pop("enabled") + humanitarian_immigration = deepcopy( + dict( + _mapping_like( + post_transfer["humanitarian_immigration"], + "humanitarian-immigration post-transfer contract", + ) + ) + ) + humanitarian_immigration.pop("enabled") passive = deepcopy( dict( _mapping_like( @@ -923,6 +932,17 @@ def _normalise_transfer_execution( }, "contract": adult_care, }, + "humanitarian_immigration": { + "activation": { + "all_targets": list( + _array_like( + result["immigration_status_targets"], + "immigration status targets", + ) + ) + }, + "contract": humanitarian_immigration, + }, "schedule_d_capital_gain_distributions": { "activation": { "derive_schedule_d": True, @@ -2148,7 +2168,7 @@ def _assert_invariants( late_authored_output_count, len(tolerated_receipts), ) - expected_graph_counts = (2747, 92, 230, 35, 0, 214) + expected_graph_counts = (2753, 92, 230, 35, 0, 214) if graph_counts != expected_graph_counts: raise RuntimeError( "US producer graph input/output/absence counts changed: "