From 0f5edbd6c4c89c285796ce8ea1de9e1ddf5bc20f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 25 Aug 2026 15:36:13 +0200 Subject: [PATCH 01/49] Draw humanitarian immigration statuses to cited stocks in the US stage REFUGEE / ASYLEE / PAROLED_ONE_YEAR (per program origin) / TPS (per country) are now selected between the ASEC-UA indicators and the EAD spill, each to a manifest-cited target, so the H.R.1 $71109/$71301/ $71302 and SNAP $10108 eligibility channels stop computing $0 on the release. DEPORTATION_WITHHELD carries an explicit cited zero (no published stock). Residual-pool parole/TPS draws receive EAD SSN cards, keeping the NONE-undocumented invariant; the composition gate now bands each category against its target. Refs #767 Co-Authored-By: Claude Fable 5 --- .../src/microcosm/build/us/source_stages.json | 62 +- .../microcosm/build/us_runtime/__init__.py | 6 + .../microcosm/build/us_runtime/immigration.py | 619 ++++++++++++++++-- 3 files changed, 621 insertions(+), 66 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/us/source_stages.json b/packages/microcosm-build/src/microcosm/build/us/source_stages.json index 5304c4352d..af0b376f54 100644 --- a/packages/microcosm-build/src/microcosm/build/us/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/us/source_stages.json @@ -1879,7 +1879,7 @@ "kind": "administrative_table", "format": "published_estimate", "vintage": "latest_available", - "locator": "Pew Research Center unauthorized-immigrant population and worker estimates; Higher Ed Immigration Portal undocumented-student estimates" + "locator": "Pew Research Center unauthorized-immigrant population and worker estimates; Higher Ed Immigration Portal undocumented-student estimates; OHSS refugee and asylee annual flow reports; DHS OAW/U4U parole reports and CBP CHNV releases; CRS RS20844 TPS designations table; EOIR adjudication statistics via CRS IN12501" } ], "operations": [ @@ -1903,6 +1903,64 @@ "undocumented_population_anchor": { "value": 11000000, "source": "https://www.pewresearch.org/short-reads/2024/07/22/what-we-know-about-unauthorized-immigrants-living-in-the-us/" + }, + "humanitarian_status_stocks": { + "paroled_one_year": { + "afghanistan": { + "target": 73566, + "source": "https://www.dhs.gov/sites/default/files/2023-06/PLCY%20-%20AFG%202503%20Update%202_0.pdf" + }, + "ukraine": { + "target": 158000, + "source": "https://www.dhs.gov/sites/default/files/2024-12/2024_1104_dmo_plcy_uniting_for_ukraine_process_overview_and_assessment.pdf" + }, + "nicaragua": { + "target": 93070, + "source": "https://www.cbp.gov/newsroom/national-media-release/cbp-releases-december-2024-monthly-update" + }, + "venezuela": { + "target": 117330, + "source": "https://www.cbp.gov/newsroom/national-media-release/cbp-releases-december-2024-monthly-update" + } + }, + "refugee": { + "target": 160000, + "source": "https://ohss.dhs.gov/sites/default/files/2024-11/2024_1108_ohss_refugee_annual_flow_report_2023.pdf" + }, + "asylee": { + "target": 160000, + "source": "https://ohss.dhs.gov/sites/default/files/2024-10/2024_1002_ohss_asylees_fy2023.pdf" + }, + "deportation_withheld": { + "target": 0, + "source": "https://crsreports.congress.gov/product/pdf/IN/IN12501" + }, + "tps": { + "venezuela": { + "target": 605015, + "source": "https://www.everycrsreport.com/reports/RS20844.html" + }, + "el_salvador": { + "target": 170125, + "source": "https://www.everycrsreport.com/reports/RS20844.html" + }, + "honduras": { + "target": 51225, + "source": "https://www.everycrsreport.com/reports/RS20844.html" + }, + "nicaragua": { + "target": 2910, + "source": "https://www.everycrsreport.com/reports/RS20844.html" + }, + "nepal": { + "target": 7160, + "source": "https://www.everycrsreport.com/reports/RS20844.html" + }, + "other_designated": { + "target": 21215, + "source": "https://www.everycrsreport.com/reports/RS20844.html" + } + } } } ], @@ -1910,7 +1968,7 @@ "ssn_card_type", "immigration_status_str" ], - "notes": "Citizenship is measured (PRCITSHP), not imputed. Non-citizens with any ASEC-UA legal-status indicator (Van Hook et al., SSRN 4662801) become OTHER_NON_CITIZEN. The CPS carries no work-authorization variable, so among residual non-citizens, workers and students spill to NON_CITIZEN_VALID_EAD in deterministic seeded order until the remaining undocumented worker count matches Pew's 8.3M and the undocumented student count matches the Higher Ed Immigration Portal's ~408k \u2014 the only forced margins. The total undocumented population is emergent (\u224813M on the 2024 ASEC, within the range of published 2023-24 estimates) and is gated against Pew's 11.0M 2022 anchor with a coarse plausibility band rather than forced; reconciling the level is the calibration lane's job. Immigration-status tags carry only statutory tests the data supports: DACA (arrival cohort among EAD holders) and CUBAN_HAITIAN_ENTRANT (nativity plus post-1980 arrival); other documented non-citizens stay LEGAL_PERMANENT_RESIDENT, the modal true status \u2014 blanket REFUGEE/TPS labels are deliberately not fabricated because they would mislabel millions and over-grant refugee-class benefit exemptions." + "notes": "Citizenship is measured (PRCITSHP), not imputed. Non-citizens with any ASEC-UA legal-status indicator (Van Hook et al., SSRN 4662801) become OTHER_NON_CITIZEN. Humanitarian/temporary-protection statuses (microcosm #767) are drawn between the indicators and the EAD spill, sequentially and mutually exclusively in decreasing order of target precision \u2014 PAROLED_ONE_YEAR per program origin (OAW Afghans 73,566 as of 2022-03-31; U4U Ukrainians over 158,000 as of 2023-09-30; CHNV Nicaraguans 93,070 and Venezuelans 117,330 paroled through 2024-12), REFUGEE (FY2023+FY2024 admissions, 60,050+100,060 OHSS \u2014 refugees must apply for LPR adjustment after one year, so the not-yet-adjusted stock is about the trailing two years), ASYLEE (FY2022-FY2024 grants 35,720+54,350+~70,000 OHSS; 88% of 2015-22 affirmative adult grantees had adjusted by end-2024, so about three trailing years), then TPS per country (CRS RS20844 Table 1 as of 2025-03-31, with required-arrival cutoffs binding at PEINUSYR granularity: El Salvador <=2001, Honduras/Nicaragua <=1999, Nepal <=2015). Candidates are restricted by program origin countries (Census Appendix I PENATVTY codes) and arrival windows; Cuba/Haiti-born are excluded everywhere (the CUBAN_HAITIAN_ENTRANT class is the better statutory label and keeps H.R.1 eligibility \u2014 Haiti TPS 330,735 rides that label), the DACA statutory cohort is excluded (dual TPS/DACA holders stay DACA), and Ukraine/Afghanistan TPS registrants (101,150/8,105) are carried by the parole draw since they are overwhelmingly the same U4U/OAW parolees. Parole targets are gross admissions: cohort members who later won asylum keep the parole label, which is channel-equivalent under H.R.1 (both classes lose Medicaid/SNAP/PTC on identical dates). DEPORTATION_WITHHELD carries an explicit zero target: EOIR grants about 2.2k withholdings a year (CRS IN12501: under 1% of 666,177 FY2024 decisions) and publishes no stock, so the accumulated population is below the CPS's resolution for a defensible draw. CONDITIONAL_ENTRANT is deliberately not emitted (the INA 203(a)(7) class closed in 1980). REFUGEE/ASYLEE draw from the indicator-documented pool only; parole/TPS may draw residual-pool people, who then receive NON_CITIZEN_VALID_EAD SSN cards (both programs confer employment authorization), consistent with Pew's own accounting of TPS/parolees inside its unauthorized totals. The EAD worker/student spill then binds the remaining residual pool to Pew's 8.3M workers and the Higher Ed Immigration Portal's ~408k students \u2014 still the only forced margins. The total undocumented population is emergent and gated against Pew's 11.0M 2022 anchor with a coarse band; per-category humanitarian masses are gated against their cited targets with a coarse band (the ASEC undercovers 2022-24 arrivals \u2014 the Ukraine parole pool saturates below its admin count); reconciling levels is the calibration lane's job. DACA (arrival cohort among EAD holders) and CUBAN_HAITIAN_ENTRANT (nativity plus post-1980 arrival) keep their statutory tests; other documented non-citizens stay LEGAL_PERMANENT_RESIDENT, the modal true status." }, { "stage": "hours_worked", diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py index e9e7564735..35dc9b7fa9 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py @@ -371,12 +371,15 @@ with_us_housing_inputs, ) from microcosm.build.us_runtime.immigration import ( + HUMANITARIAN_STATUS_CATEGORIES, IMMIGRATION_STATUS_VALUES, SSN_CARD_TYPE_VALUES, US_IMMIGRATION_NONCONSTANT_PERSON_COLUMNS, US_IMMIGRATION_OUTPUT_COLUMNS, US_IMMIGRATION_REQUIRED_SOURCE_COLUMNS, US_IMMIGRATION_STAGE_NAME, + HumanitarianDraw, + ImmigrationControls, UndocumentedControls, derive_us_immigration_status_from_manifest, us_immigration_composition_gate, @@ -1149,12 +1152,15 @@ "US_JCT_TAX_EXPENDITURE_TARGET_SPECS", "US_JCT_TAX_EXPENDITURE_TARGET_REFERENCES", "SOI_VARIABLE_MAP", + "HUMANITARIAN_STATUS_CATEGORIES", "IMMIGRATION_STATUS_VALUES", "SSN_CARD_TYPE_VALUES", "US_IMMIGRATION_NONCONSTANT_PERSON_COLUMNS", "US_IMMIGRATION_OUTPUT_COLUMNS", "US_IMMIGRATION_REQUIRED_SOURCE_COLUMNS", "US_IMMIGRATION_STAGE_NAME", + "HumanitarianDraw", + "ImmigrationControls", "UndocumentedControls", "US_HOURS_WORKED_NONCONSTANT_PERSON_COLUMNS", "US_HOURS_WORKED_OUTPUT_COLUMNS", diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py b/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py index 9c5bffbf12..48dc63557a 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py @@ -41,16 +41,33 @@ emergent total is ≈13M — inside the range of published 2023–24 estimates — with 26.6M non-citizens, 18.0M residual after the indicators, and 4.8M spilled to EAD. -5. **Status tags carry only statutory tests the data can support.** +5. **Humanitarian/temporary-protection statuses are drawn to published + stocks, never blanketed (microcosm #767).** Blanket ``REFUGEE``/``TPS`` + labels for recent arrivals were originally refused because LPR treatment + gave near-identical means-tested eligibility; H.R.1 repealed exactly that + equivalence (Medicaid §71109 effective 2026-10-01, SNAP §10108 effective + 2025-07-01, ACA §71301/§71302 effective 2026/2027 — all keyed to these + enum values in policyengine-us parameters), so an LPR-only file silently + zeroes every one of those channels. Instead of blankets, the stage now + draws ``REFUGEE`` / ``ASYLEE`` / ``DEPORTATION_WITHHELD`` / + ``PAROLED_ONE_YEAR`` / ``TPS`` — in that order, mutually exclusive — + from candidates whose survey signature supports the status (origin + country x arrival window x legal-status indicators), each to a + manifest-cited weighted stock target. Cuba/Haiti-born persons are + excluded from every humanitarian draw (the ``CUBAN_HAITIAN_ENTRANT`` + class is the better statutory label and keeps H.R.1 eligibility), as is + the DACA statutory cohort. Draws from the residual pool (TPS and parole + only) flip ``ssn_card_type`` ``NONE`` to ``NON_CITIZEN_VALID_EAD`` — + both programs grant employment authorization — so the two output columns + never disagree. ``CONDITIONAL_ENTRANT`` is deliberately not emitted: the + INA 203(a)(7) class closed in 1980 and surviving holders are + indistinguishable from LPRs at this granularity. +6. **Status tags carry only statutory tests the data can support.** ``DACA`` applies the statutory cohort test (arrived by 2007 before age 16, aged 15+) to EAD holders; ``CUBAN_HAITIAN_ENTRANT`` applies the nationality-plus-arrival class to documented non-citizens; every other documented non-citizen stays ``LEGAL_PERMANENT_RESIDENT`` (the modal true - status). Blanket ``REFUGEE``/``TPS`` labels for recent arrivals or - leftover EAD holders are deliberately not emitted: they would mislabel - millions (true stocks are under a million each) and over-grant - refugee-class exemptions in benefit rules, while LPR treatment gives - near-identical means-tested eligibility for those populations. + status). Selection draws are seeded blake2b hashes keyed by the person's stable source identity (``source_year``/``source_person_id`` when present), so @@ -89,12 +106,15 @@ from microcosm.frame.units import US_SCHEMA __all__ = [ + "HUMANITARIAN_STATUS_CATEGORIES", "IMMIGRATION_STATUS_VALUES", "SSN_CARD_TYPE_VALUES", "US_IMMIGRATION_NONCONSTANT_PERSON_COLUMNS", "US_IMMIGRATION_OUTPUT_COLUMNS", "US_IMMIGRATION_REQUIRED_SOURCE_COLUMNS", "US_IMMIGRATION_STAGE_NAME", + "HumanitarianDraw", + "ImmigrationControls", "UndocumentedControls", "derive_us_immigration_status_from_manifest", "us_immigration_composition_gate", @@ -126,13 +146,18 @@ #: PolicyEngine-US ``ImmigrationStatus`` enum member names this stage emits #: (a deliberate subset of the engine's full enum domain; see module -#: docstring for why blanket REFUGEE/TPS labels are not fabricated). +#: docstring for why ``CONDITIONAL_ENTRANT`` is not fabricated). IMMIGRATION_STATUS_VALUES: tuple[str, ...] = ( "CITIZEN", "LEGAL_PERMANENT_RESIDENT", "CUBAN_HAITIAN_ENTRANT", "DACA", "UNDOCUMENTED", + "PAROLED_ONE_YEAR", + "REFUGEE", + "ASYLEE", + "DEPORTATION_WITHHELD", + "TPS", ) #: Raw CPS ASEC person columns the derivation reads. All of them ship in the @@ -169,11 +194,13 @@ _FORCED_CONTROL_KEYS = ("undocumented_workers", "undocumented_students") _ANCHOR_KEY = "undocumented_population_anchor" +_HUMANITARIAN_KEY = "humanitarian_status_stocks" _DERIVE_IMMIGRATION_STATUS_PARAMETER_KEYS = frozenset( { *_FORCED_CONTROL_KEYS, _ANCHOR_KEY, + _HUMANITARIAN_KEY, "seed_from_build_config", "time_period_from_build_config", } @@ -226,6 +253,118 @@ _DACA_MAX_AGE_AT_ENTRY = 16 _DACA_MIN_CURRENT_AGE = 15 +#: Humanitarian draw order: decreasing target precision (exact per-origin +#: program admissions first, national flow aggregates next, per-country TPS +#: registrations last). Draws are sequential and mutually exclusive. +HUMANITARIAN_STATUS_CATEGORIES: tuple[str, ...] = ( + "paroled_one_year", + "refugee", + "asylee", + "deportation_withheld", + "tps", +) + +_HUMANITARIAN_STATUS_BY_CATEGORY: Mapping[str, str] = { + "paroled_one_year": "PAROLED_ONE_YEAR", + "refugee": "REFUGEE", + "asylee": "ASYLEE", + "deportation_withheld": "DEPORTATION_WITHHELD", + "tps": "TPS", +} + +#: PEINUSYR codes for 2020+ arrivals (2024 ASEC data dictionary: 27 = +#: 2020-2021, 28 = 2022-2024) — the humanitarian-parole and recent-refugee +#: cohorts. The Census codebook rebins this variable across vintages, so +#: these windows are survey-vintage facts, not statutory dates. +_RECENT_ARRIVAL_CODES = (27, 28) +#: Asylum grants lag arrival by filing queues plus court backlog; grantees +#: in 2022-2024 overwhelmingly arrived 2016+ (OHSS asylee flow reports). +_ASYLEE_ARRIVAL_CODES = (25, 26, 27, 28) +#: Withholding of removal follows years of proceedings: settled arrivals only. +_WITHHELD_MAX_ARRIVAL_CODE = 26 + +#: PENATVTY birth-country codes, verified against Census CPS technical +#: documentation (cpsmar24.pdf) Appendix I "Countries and Areas of the +#: World". Cuba (327) and Haiti (332) stay out of every humanitarian draw: +#: the Cuban/Haitian-entrant class is the better statutory label. +#: +#: Parole per-origin keys: Operation Allies Welcome (Afghanistan), Uniting +#: for Ukraine, and the non-Cuban/Haitian CHNV nationalities. +_PAROLE_ORIGIN_CODES: Mapping[str, tuple[int, ...]] = { + "afghanistan": (200,), + "ukraine": (164,), + "nicaragua": (315,), + "venezuela": (373,), +} + +#: Top refugee-resettlement nationalities, OHSS Refugees Annual Flow Report +#: FY2023 Table 3 and the OHSS FY-24 report (DR Congo, Syria, Afghanistan, +#: Burma, Venezuela lead; Congo appears as both 412 Congo and 459 Zaire). +_REFUGEE_ORIGIN_CODES: tuple[int, ...] = ( + 412, # Congo + 459, # Zaire (Democratic Republic of the Congo) + 239, # Syria + 200, # Afghanistan + 205, # Myanmar (Burma) + 373, # Venezuela + 448, # Somalia + 451, # Sudan + 417, # Eritrea + 213, # Iraq + 313, # Guatemala + 164, # Ukraine +) + +#: Top asylum-grant nationalities, OHSS Asylees flow reports FY2022-FY2024 +#: (Afghanistan, China, Venezuela, Russia lead; Central/South America and +#: Egypt/Cameroon/Turkey round out the recurring top grant countries). +_ASYLEE_ORIGIN_CODES: tuple[int, ...] = ( + 200, # Afghanistan + 207, # China + 373, # Venezuela + 163, # Russia + 312, # El Salvador + 313, # Guatemala + 314, # Honduras + 315, # Nicaragua + 210, # India + 414, # Egypt + 407, # Cameroon + 364, # Colombia + 365, # Ecuador + 370, # Peru + 243, # Turkey + 239, # Syria +) + +#: TPS per-country keys: birth codes plus the latest PEINUSYR arrival code +#: compatible with the designation's required continuous-residence date +#: (CRS RS20844 Table 1). Legacy designations bind hard (El Salvador +#: 2001-02-13 -> code 17; Honduras/Nicaragua 1998-12-30 -> code 16; Nepal +#: 2015-06-24 -> code 24); 2021+ designations reach the top code, where the +#: 2022-2024 bin unavoidably includes some post-cutoff arrivals. +#: Ukraine and Afghanistan TPS registrants are carried by the parole draw +#: (same U4U/OAW people); Haiti TPS is carried by CUBAN_HAITIAN_ENTRANT. +_TPS_ORIGIN_CODES: Mapping[str, tuple[tuple[int, ...], int]] = { + "venezuela": ((373,), 28), + "el_salvador": ((312,), 17), + "honduras": ((314,), 16), + "nicaragua": ((315,), 16), + "nepal": ((229,), 24), + # Burma, Syria, Yemen, Lebanon, Cameroon, Ethiopia, Somalia, Sudan + # (South Sudan shares 451 in the CPS codebook). + "other_designated": ((205, 239, 248, 224, 407, 416, 448, 451), 28), +} + +#: Categories whose flat manifest block carries one national target. +_FLAT_HUMANITARIAN_CATEGORIES = ("refugee", "asylee", "deportation_withheld") +#: Categories whose manifest block carries per-origin targets, and the +#: module table each block's keys must match exactly. +_PER_ORIGIN_HUMANITARIAN_CATEGORIES: Mapping[str, tuple[str, ...]] = { + "paroled_one_year": tuple(_PAROLE_ORIGIN_CODES), + "tps": tuple(_TPS_ORIGIN_CODES), +} + #: Weighted share of persons whose SSN card type is not ``CITIZEN`` must land #: in this band (Census counts ~22–27M non-citizens of ~336M residents; a #: share outside it means the imputation collapsed or exploded). @@ -235,6 +374,12 @@ #: collapse or explosion, not a calibration objective; the level belongs to #: the calibration lane. _UNDOCUMENTED_ANCHOR_RELATIVE_BAND = (0.5, 1.6) +#: Emitted humanitarian mass per category relative to its cited target. +#: The draw forces the target when candidates suffice, so the band only +#: bites on pool exhaustion (the ASEC undercovers 2022-24 arrivals — the +#: Ukraine parole pool saturates below its admin count) or collapse. A +#: category with an explicit zero target must emit exactly zero. +_HUMANITARIAN_TARGET_RELATIVE_BAND = (0.35, 1.5) @dataclass(frozen=True) @@ -273,6 +418,69 @@ def __post_init__(self) -> None: ) +@dataclass(frozen=True) +class HumanitarianDraw: + """One seeded humanitarian draw: a cited stock the stage selects toward. + + Attributes: + category: Manifest category key (``paroled_one_year`` … ``tps``). + origin: Per-origin key inside the category, or ``None`` for a + national draw. + status: ``ImmigrationStatus`` enum member name the draw emits. + target: Cited weighted stock. Zero is allowed and means the + category is explicitly not imputed (the citation documents why). + source: Citation URL or reference for the target. + """ + + category: str + origin: str | None + status: str + target: float + source: str + + def __post_init__(self) -> None: + if not np.isfinite(self.target) or self.target < 0: + raise ValueError( + f"Humanitarian target {self.label!r} must be non-negative, " + f"got {self.target!r}." + ) + if not self.source: + raise ValueError( + f"Humanitarian target {self.label!r} requires a source citation." + ) + + @property + def label(self) -> str: + return self.category if self.origin is None else f"{self.category}:{self.origin}" + + @property + def salt(self) -> str: + return f"immigration:{self.label}" + + +@dataclass(frozen=True) +class ImmigrationControls: + """Every manifest-sourced control the immigration stage consumes. + + Attributes: + undocumented: The legacy forced margins and the composition-gate + anchor for the residual pool. + humanitarian: The humanitarian draws in assignment order — the + per-origin draws of a category are adjacent, categories follow + ``HUMANITARIAN_STATUS_CATEGORIES``. + """ + + undocumented: UndocumentedControls + humanitarian: tuple[HumanitarianDraw, ...] + + def humanitarian_target(self, category: str) -> float: + """Summed cited target for one category across its origins.""" + + return float( + sum(draw.target for draw in self.humanitarian if draw.category == category) + ) + + def us_immigration_stage_spec() -> SourceStageSpec: """Load the packaged ``immigration_status`` source-stage manifest entry.""" @@ -346,15 +554,17 @@ def derive_us_immigration_status_from_manifest( raise SourceRuntimeError( "US immigration derivation requires finite non-negative person weights." ) - ssn_codes = _assign_ssn_card_codes( + ssn_codes, humanitarian_marks = _assign_ssn_card_codes( result, weights.to_numpy(dtype=np.float64), seed=int(context.config.seed), controls=controls, + time_period=int(context.config.target_year), ) status = _derive_immigration_status( result, ssn_codes, + humanitarian_marks, time_period=int(context.config.target_year), ) code_to_name = { @@ -368,7 +578,119 @@ def derive_us_immigration_status_from_manifest( return result -def _controls_from_parameters(params: Mapping[str, object]) -> UndocumentedControls: +def _target_and_source( + block: object, + *, + label: str, + value_key: str, + allow_zero: bool, +) -> tuple[float, str]: + if not isinstance(block, Mapping): + raise SourceRuntimeError( + f"US immigration control {label!r} requires an object with " + f"{value_key!r} and 'source'." + ) + unexpected = sorted(set(block) - {value_key, "source"}) + if unexpected: + raise SourceRuntimeError( + f"US immigration control {label!r} has unsupported key(s): {unexpected}." + ) + value = block.get(value_key) + source = block.get("source") + valid_number = isinstance(value, int | float) and not isinstance(value, bool) + if not valid_number or (float(value) <= 0 and not (allow_zero and value == 0)): + requirement = "non-negative" if allow_zero else "positive" + raise SourceRuntimeError( + f"US immigration control {label!r} requires a {requirement} numeric " + f"{value_key!r}." + ) + if not isinstance(source, str) or not source: + raise SourceRuntimeError( + f"US immigration control {label!r} requires a source citation." + ) + return float(value), source + + +def _humanitarian_draws_from_parameters( + params: Mapping[str, object], +) -> tuple[HumanitarianDraw, ...]: + block = params.get(_HUMANITARIAN_KEY) + if not isinstance(block, Mapping): + raise SourceRuntimeError( + f"US immigration derivation requires a {_HUMANITARIAN_KEY!r} object " + f"with one block per category {list(HUMANITARIAN_STATUS_CATEGORIES)}." + ) + unexpected = sorted(set(block) - set(HUMANITARIAN_STATUS_CATEGORIES)) + if unexpected: + raise SourceRuntimeError( + f"US immigration {_HUMANITARIAN_KEY} has unsupported category(ies): " + f"{unexpected}." + ) + missing = sorted(set(HUMANITARIAN_STATUS_CATEGORIES) - set(block)) + if missing: + raise SourceRuntimeError( + f"US immigration {_HUMANITARIAN_KEY} is missing category(ies): " + f"{missing}." + ) + draws: list[HumanitarianDraw] = [] + for category in HUMANITARIAN_STATUS_CATEGORIES: + status = _HUMANITARIAN_STATUS_BY_CATEGORY[category] + category_block = block[category] + if category in _PER_ORIGIN_HUMANITARIAN_CATEGORIES: + origin_keys = _PER_ORIGIN_HUMANITARIAN_CATEGORIES[category] + if not isinstance(category_block, Mapping): + raise SourceRuntimeError( + f"US immigration control {category!r} requires per-origin " + f"blocks {list(origin_keys)}." + ) + unexpected_origins = sorted(set(category_block) - set(origin_keys)) + if unexpected_origins: + raise SourceRuntimeError( + f"US immigration control {category!r} has origin(s) outside " + f"the stage's codebook table: {unexpected_origins}." + ) + missing_origins = sorted(set(origin_keys) - set(category_block)) + if missing_origins: + raise SourceRuntimeError( + f"US immigration control {category!r} is missing origin(s): " + f"{missing_origins}." + ) + for origin in origin_keys: + target, source = _target_and_source( + category_block[origin], + label=f"{category}:{origin}", + value_key="target", + allow_zero=True, + ) + draws.append( + HumanitarianDraw( + category=category, + origin=origin, + status=status, + target=target, + source=source, + ) + ) + else: + target, source = _target_and_source( + category_block, + label=category, + value_key="target", + allow_zero=True, + ) + draws.append( + HumanitarianDraw( + category=category, + origin=None, + status=status, + target=target, + source=source, + ) + ) + return tuple(draws) + + +def _controls_from_parameters(params: Mapping[str, object]) -> ImmigrationControls: values: dict[str, float] = {} sources: dict[str, str] = {} for key, value_key in ( @@ -376,40 +698,21 @@ def _controls_from_parameters(params: Mapping[str, object]) -> UndocumentedContr ("undocumented_students", "target"), (_ANCHOR_KEY, "value"), ): - block = params.get(key) - if not isinstance(block, Mapping): - raise SourceRuntimeError( - f"US immigration derivation requires a {key!r} object with " - f"{value_key!r} and 'source'." - ) - unexpected = sorted(set(block) - {value_key, "source"}) - if unexpected: - raise SourceRuntimeError( - f"US immigration control {key!r} has unsupported key(s): {unexpected}." - ) - value = block.get(value_key) - source = block.get("source") - if ( - not isinstance(value, int | float) - or isinstance(value, bool) - or float(value) <= 0 - ): - raise SourceRuntimeError( - f"US immigration control {key!r} requires a positive numeric " - f"{value_key!r}." - ) - if not isinstance(source, str) or not source: - raise SourceRuntimeError( - f"US immigration control {key!r} requires a source citation." - ) - values[key] = float(value) + value, source = _target_and_source( + params.get(key), label=key, value_key=value_key, allow_zero=False + ) + values[key] = value sources[key] = source - return UndocumentedControls( + undocumented = UndocumentedControls( workers=values["undocumented_workers"], students=values["undocumented_students"], population_anchor=values[_ANCHOR_KEY], sources=sources, ) + return ImmigrationControls( + undocumented=undocumented, + humanitarian=_humanitarian_draws_from_parameters(params), + ) def _integer_column(person: pd.DataFrame, column: str) -> np.ndarray: @@ -500,13 +803,145 @@ def _select_weight_to_target( return selected -def _assign_ssn_card_codes( +def _arrival_profile( + person: pd.DataFrame, *, time_period: int +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """(PEINUSYR code, arrival-year midpoint, age at entry) per person.""" + + arrival_code = _integer_column(person, "PEINUSYR") + arrival_year = np.full(len(person), time_period, dtype=np.int64) + for code, midpoint in _ARRIVAL_YEAR_MIDPOINTS.items(): + arrival_year[arrival_code == code] = midpoint + years_in_us = time_period - arrival_year + age = _integer_column(person, "A_AGE") + age_at_entry = np.maximum(0, age - years_in_us) + return arrival_code, arrival_year, age_at_entry + + +def _daca_statutory_cohort( + arrival_year: np.ndarray, age_at_entry: np.ndarray, age: np.ndarray +) -> np.ndarray: + return ( + (arrival_year <= _DACA_LATEST_ARRIVAL_YEAR) + & (age_at_entry < _DACA_MAX_AGE_AT_ENTRY) + & (age >= _DACA_MIN_CURRENT_AGE) + ) + + +def _humanitarian_draw_candidates( + draw: HumanitarianDraw, + *, + ssn_codes: np.ndarray, + birth_country: np.ndarray, + arrival_code: np.ndarray, +) -> np.ndarray: + """Candidate mask for one draw, before exclusions shared by all draws. + + ``REFUGEE``/``ASYLEE``/``DEPORTATION_WITHHELD`` draw from the + indicator-documented pool only (code 3: those statuses carry immediate + federal benefits access, the same signals the ASEC-UA method reads); + parole and TPS may also draw from the still-unspilled residual pool + (code 0), whose members then receive EAD-backed SSN cards. + """ + + if draw.category == "paroled_one_year": + assert draw.origin is not None + return ( + np.isin(ssn_codes, (0, 3)) + & np.isin(birth_country, _PAROLE_ORIGIN_CODES[draw.origin]) + & np.isin(arrival_code, _RECENT_ARRIVAL_CODES) + ) + if draw.category == "refugee": + return ( + (ssn_codes == 3) + & np.isin(birth_country, _REFUGEE_ORIGIN_CODES) + & np.isin(arrival_code, _RECENT_ARRIVAL_CODES) + ) + if draw.category == "asylee": + return ( + (ssn_codes == 3) + & np.isin(birth_country, _ASYLEE_ORIGIN_CODES) + & np.isin(arrival_code, _ASYLEE_ARRIVAL_CODES) + ) + if draw.category == "deportation_withheld": + return ( + (ssn_codes == 3) + & (arrival_code >= 1) + & (arrival_code <= _WITHHELD_MAX_ARRIVAL_CODE) + ) + if draw.category == "tps": + assert draw.origin is not None + codes, max_arrival_code = _TPS_ORIGIN_CODES[draw.origin] + return ( + np.isin(ssn_codes, (0, 3)) + & np.isin(birth_country, codes) + & (arrival_code >= 1) + & (arrival_code <= max_arrival_code) + ) + raise SourceRuntimeError( + f"US immigration derivation has no candidate rule for draw " + f"{draw.label!r}." + ) + + +def _assign_humanitarian_statuses( person: pd.DataFrame, + ssn_codes: np.ndarray, weights: np.ndarray, *, seed: int, - controls: UndocumentedControls, + controls: ImmigrationControls, + time_period: int, ) -> np.ndarray: + """Mark humanitarian statuses and upgrade residual draws' SSN codes. + + Runs after the legal-status indicators and before the EAD worker/student + spill, so the Pew worker/student controls still bind the remaining + residual pool exactly. Draws are sequential over + ``controls.humanitarian``; a person marked by an earlier draw is out of + every later candidate pool. Residual-pool selections (parole/TPS only) + move to ``NON_CITIZEN_VALID_EAD`` — both programs confer employment + authorization — keeping the ``NONE`` ⇔ ``UNDOCUMENTED`` invariant. + """ + + marks = np.full(len(person), "", dtype="U24") + arrival_code, arrival_year, age_at_entry = _arrival_profile( + person, time_period=time_period + ) + age = _integer_column(person, "A_AGE") + birth_country = _integer_column(person, "PENATVTY") + # Cuba/Haiti-born keep the statutorily-favorable entrant class; the DACA + # cohort keeps its statutory tag (dual TPS/DACA holders stay DACA). + excluded = np.isin(birth_country, _CUBAN_HAITIAN_BIRTH_CODES) + excluded |= _daca_statutory_cohort(arrival_year, age_at_entry, age) + for draw in controls.humanitarian: + if draw.target <= 0: + continue + candidates = ( + _humanitarian_draw_candidates( + draw, + ssn_codes=ssn_codes, + birth_country=birth_country, + arrival_code=arrival_code, + ) + & ~excluded + & (marks == "") + ) + draws = _stable_person_draws(person, seed=seed, salt=draw.salt) + selected = _select_weight_to_target(candidates, weights, draws, draw.target) + marks[selected] = draw.status + ssn_codes[selected & (ssn_codes == 0)] = 2 + return marks + + +def _assign_ssn_card_codes( + person: pd.DataFrame, + weights: np.ndarray, + *, + seed: int, + controls: ImmigrationControls, + time_period: int, +) -> tuple[np.ndarray, np.ndarray]: citizenship = _integer_column(person, "PRCITSHP") unknown = ~np.isin(citizenship, [1, 2, 3, 4, 5]) if unknown.any(): @@ -566,6 +1001,18 @@ def _assign_ssn_card_codes( ) ssn_codes[(ssn_codes == 0) & assumed_documented] = 3 + # Humanitarian draws run between the indicators and the EAD spill: the + # spill controls then bind the *remaining* residual pool, so the final + # undocumented worker/student counts still land on the Pew anchors. + humanitarian_marks = _assign_humanitarian_statuses( + person, + ssn_codes, + weights, + seed=seed, + controls=controls, + time_period=time_period, + ) + is_worker = (_float_column(person, "WSAL_VAL") > 0) | ( _float_column(person, "SEMP_VAL") > 0 ) @@ -579,7 +1026,9 @@ def _assign_ssn_card_codes( # composition gate checks it against a cited anchor, and reconciling the # level is the calibration lane's job, not this label stage's. worker_candidates = (ssn_codes == 0) & noncitizens & is_worker - worker_excess = float(weights[worker_candidates].sum()) - controls.workers + worker_excess = ( + float(weights[worker_candidates].sum()) - controls.undocumented.workers + ) worker_draws = _stable_person_draws( person, seed=seed, salt="immigration:ead_workers" ) @@ -590,7 +1039,9 @@ def _assign_ssn_card_codes( ] = 2 student_candidates = (ssn_codes == 0) & noncitizens & is_student - student_excess = float(weights[student_candidates].sum()) - controls.students + student_excess = ( + float(weights[student_candidates].sum()) - controls.undocumented.students + ) student_draws = _stable_person_draws( person, seed=seed, salt="immigration:ead_students" ) @@ -599,22 +1050,18 @@ def _assign_ssn_card_codes( student_candidates, weights, student_draws, student_excess ) ] = 2 - return ssn_codes + return ssn_codes, humanitarian_marks def _derive_immigration_status( person: pd.DataFrame, ssn_codes: np.ndarray, + humanitarian_marks: np.ndarray, *, time_period: int, ) -> np.ndarray: - arrival_code = _integer_column(person, "PEINUSYR") - arrival_year = np.full(len(person), time_period, dtype=np.int64) - for code, midpoint in _ARRIVAL_YEAR_MIDPOINTS.items(): - arrival_year[arrival_code == code] = midpoint - years_in_us = time_period - arrival_year + _, arrival_year, age_at_entry = _arrival_profile(person, time_period=time_period) age = _integer_column(person, "A_AGE") - age_at_entry = np.maximum(0, age - years_in_us) birth_country = _integer_column(person, "PENATVTY") # Documented non-citizens default to LPR — the modal true status — and @@ -633,13 +1080,14 @@ def _derive_immigration_status( ) status[cuban_haitian] = "CUBAN_HAITIAN_ENTRANT" - daca = ( - (ssn_codes == 2) - & (arrival_year <= _DACA_LATEST_ARRIVAL_YEAR) - & (age_at_entry < _DACA_MAX_AGE_AT_ENTRY) - & (age >= _DACA_MIN_CURRENT_AGE) - ) + daca = (ssn_codes == 2) & _daca_statutory_cohort(arrival_year, age_at_entry, age) status[daca] = "DACA" + + # Humanitarian draws exclude Cuba/Haiti-born and the DACA cohort, so + # the marks are disjoint from both tags; every marked person carries a + # non-NONE SSN code, so the columns still agree about the undocumented. + marked = humanitarian_marks != "" + status[marked] = humanitarian_marks[marked] return status @@ -746,7 +1194,7 @@ def us_immigration_composition_summary(frame: Frame) -> dict[str, object]: def us_immigration_composition_gate( frame: Frame, *, - controls: UndocumentedControls | None = None, + controls: ImmigrationControls | None = None, ) -> GateResult: """Release gate: the SSN/immigration surface exists and is plausible. @@ -754,8 +1202,12 @@ def us_immigration_composition_gate( mode: everyone a citizen with a valid SSN), when a value falls outside the engine enum domain, when the two columns disagree about citizenship or undocumented status, when the weighted non-citizen share leaves its - plausibility band, or when the emergent undocumented population strays - outside a coarse band around its cited published anchor. + plausibility band, when the emergent undocumented population strays + outside a coarse band around its cited published anchor, or when a + humanitarian category's emitted mass leaves the coarse band around its + cited stock target (microcosm #767 — the H.R.1 §71109/§71301/§71302 and + SNAP §10108 eligibility channels all bind through these categories; an + explicit zero target must emit exactly zero). """ if controls is None: @@ -779,13 +1231,20 @@ def us_immigration_composition_gate( details: dict[str, object] = { "summary": us_immigration_composition_summary(frame), "controls": { - "undocumented_workers": controls.workers, - "undocumented_students": controls.students, - "undocumented_population_anchor": controls.population_anchor, - "sources": dict(controls.sources), + "undocumented_workers": controls.undocumented.workers, + "undocumented_students": controls.undocumented.students, + "undocumented_population_anchor": ( + controls.undocumented.population_anchor + ), + "sources": dict(controls.undocumented.sources), + "humanitarian_status_stocks": { + draw.label: {"target": draw.target, "source": draw.source} + for draw in controls.humanitarian + }, }, "non_citizen_share_band": list(_NON_CITIZEN_SHARE_BAND), "undocumented_anchor_relative_band": list(_UNDOCUMENTED_ANCHOR_RELATIVE_BAND), + "humanitarian_target_relative_band": list(_HUMANITARIAN_TARGET_RELATIVE_BAND), } missing = [ @@ -850,16 +1309,48 @@ def us_immigration_composition_gate( ) undocumented = float(weights[ssn_none].sum()) - relative = undocumented / controls.population_anchor + relative = undocumented / controls.undocumented.population_anchor rel_low, rel_high = _UNDOCUMENTED_ANCHOR_RELATIVE_BAND if not (rel_low <= relative <= rel_high): failures.append( f"emergent undocumented population {undocumented:,.0f} is " f"{relative:.2f}x the published anchor " - f"{controls.population_anchor:,.0f} (band [{rel_low}, {rel_high}], " - f"{controls.sources[_ANCHOR_KEY]})." + f"{controls.undocumented.population_anchor:,.0f} " + f"(band [{rel_low}, {rel_high}], " + f"{controls.undocumented.sources[_ANCHOR_KEY]})." ) + status_values = status.to_numpy() + hum_low, hum_high = _HUMANITARIAN_TARGET_RELATIVE_BAND + achieved_by_category: dict[str, dict[str, object]] = {} + for category in HUMANITARIAN_STATUS_CATEGORIES: + status_name = _HUMANITARIAN_STATUS_BY_CATEGORY[category] + target = controls.humanitarian_target(category) + emitted = float(weights[status_values == status_name].sum()) + achieved_by_category[category] = { + "status": status_name, + "target": target, + "population": emitted, + "relative": (emitted / target) if target > 0 else None, + } + if target <= 0: + if emitted > 0: + failures.append( + f"{status_name}: {emitted:,.0f} weighted persons emitted " + "against an explicit zero target — the manifest documents " + "this category as not imputed." + ) + continue + category_relative = emitted / target + if not (hum_low <= category_relative <= hum_high): + failures.append( + f"{status_name}: emitted population {emitted:,.0f} is " + f"{category_relative:.2f}x the cited stock target " + f"{target:,.0f} (band [{hum_low}, {hum_high}]) — the H.R.1 " + "eligibility channels through this category are degenerate." + ) + details["humanitarian_achieved"] = achieved_by_category + return GateResult( name="immigration_composition", passed=not failures, From 0e2cb72161276f724607242f69b29ca473805cee Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 25 Aug 2026 15:39:27 +0200 Subject: [PATCH 02/49] Declare the humanitarian draw sites and regenerate the US spec bundle Thirteen stable blake2b sites (per-origin parole, refugee, asylee, deportation-withheld, per-country TPS) join the seed protocol; the seed-owner ledger binds them to the immigration_status stage; the sources schema gains the required humanitarian_status_stocks block; the bundle regenerates with the new source_stages attestation. Refs #767 Co-Authored-By: Claude Fable 5 --- .../spec_engine/schema/sources.schema.json | 261 ++++++++++++++++++ .../src/microcosm/build/spec_engine/seeds.py | 25 ++ .../src/microcosm/build/us/spec/sources.yaml | 89 +++++- .../src/microcosm/build/us/spec/spine.yaml | 52 ++++ tools/generate_us_bundle_from_constants.py | 2 +- .../identity_contracts.py | 24 +- 6 files changed, 437 insertions(+), 16 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json index c6f16826fd..ff05c8a214 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json @@ -1594,9 +1594,270 @@ "source", "target" ] + }, + "humanitarian_status_stocks": { + "type": "object", + "additionalProperties": false, + "properties": { + "asylee": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + }, + "deportation_withheld": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + }, + "paroled_one_year": { + "type": "object", + "additionalProperties": false, + "properties": { + "afghanistan": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + }, + "nicaragua": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + }, + "ukraine": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + }, + "venezuela": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + } + }, + "required": [ + "afghanistan", + "nicaragua", + "ukraine", + "venezuela" + ] + }, + "refugee": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + }, + "tps": { + "type": "object", + "additionalProperties": false, + "properties": { + "el_salvador": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + }, + "honduras": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + }, + "nepal": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + }, + "nicaragua": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + }, + "other_designated": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + }, + "venezuela": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "target": { + "type": "number" + } + }, + "required": [ + "source", + "target" + ] + } + }, + "required": [ + "el_salvador", + "honduras", + "nepal", + "nicaragua", + "other_designated", + "venezuela" + ] + } + }, + "required": [ + "asylee", + "deportation_withheld", + "paroled_one_year", + "refugee", + "tps" + ] } }, "required": [ + "humanitarian_status_stocks", "kind", "seed_from_build_config", "time_period_from_build_config", diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/seeds.py b/packages/microcosm-build/src/microcosm/build/spec_engine/seeds.py index 36e6c27e85..4021114957 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/seeds.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/seeds.py @@ -741,6 +741,31 @@ def _stable_site( ), candidate_universe="all_person_rows_before_abawd_eligibility_mask", ), + *( + _stable_site( + f"immigration_humanitarian_{label.replace(':', '_')}_assignment", + salt=f"immigration:{label}", + key_grammar=("source_year:source_person_id_if_present", "else_person_id"), + candidate_universe=( + "all_person_rows_then_category_origin_window_pool_mask" + ), + ) + for label in ( + "paroled_one_year:afghanistan", + "paroled_one_year:ukraine", + "paroled_one_year:nicaragua", + "paroled_one_year:venezuela", + "refugee", + "asylee", + "deportation_withheld", + "tps:venezuela", + "tps:el_salvador", + "tps:honduras", + "tps:nicaragua", + "tps:nepal", + "tps:other_designated", + ) + ), _stable_site( "immigration_ead_workers_assignment", salt="immigration:ead_workers", diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml index a160533f4a..ba0b87935e 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml @@ -99,7 +99,7 @@ sources: stage_asset: id: source_stages path: microcosm.build.us/source_stages.json - sha256: dc58a0d700f0add7b658cec774df6e9587303beb58a1f432a35a18dcd1ac4097 + sha256: 619a974abd80c34201a0ccf813968ebc09ed52e144ff1d39722a4d9be7bd7821 stage_manifest: version: 1 country: us @@ -1896,7 +1896,9 @@ stages: format: published_estimate vintage: latest_available locator: Pew Research Center unauthorized-immigrant population and worker estimates; Higher Ed Immigration - Portal undocumented-student estimates + Portal undocumented-student estimates; OHSS refugee and asylee annual flow reports; DHS OAW/U4U + parole reports and CBP CHNV releases; CRS RS20844 TPS designations table; EOIR adjudication statistics + via CRS IN12501 operations: - kind: read_table table: person @@ -1913,21 +1915,82 @@ stages: undocumented_population_anchor: value: 11000000 source: https://www.pewresearch.org/short-reads/2024/07/22/what-we-know-about-unauthorized-immigrants-living-in-the-us/ + humanitarian_status_stocks: + paroled_one_year: + afghanistan: + target: 73566 + source: https://www.dhs.gov/sites/default/files/2023-06/PLCY%20-%20AFG%202503%20Update%202_0.pdf + ukraine: + target: 158000 + source: https://www.dhs.gov/sites/default/files/2024-12/2024_1104_dmo_plcy_uniting_for_ukraine_process_overview_and_assessment.pdf + nicaragua: + target: 93070 + source: https://www.cbp.gov/newsroom/national-media-release/cbp-releases-december-2024-monthly-update + venezuela: + target: 117330 + source: https://www.cbp.gov/newsroom/national-media-release/cbp-releases-december-2024-monthly-update + refugee: + target: 160000 + source: https://ohss.dhs.gov/sites/default/files/2024-11/2024_1108_ohss_refugee_annual_flow_report_2023.pdf + asylee: + target: 160000 + source: https://ohss.dhs.gov/sites/default/files/2024-10/2024_1002_ohss_asylees_fy2023.pdf + deportation_withheld: + target: 0 + source: https://crsreports.congress.gov/product/pdf/IN/IN12501 + tps: + venezuela: + target: 605015 + source: https://www.everycrsreport.com/reports/RS20844.html + el_salvador: + target: 170125 + source: https://www.everycrsreport.com/reports/RS20844.html + honduras: + target: 51225 + source: https://www.everycrsreport.com/reports/RS20844.html + nicaragua: + target: 2910 + source: https://www.everycrsreport.com/reports/RS20844.html + nepal: + target: 7160 + source: https://www.everycrsreport.com/reports/RS20844.html + other_designated: + target: 21215 + source: https://www.everycrsreport.com/reports/RS20844.html outputs: - ssn_card_type - immigration_status_str notes: 'Citizenship is measured (PRCITSHP), not imputed. Non-citizens with any ASEC-UA legal-status - indicator (Van Hook et al., SSRN 4662801) become OTHER_NON_CITIZEN. The CPS carries no work-authorization - variable, so among residual non-citizens, workers and students spill to NON_CITIZEN_VALID_EAD in deterministic - seeded order until the remaining undocumented worker count matches Pew''s 8.3M and the undocumented - student count matches the Higher Ed Immigration Portal''s ~408k — the only forced margins. The total - undocumented population is emergent (≈13M on the 2024 ASEC, within the range of published 2023-24 - estimates) and is gated against Pew''s 11.0M 2022 anchor with a coarse plausibility band rather than - forced; reconciling the level is the calibration lane''s job. Immigration-status tags carry only statutory - tests the data supports: DACA (arrival cohort among EAD holders) and CUBAN_HAITIAN_ENTRANT (nativity - plus post-1980 arrival); other documented non-citizens stay LEGAL_PERMANENT_RESIDENT, the modal true - status — blanket REFUGEE/TPS labels are deliberately not fabricated because they would mislabel millions - and over-grant refugee-class benefit exemptions.' + indicator (Van Hook et al., SSRN 4662801) become OTHER_NON_CITIZEN. Humanitarian/temporary-protection + statuses (microcosm #767) are drawn between the indicators and the EAD spill, sequentially and mutually + exclusively in decreasing order of target precision — PAROLED_ONE_YEAR per program origin (OAW Afghans + 73,566 as of 2022-03-31; U4U Ukrainians over 158,000 as of 2023-09-30; CHNV Nicaraguans 93,070 and + Venezuelans 117,330 paroled through 2024-12), REFUGEE (FY2023+FY2024 admissions, 60,050+100,060 OHSS + — refugees must apply for LPR adjustment after one year, so the not-yet-adjusted stock is about the + trailing two years), ASYLEE (FY2022-FY2024 grants 35,720+54,350+~70,000 OHSS; 88% of 2015-22 affirmative + adult grantees had adjusted by end-2024, so about three trailing years), then TPS per country (CRS + RS20844 Table 1 as of 2025-03-31, with required-arrival cutoffs binding at PEINUSYR granularity: El + Salvador <=2001, Honduras/Nicaragua <=1999, Nepal <=2015). Candidates are restricted by program origin + countries (Census Appendix I PENATVTY codes) and arrival windows; Cuba/Haiti-born are excluded everywhere + (the CUBAN_HAITIAN_ENTRANT class is the better statutory label and keeps H.R.1 eligibility — Haiti + TPS 330,735 rides that label), the DACA statutory cohort is excluded (dual TPS/DACA holders stay DACA), + and Ukraine/Afghanistan TPS registrants (101,150/8,105) are carried by the parole draw since they + are overwhelmingly the same U4U/OAW parolees. Parole targets are gross admissions: cohort members + who later won asylum keep the parole label, which is channel-equivalent under H.R.1 (both classes + lose Medicaid/SNAP/PTC on identical dates). DEPORTATION_WITHHELD carries an explicit zero target: + EOIR grants about 2.2k withholdings a year (CRS IN12501: under 1% of 666,177 FY2024 decisions) and + publishes no stock, so the accumulated population is below the CPS''s resolution for a defensible + draw. CONDITIONAL_ENTRANT is deliberately not emitted (the INA 203(a)(7) class closed in 1980). REFUGEE/ASYLEE + draw from the indicator-documented pool only; parole/TPS may draw residual-pool people, who then receive + NON_CITIZEN_VALID_EAD SSN cards (both programs confer employment authorization), consistent with Pew''s + own accounting of TPS/parolees inside its unauthorized totals. The EAD worker/student spill then binds + the remaining residual pool to Pew''s 8.3M workers and the Higher Ed Immigration Portal''s ~408k students + — still the only forced margins. The total undocumented population is emergent and gated against Pew''s + 11.0M 2022 anchor with a coarse band; per-category humanitarian masses are gated against their cited + targets with a coarse band (the ASEC undercovers 2022-24 arrivals — the Ukraine parole pool saturates + below its admin count); reconciling levels is the calibration lane''s job. DACA (arrival cohort among + EAD holders) and CUBAN_HAITIAN_ENTRANT (nativity plus post-1980 arrival) keep their statutory tests; + other documented non-citizens stay LEGAL_PERMANENT_RESIDENT, the modal true status.' - stage: hours_worked survey: Census CPS ASEC source: https://www.census.gov/programs-surveys/cps.html diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/spine.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/spine.yaml index e932429a61..42e100f8c2 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/spine.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/spine.yaml @@ -330,6 +330,58 @@ seed_site_bindings: owners: - kind: source_stage id: snap_abawd_discretionary_exemption +- site: immigration_humanitarian_paroled_one_year_afghanistan_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_paroled_one_year_ukraine_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_paroled_one_year_nicaragua_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_paroled_one_year_venezuela_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_refugee_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_asylee_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_deportation_withheld_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_tps_venezuela_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_tps_el_salvador_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_tps_honduras_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_tps_nicaragua_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_tps_nepal_assignment + owners: + - kind: source_stage + id: immigration_status +- site: immigration_humanitarian_tps_other_designated_assignment + owners: + - kind: source_stage + id: immigration_status - site: immigration_ead_workers_assignment owners: - kind: source_stage diff --git a/tools/generate_us_bundle_from_constants.py b/tools/generate_us_bundle_from_constants.py index 465bd4a4f6..9909da4782 100644 --- a/tools/generate_us_bundle_from_constants.py +++ b/tools/generate_us_bundle_from_constants.py @@ -122,7 +122,7 @@ def _domain_builders() -> dict[str, Callable[[], object]]: # frozen files, is the forward YAML -> legacy-payload path. FROZEN_LEGACY_RESOURCE_SHA256 = { "source_stages.json": ( - "dc58a0d700f0add7b658cec774df6e9587303beb58a1f432a35a18dcd1ac4097" + "619a974abd80c34201a0ccf813968ebc09ed52e144ff1d39722a4d9be7bd7821" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" diff --git a/tools/us_bundle_generation/identity_contracts.py b/tools/us_bundle_generation/identity_contracts.py index 06b2bf04d8..9475b1eee7 100644 --- a/tools/us_bundle_generation/identity_contracts.py +++ b/tools/us_bundle_generation/identity_contracts.py @@ -173,6 +173,26 @@ def build_seed_site_bindings( "snap_discretionary_exemption_assignment": source( "snap_abawd_discretionary_exemption" ), + **{ + f"immigration_humanitarian_{label}_assignment": source( + "immigration_status" + ) + for label in ( + "paroled_one_year_afghanistan", + "paroled_one_year_ukraine", + "paroled_one_year_nicaragua", + "paroled_one_year_venezuela", + "refugee", + "asylee", + "deportation_withheld", + "tps_venezuela", + "tps_el_salvador", + "tps_honduras", + "tps_nicaragua", + "tps_nepal", + "tps_other_designated", + ) + }, "immigration_ead_workers_assignment": source("immigration_status"), "immigration_ead_students_assignment": source("immigration_status"), "ssi_take_up_assignment": source("ssi_take_up"), @@ -203,9 +223,9 @@ def build_seed_site_bindings( } protocol_site_ids = tuple(site.id for site in LEGACY_V1_PROTOCOL.sites) - if len(protocol_site_ids) != 53 or set(owners) != set(protocol_site_ids): + if len(protocol_site_ids) != 66 or set(owners) != set(protocol_site_ids): raise RuntimeError( - "legacy-v1 seed owner ledger must cover exactly 53 protocol sites; " + "legacy-v1 seed owner ledger must cover exactly 66 protocol sites; " f"missing={sorted(set(protocol_site_ids) - owners.keys())!r}, " f"extra={sorted(owners.keys() - set(protocol_site_ids))!r}" ) From 28a733a025314581df05c4cc36dc6a204d16ec99 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 25 Aug 2026 15:44:31 +0200 Subject: [PATCH 03/49] Cover the humanitarian draws, manifest validation, and gate bands with tests Refs #767 Co-Authored-By: Claude Fable 5 --- .../microcosm/build/us_runtime/immigration.py | 14 +- .../tests/test_us_immigration.py | 462 +++++++++++++++++- 2 files changed, 452 insertions(+), 24 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py b/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py index 48dc63557a..151ad3ebc7 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py @@ -451,7 +451,9 @@ def __post_init__(self) -> None: @property def label(self) -> str: - return self.category if self.origin is None else f"{self.category}:{self.origin}" + return ( + self.category if self.origin is None else f"{self.category}:{self.origin}" + ) @property def salt(self) -> str: @@ -629,8 +631,7 @@ def _humanitarian_draws_from_parameters( missing = sorted(set(HUMANITARIAN_STATUS_CATEGORIES) - set(block)) if missing: raise SourceRuntimeError( - f"US immigration {_HUMANITARIAN_KEY} is missing category(ies): " - f"{missing}." + f"US immigration {_HUMANITARIAN_KEY} is missing category(ies): {missing}." ) draws: list[HumanitarianDraw] = [] for category in HUMANITARIAN_STATUS_CATEGORIES: @@ -879,8 +880,7 @@ def _humanitarian_draw_candidates( & (arrival_code <= max_arrival_code) ) raise SourceRuntimeError( - f"US immigration derivation has no candidate rule for draw " - f"{draw.label!r}." + f"US immigration derivation has no candidate rule for draw {draw.label!r}." ) @@ -1233,9 +1233,7 @@ def us_immigration_composition_gate( "controls": { "undocumented_workers": controls.undocumented.workers, "undocumented_students": controls.undocumented.students, - "undocumented_population_anchor": ( - controls.undocumented.population_anchor - ), + "undocumented_population_anchor": (controls.undocumented.population_anchor), "sources": dict(controls.undocumented.sources), "humanitarian_status_stocks": { draw.label: {"target": draw.target, "source": draw.source} diff --git a/packages/microcosm-build/tests/test_us_immigration.py b/packages/microcosm-build/tests/test_us_immigration.py index 23a1cb19fb..3040dec198 100644 --- a/packages/microcosm-build/tests/test_us_immigration.py +++ b/packages/microcosm-build/tests/test_us_immigration.py @@ -13,6 +13,7 @@ run_source_stage, ) from microcosm.build.us_runtime import ( + HUMANITARIAN_STATUS_CATEGORIES, IMMIGRATION_STATUS_VALUES, SSN_CARD_TYPE_VALUES, US_DONORS, @@ -20,6 +21,8 @@ US_IMMIGRATION_STAGE_NAME, US_SOURCE_MANIFEST, US_STAGE_NAMES, + HumanitarianDraw, + ImmigrationControls, UndocumentedControls, derive_us_immigration_status_from_manifest, us_immigration_composition_gate, @@ -33,12 +36,56 @@ TIME_PERIOD = 2024 +_PAROLE_ORIGIN_KEYS = ("afghanistan", "ukraine", "nicaragua", "venezuela") +_TPS_ORIGIN_KEYS = ( + "venezuela", + "el_salvador", + "honduras", + "nicaragua", + "nepal", + "other_designated", +) + + +def _humanitarian_block( + *, + paroled_one_year: dict[str, float] | None = None, + refugee: float = 0.0, + asylee: float = 0.0, + deportation_withheld: float = 0.0, + tps: dict[str, float] | None = None, +) -> dict: + """Manifest-shaped humanitarian block; unnamed targets default to zero.""" + + parole_targets = dict.fromkeys(_PAROLE_ORIGIN_KEYS, 0.0) | (paroled_one_year or {}) + tps_targets = dict.fromkeys(_TPS_ORIGIN_KEYS, 0.0) | (tps or {}) + return { + "paroled_one_year": { + origin: { + "target": target, + "source": f"https://example.com/parole/{origin}", + } + for origin, target in parole_targets.items() + }, + "refugee": {"target": refugee, "source": "https://example.com/refugee"}, + "asylee": {"target": asylee, "source": "https://example.com/asylee"}, + "deportation_withheld": { + "target": deportation_withheld, + "source": "https://example.com/withheld", + }, + "tps": { + origin: {"target": target, "source": f"https://example.com/tps/{origin}"} + for origin, target in tps_targets.items() + }, + } + def _stage_spec( *, workers: float, students: float, anchor: float, + humanitarian: dict | None = None, ) -> SourceStageSpec: return SourceStageSpec.from_mapping( { @@ -64,6 +111,11 @@ def _stage_spec( "value": anchor, "source": "https://example.com/population", }, + "humanitarian_status_stocks": ( + humanitarian + if humanitarian is not None + else _humanitarian_block() + ), }, ], "outputs": list(US_IMMIGRATION_OUTPUT_COLUMNS), @@ -126,10 +178,16 @@ def _run( workers: float = 100.0, students: float = 100.0, anchor: float = 10.0, + humanitarian: dict | None = None, seed: int = 0, ) -> pd.DataFrame: return run_source_stage( - _stage_spec(workers=workers, students=students, anchor=anchor), + _stage_spec( + workers=workers, + students=students, + anchor=anchor, + humanitarian=humanitarian, + ), tables={"person": person}, operation_handlers=_HANDLERS, config=SourceRuntimeConfig(seed=seed, target_year=TIME_PERIOD), @@ -308,6 +366,269 @@ def test_emitted_values_stay_inside_engine_enum_domains(self) -> None: assert set(output["immigration_status_str"]) <= set(IMMIGRATION_STATUS_VALUES) +class TestHumanitarianDraws: + def test_parole_draw_takes_documented_origin_cohort(self) -> None: + person = _person_table( + [ + # 2022-arrived Ukrainian reporting Medicaid: the U4U signature. + _noncitizen(PENATVTY=164, PEINUSYR=28, CAID=1), + # Same profile from a non-parole origin stays LPR. + _noncitizen(PENATVTY=303, PEINUSYR=28, CAID=1), + ] + ) + output = _run( + person, + humanitarian=_humanitarian_block(paroled_one_year={"ukraine": 1.0}), + ) + assert output.loc[0, "immigration_status_str"] == "PAROLED_ONE_YEAR" + assert output.loc[0, "ssn_card_type"] == "OTHER_NON_CITIZEN" + assert output.loc[1, "immigration_status_str"] == "LEGAL_PERMANENT_RESIDENT" + + def test_parole_origin_targets_do_not_cross(self) -> None: + person = _person_table( + [ + _noncitizen(PENATVTY=315, PEINUSYR=28, CAID=1), # Nicaraguan + ] + ) + output = _run( + person, + humanitarian=_humanitarian_block(paroled_one_year={"ukraine": 5.0}), + ) + assert output.loc[0, "immigration_status_str"] == "LEGAL_PERMANENT_RESIDENT" + + def test_parole_needs_recent_arrival(self) -> None: + # A 2014-2015-arrived Ukrainian predates every parole program. + person = _person_table([_noncitizen(PENATVTY=164, PEINUSYR=24, CAID=1)]) + output = _run( + person, + humanitarian=_humanitarian_block(paroled_one_year={"ukraine": 5.0}), + ) + assert output.loc[0, "immigration_status_str"] == "LEGAL_PERMANENT_RESIDENT" + + def test_refugee_draw_excludes_residual_pool(self) -> None: + # A 2022-arrived Congolese person with no legal-status indicator is + # not refugee-consistent (refugees carry immediate benefits access). + person = _person_table( + [ + _noncitizen(PENATVTY=412, PEINUSYR=28), + _noncitizen(PENATVTY=412, PEINUSYR=28, CAID=1), + ] + ) + output = _run(person, humanitarian=_humanitarian_block(refugee=1.0)) + assert output.loc[0, "immigration_status_str"] == "UNDOCUMENTED" + assert output.loc[1, "immigration_status_str"] == "REFUGEE" + + def test_asylee_draw_uses_backlog_window(self) -> None: + person = _person_table( + [ + # 2018-2019 arrival from China with Medicaid: grant-consistent. + _noncitizen(PENATVTY=207, PEINUSYR=26, CAID=1), + # 2004-2005 arrival predates the grant backlog window. + _noncitizen(PENATVTY=207, PEINUSYR=19, CAID=1, A_AGE=50), + ] + ) + output = _run(person, humanitarian=_humanitarian_block(asylee=1.0)) + assert output.loc[0, "immigration_status_str"] == "ASYLEE" + assert output.loc[1, "immigration_status_str"] == "LEGAL_PERMANENT_RESIDENT" + + def test_tps_residual_draw_flips_ssn_to_ead(self) -> None: + person = _person_table([_noncitizen(PENATVTY=373, PEINUSYR=28)]) + output = _run(person, humanitarian=_humanitarian_block(tps={"venezuela": 1.0})) + assert output.loc[0, "immigration_status_str"] == "TPS" + assert output.loc[0, "ssn_card_type"] == "NON_CITIZEN_VALID_EAD" + none_ssn = output["ssn_card_type"] == "NONE" + undocumented = output["immigration_status_str"] == "UNDOCUMENTED" + assert (none_ssn == undocumented).all() + + def test_tps_legacy_continuous_residence_window_binds(self) -> None: + person = _person_table( + [ + # Arrived 1996-1997: inside the El Salvador 2001 cutoff. + _noncitizen(PENATVTY=312, PEINUSYR=15, CAID=1, A_AGE=55), + # Arrived 2014-2015: cannot hold El Salvador TPS. + _noncitizen(PENATVTY=312, PEINUSYR=24, CAID=1, A_AGE=40), + ] + ) + output = _run( + person, humanitarian=_humanitarian_block(tps={"el_salvador": 5.0}) + ) + assert output.loc[0, "immigration_status_str"] == "TPS" + assert output.loc[1, "immigration_status_str"] == "LEGAL_PERMANENT_RESIDENT" + + def test_daca_cohort_is_excluded_from_humanitarian_draws(self) -> None: + # Arrived 2000-2001 aged ~7: the DACA statutory cohort. The worker + # spill gives the EAD card and the DACA tag survives TPS targeting. + person = _person_table( + [_noncitizen(PENATVTY=312, PEINUSYR=17, A_AGE=30, WSAL_VAL=20_000.0)] + ) + output = _run( + person, + workers=0.001, + humanitarian=_humanitarian_block(tps={"el_salvador": 5.0}), + ) + assert output.loc[0, "immigration_status_str"] == "DACA" + + def test_cuban_haitian_entrants_survive_humanitarian_targets(self) -> None: + person = _person_table([_noncitizen(PENATVTY=327, PEINUSYR=28, CAID=1)]) + output = _run( + person, + humanitarian=_humanitarian_block( + paroled_one_year={"venezuela": 5.0}, tps={"venezuela": 5.0} + ), + ) + assert output.loc[0, "immigration_status_str"] == "CUBAN_HAITIAN_ENTRANT" + + def test_draw_order_gives_parole_precedence_over_tps(self) -> None: + person = _person_table([_noncitizen(PENATVTY=373, PEINUSYR=28, CAID=1)]) + both = _run( + person, + humanitarian=_humanitarian_block( + paroled_one_year={"venezuela": 1.0}, tps={"venezuela": 1.0} + ), + ) + tps_only = _run( + person, humanitarian=_humanitarian_block(tps={"venezuela": 1.0}) + ) + assert both.loc[0, "immigration_status_str"] == "PAROLED_ONE_YEAR" + assert tps_only.loc[0, "immigration_status_str"] == "TPS" + + def test_draws_are_weight_targeted(self) -> None: + person = _person_table( + [ + _noncitizen(PENATVTY=164, PEINUSYR=28, CAID=1, person_weight=3.0), + _noncitizen(PENATVTY=164, PEINUSYR=28, CAID=1, person_weight=3.0), + ] + ) + output = _run( + person, + humanitarian=_humanitarian_block(paroled_one_year={"ukraine": 3.0}), + ) + statuses = sorted(output["immigration_status_str"]) + assert statuses == ["LEGAL_PERMANENT_RESIDENT", "PAROLED_ONE_YEAR"] + + def test_zero_targets_draw_nothing(self) -> None: + person = _person_table( + [ + _noncitizen(PENATVTY=164, PEINUSYR=28, CAID=1), + _noncitizen(PENATVTY=373, PEINUSYR=28), + _noncitizen(PENATVTY=412, PEINUSYR=28, CAID=1), + ] + ) + output = _run(person, humanitarian=_humanitarian_block()) + assert set(output["immigration_status_str"]) <= { + "LEGAL_PERMANENT_RESIDENT", + "UNDOCUMENTED", + } + + def test_humanitarian_spill_interaction_preserves_worker_control(self) -> None: + # TPS extraction removes a residual worker before the spill, so the + # remaining undocumented workers still land on the control exactly. + person = _person_table( + [_noncitizen(PENATVTY=373, PEINUSYR=28, WSAL_VAL=10_000.0)] + + [_noncitizen(WSAL_VAL=10_000.0) for _ in range(10)] + ) + output = _run( + person, + workers=4.0, + humanitarian=_humanitarian_block(tps={"venezuela": 1.0}), + ) + workers = output["WSAL_VAL"] > 0 + undocumented_workers = ((output["ssn_card_type"] == "NONE") & workers).sum() + assert output.loc[0, "immigration_status_str"] == "TPS" + assert undocumented_workers == 4 + + def test_missing_humanitarian_block_is_refused(self) -> None: + spec_mapping = { + "stage": US_IMMIGRATION_STAGE_NAME, + "survey": "test", + "source": "https://example.com", + "grain": "person", + "operations": [ + {"kind": "read_table", "table": "person"}, + { + "kind": "derive_immigration_status", + "seed_from_build_config": True, + "time_period_from_build_config": True, + "undocumented_workers": { + "target": 1, + "source": "https://example.com", + }, + "undocumented_students": { + "target": 1, + "source": "https://example.com", + }, + "undocumented_population_anchor": { + "value": 1, + "source": "https://example.com", + }, + }, + ], + "outputs": list(US_IMMIGRATION_OUTPUT_COLUMNS), + } + with pytest.raises(SourceRuntimeError, match="humanitarian_status_stocks"): + run_source_stage( + SourceStageSpec.from_mapping(spec_mapping), + tables={"person": _person_table([_noncitizen()])}, + operation_handlers=_HANDLERS, + config=SourceRuntimeConfig(seed=0, target_year=TIME_PERIOD), + ) + + @pytest.mark.parametrize( + "mutate, match", + [ + (lambda block: block.pop("refugee"), "missing category"), + ( + lambda block: block.__setitem__( + "mystery", {"target": 1, "source": "https://example.com"} + ), + "unsupported category", + ), + ( + lambda block: block["paroled_one_year"].pop("ukraine"), + "missing origin", + ), + ( + lambda block: block["tps"].__setitem__( + "atlantis", {"target": 1, "source": "https://example.com"} + ), + "outside the stage's codebook table", + ), + ( + lambda block: block["refugee"].pop("source"), + "source citation", + ), + ( + lambda block: block["refugee"].__setitem__("target", -1), + "non-negative", + ), + ], + ) + def test_malformed_humanitarian_blocks_are_refused(self, mutate, match) -> None: + block = _humanitarian_block(refugee=1.0) + mutate(block) + with pytest.raises(SourceRuntimeError, match=match): + _run(_person_table([_noncitizen()]), humanitarian=block) + + def test_humanitarian_clones_stay_consistent(self) -> None: + rows = [ + _noncitizen( + PENATVTY=164, + PEINUSYR=28, + CAID=1, + person_id=index + 1, + source_year=2024, + source_person_id=f"P{index % 5}", + ) + for index in range(10) + ] + output = _run( + _person_table(rows), + humanitarian=_humanitarian_block(paroled_one_year={"ukraine": 5.0}), + ) + by_source = output.groupby("source_person_id")["immigration_status_str"] + assert (by_source.nunique() == 1).all() + + class TestDeterminism: def _worker_pool(self) -> pd.DataFrame: return _person_table([_noncitizen(WSAL_VAL=10_000.0) for _ in range(20)]) @@ -363,6 +684,38 @@ def test_manifest_controls_carry_citations(self) -> None: assert float(block[value_key]) > 0 assert str(block["source"]).startswith("https://") + def test_manifest_humanitarian_stocks_carry_citations(self) -> None: + humanitarian = ( + us_immigration_stage_spec() + .operations[1] + .parameters["humanitarian_status_stocks"] + ) + assert set(humanitarian) == set(HUMANITARIAN_STATUS_CATEGORIES) + flat_blocks = { + "refugee": humanitarian["refugee"], + "asylee": humanitarian["asylee"], + "deportation_withheld": humanitarian["deportation_withheld"], + **{ + f"paroled_one_year:{origin}": block + for origin, block in humanitarian["paroled_one_year"].items() + }, + **{f"tps:{origin}": block for origin, block in humanitarian["tps"].items()}, + } + assert set(humanitarian["paroled_one_year"]) == set(_PAROLE_ORIGIN_KEYS) + assert set(humanitarian["tps"]) == set(_TPS_ORIGIN_KEYS) + for label, block in flat_blocks.items(): + assert float(block["target"]) >= 0, label + assert str(block["source"]).startswith("https://"), label + # The withheld stock is an explicit cited zero (no published stock); + # every other category carries a positive stock. + assert float(humanitarian["deportation_withheld"]["target"]) == 0 + positive = [ + label + for label, block in flat_blocks.items() + if label != "deportation_withheld" + ] + assert all(float(flat_blocks[label]["target"]) > 0 for label in positive) + def test_unexpected_parameter_is_refused(self) -> None: spec = SourceStageSpec.from_mapping( { @@ -521,16 +874,41 @@ def test_missing_raw_columns_raise_loudly(self) -> None: with_us_immigration_inputs(stripped, seed=0, time_period=TIME_PERIOD) -def _plausible_controls() -> UndocumentedControls: - return UndocumentedControls( - workers=20.0, - students=5.0, - population_anchor=30.0, - sources={ - "undocumented_workers": "https://example.com/workers", - "undocumented_students": "https://example.com/students", - "undocumented_population_anchor": "https://example.com/population", - }, +def _humanitarian_draws(**targets: float) -> tuple[HumanitarianDraw, ...]: + """Zero-target draw per category, overridden by keyword (national sums).""" + + status_by_category = { + "paroled_one_year": "PAROLED_ONE_YEAR", + "refugee": "REFUGEE", + "asylee": "ASYLEE", + "deportation_withheld": "DEPORTATION_WITHHELD", + "tps": "TPS", + } + return tuple( + HumanitarianDraw( + category=category, + origin=None, + status=status_by_category[category], + target=float(targets.get(category, 0.0)), + source=f"https://example.com/{category}", + ) + for category in HUMANITARIAN_STATUS_CATEGORIES + ) + + +def _plausible_controls(**humanitarian_targets: float) -> ImmigrationControls: + return ImmigrationControls( + undocumented=UndocumentedControls( + workers=20.0, + students=5.0, + population_anchor=30.0, + sources={ + "undocumented_workers": "https://example.com/workers", + "undocumented_students": "https://example.com/students", + "undocumented_population_anchor": "https://example.com/population", + }, + ), + humanitarian=_humanitarian_draws(**humanitarian_targets), ) @@ -540,6 +918,7 @@ def _composition_frame( other: int = 30, ead: int = 10, none: int = 30, + humanitarian: dict[str, int] | None = None, ) -> Frame: rows: list[dict] = [] values: list[tuple[str, str]] = ( @@ -547,6 +926,11 @@ def _composition_frame( + [("OTHER_NON_CITIZEN", "LEGAL_PERMANENT_RESIDENT")] * other + [("NON_CITIZEN_VALID_EAD", "LEGAL_PERMANENT_RESIDENT")] * ead + [("NONE", "UNDOCUMENTED")] * none + + [ + ("OTHER_NON_CITIZEN", status) + for status, count in (humanitarian or {}).items() + for _ in range(count) + ] ) for ssn, status in values: rows.append( @@ -616,11 +1000,14 @@ def test_fails_when_undocumented_far_from_anchor(self) -> None: def test_fails_when_non_citizen_share_implausible(self) -> None: gate = us_immigration_composition_gate( _composition_frame(citizens=40, other=20, ead=20, none=20), - controls=UndocumentedControls( - workers=8.0, - students=1.0, - population_anchor=20.0, - sources=_plausible_controls().sources, + controls=ImmigrationControls( + undocumented=UndocumentedControls( + workers=8.0, + students=1.0, + population_anchor=20.0, + sources=_plausible_controls().undocumented.sources, + ), + humanitarian=_humanitarian_draws(), ), ) assert not gate.passed @@ -632,6 +1019,49 @@ def test_gate_reads_packaged_controls_by_default(self) -> None: controls = gate.details["controls"] assert controls["undocumented_workers"] == 8_300_000 assert controls["undocumented_population_anchor"] == 11_000_000 + stocks = controls["humanitarian_status_stocks"] + assert stocks["paroled_one_year:afghanistan"]["target"] == 73_566 + assert stocks["tps:venezuela"]["target"] == 605_015 + assert stocks["deportation_withheld"]["target"] == 0 + + def test_gate_passes_with_in_band_humanitarian_masses(self) -> None: + gate = us_immigration_composition_gate( + _composition_frame(humanitarian={"REFUGEE": 4, "TPS": 6}), + controls=_plausible_controls(refugee=4.0, tps=6.0), + ) + assert gate.passed, gate.failures + achieved = gate.details["humanitarian_achieved"] + assert achieved["refugee"]["population"] == 4.0 + assert achieved["refugee"]["relative"] == 1.0 + assert achieved["tps"]["target"] == 6.0 + + def test_gate_fails_when_humanitarian_category_collapses(self) -> None: + gate = us_immigration_composition_gate( + _composition_frame(), + controls=_plausible_controls(refugee=4.0), + ) + assert not gate.passed + assert any( + "REFUGEE" in failure and "degenerate" in failure + for failure in gate.failures + ) + + def test_gate_fails_when_explicit_zero_category_emits(self) -> None: + gate = us_immigration_composition_gate( + _composition_frame(humanitarian={"DEPORTATION_WITHHELD": 2}), + controls=_plausible_controls(), + ) + assert not gate.passed + assert any("explicit zero target" in failure for failure in gate.failures) + + def test_gate_accepts_saturated_draws_inside_band(self) -> None: + # The ASEC undercovers 2022-24 arrivals; a draw that saturates at + # roughly three-quarters of its admin target stays inside the band. + gate = us_immigration_composition_gate( + _composition_frame(humanitarian={"PAROLED_ONE_YEAR": 3}), + controls=_plausible_controls(paroled_one_year=4.0), + ) + assert gate.passed, gate.failures def test_summary_reports_weighted_composition(self) -> None: summary = us_immigration_composition_summary( From 889b13fda622dcb60c9b8dfd8106831ffaab5b86 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 25 Aug 2026 15:48:02 +0200 Subject: [PATCH 04/49] Pin four H.R.1 immigration reform-coverage probes on the release Restorations of the pre-H.R.1 Medicaid, SNAP, and ACA lawful-presence lists and the below-FPL PTC exception must each move their budget measure on the export; a ~$0 score means the humanitarian statuses regressed and blocks publication. Refs #767 Co-Authored-By: Claude Fable 5 --- ...nitarian-immigration-statuses-767.added.md | 1 + .../us/release_input_coverage_manifest.json | 97 +++++++++++++ ...uild_us_release_input_coverage_manifest.py | 131 ++++++++++++++++++ 3 files changed, 229 insertions(+) create mode 100644 changelog.d/us-humanitarian-immigration-statuses-767.added.md diff --git a/changelog.d/us-humanitarian-immigration-statuses-767.added.md b/changelog.d/us-humanitarian-immigration-statuses-767.added.md new file mode 100644 index 0000000000..86176b426a --- /dev/null +++ b/changelog.d/us-humanitarian-immigration-statuses-767.added.md @@ -0,0 +1 @@ +Impute humanitarian and temporary-protection immigration statuses (REFUGEE, ASYLEE, PAROLED_ONE_YEAR per program origin, TPS per country) in the US immigration stage, drawn to cited DHS/OHSS/CBP/CRS stocks, so the enacted H.R.1 §71109/§71301/§71302 and SNAP §10108 eligibility channels stop computing $0; gate each category against its target and add four release-blocking reform-coverage probes. diff --git a/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json index b352ee60e3..d93bdf47d5 100644 --- a/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json @@ -1414,6 +1414,103 @@ "parameter_changes": {}, "period": 2024, "reason": "PolicyEngine-US multiplies HUD HAP by the restored SPM-unit take-up leaf after eligibility. Microcosm keeps that leaf exactly equal to source-backed housing-assistance receipt, so neutralizing it must remove the assistance paid to measured/imputed recipients. A 6,000-household production-ingredient smoke scored $202.795 million baseline-minus-neutralized; the $100 million floor is below that observed subset effect but far above numerical noise. A default-only or absent carry makes the source reconciliation or this uniquely isolating probe fail." + }, + { + "binding_inputs": [ + "immigration_status_str" + ], + "budget_measure": "medicaid", + "effect_direction": "reform_minus_baseline", + "expected_sign": "positive", + "id": "hr1_medicaid_humanitarian_eligibility_restoration", + "issue": "PolicyEngine/microcosm#767", + "min_abs_effect": 50000000.0, + "name": "H.R.1 SS71109 Medicaid humanitarian-status restoration", + "parameter_changes": { + "gov.hhs.medicaid.eligibility.eligible_immigration_statuses": { + "2026-10-01.2100-12-31": [ + "CITIZEN", + "LEGAL_PERMANENT_RESIDENT", + "REFUGEE", + "ASYLEE", + "DEPORTATION_WITHHELD", + "CUBAN_HAITIAN_ENTRANT", + "CONDITIONAL_ENTRANT", + "PAROLED_ONE_YEAR" + ] + } + }, + "period": 2027, + "reason": "H.R.1 SS71109 narrows the Medicaid qualified-immigrant list to citizens, LPRs, and Cuban/Haitian entrants from 2026-10-01; the probe restores the pre-narrowing list at 2027 law, which binds only through the REFUGEE/ASYLEE/DEPORTATION_WITHHELD/PAROLED_ONE_YEAR values of immigration_status_str. The source stage draws roughly 160k refugees (74% reporting Medicaid on the 2024 ASEC), 160k asylees, and 405k parolees to cited DHS/OHSS stocks, so restoring their eligibility must re-enroll anchored takers and move person-level medicaid dollars by far more than the floor. A ~$0 score means the humanitarian statuses regressed to zero records (microcosm #767's silent-zero failure) or the engine channel broke." + }, + { + "binding_inputs": [ + "immigration_status_str" + ], + "budget_measure": "aca_ptc", + "effect_direction": "reform_minus_baseline", + "expected_sign": "positive", + "id": "hr1_aca_below_fpl_exception_restoration", + "issue": "PolicyEngine/microcosm#767", + "min_abs_effect": 5000000.0, + "name": "H.R.1 SS71302 ACA below-FPL immigrant exception restoration", + "parameter_changes": { + "gov.aca.below_fpl_immigration_exception_in_effect": { + "2026-01-01.2026-12-31": true + } + }, + "period": 2026, + "reason": "H.R.1 SS71302 repeals the 26 USC 36B(c)(1)(B) below-FPL lawfully-present exception from 2026. Restoring it for 2026 binds, at 2026 law, only through tax units containing a non-citizen who is ACA-lawfully-present yet Medicaid-ineligible by status - on this file exactly the TPS population the source stage draws to the CRS RS20844 per-country stocks (865k weighted persons; the other humanitarian classes remain Medicaid-qualified at 2026 annual parameters). Below-poverty TPS units with marketplace take-up must regain premium tax credits above the floor; ~$0 means the TPS records vanished or the exception channel broke." + }, + { + "binding_inputs": [ + "immigration_status_str" + ], + "budget_measure": "aca_ptc", + "effect_direction": "reform_minus_baseline", + "expected_sign": "positive", + "id": "hr1_aca_lawful_presence_restoration", + "issue": "PolicyEngine/microcosm#767", + "min_abs_effect": 10000000.0, + "name": "H.R.1 SS71301 ACA lawful-presence list restoration", + "parameter_changes": { + "gov.aca.ineligible_immigration_statuses": { + "2027-01-01.2100-12-31": [ + "DACA", + "UNDOCUMENTED" + ] + } + }, + "period": 2027, + "reason": "H.R.1 SS71301 adds TPS/REFUGEE/ASYLEE/DEPORTATION_WITHHELD/PAROLED_ONE_YEAR to the ACA ineligible-status list from 2027-01-01. The probe restores the pre-H.R.1 list (DACA and UNDOCUMENTED only) at 2027, re-qualifying every humanitarian status the source stage draws (~1.6M weighted persons across parole, refugee, asylee, and TPS). Units among them with marketplace take-up must regain premium tax credits above the floor; ~$0 means the humanitarian statuses regressed or the lawful-presence channel broke." + }, + { + "binding_inputs": [ + "immigration_status_str" + ], + "budget_measure": "snap", + "effect_direction": "reform_minus_baseline", + "expected_sign": "positive", + "id": "hr1_snap_humanitarian_eligibility_restoration", + "issue": "PolicyEngine/microcosm#767", + "min_abs_effect": 25000000.0, + "name": "H.R.1 SS10108 SNAP humanitarian-status restoration", + "parameter_changes": { + "gov.usda.snap.eligibility.eligible_immigration_statuses": { + "2025-07-01.2100-12-31": [ + "CITIZEN", + "LEGAL_PERMANENT_RESIDENT", + "REFUGEE", + "ASYLEE", + "DEPORTATION_WITHHELD", + "CUBAN_HAITIAN_ENTRANT", + "CONDITIONAL_ENTRANT", + "PAROLED_ONE_YEAR" + ] + } + }, + "period": 2026, + "reason": "H.R.1 SS10108 narrows SNAP alien eligibility to citizens, LPRs, Cuban/Haitian entrants, and COFA citizens from 2025-07-01 (modeled 2025-07). Restoring the pre-H.R.1 list at 2026 re-includes the REFUGEE/ASYLEE/DEPORTATION_WITHHELD/PAROLED_ONE_YEAR members the source stage draws to cited DHS/OHSS stocks; is_snap_excluded_member stops excluding them, so SNAP allotments for their households must rise above the floor. ~$0 means the humanitarian statuses regressed to zero records or the SNAP immigration channel broke." } ], "schema_version": 1, diff --git a/tools/build_us_release_input_coverage_manifest.py b/tools/build_us_release_input_coverage_manifest.py index fbb5b03c0c..10043a98ff 100644 --- a/tools/build_us_release_input_coverage_manifest.py +++ b/tools/build_us_release_input_coverage_manifest.py @@ -1406,6 +1406,137 @@ ), "issue": "PolicyEngine/microcosm#312", }, + { + "id": "hr1_medicaid_humanitarian_eligibility_restoration", + "name": "H.R.1 SS71109 Medicaid humanitarian-status restoration", + "parameter_changes": { + "gov.hhs.medicaid.eligibility.eligible_immigration_statuses": { + "2026-10-01.2100-12-31": [ + "CITIZEN", + "LEGAL_PERMANENT_RESIDENT", + "REFUGEE", + "ASYLEE", + "DEPORTATION_WITHHELD", + "CUBAN_HAITIAN_ENTRANT", + "CONDITIONAL_ENTRANT", + "PAROLED_ONE_YEAR", + ] + } + }, + "budget_measure": "medicaid", + "period": 2027, + "effect_direction": "reform_minus_baseline", + "expected_sign": "positive", + "binding_inputs": ["immigration_status_str"], + "min_abs_effect": 50_000_000.0, + "reason": ( + "H.R.1 SS71109 narrows the Medicaid qualified-immigrant list to " + "citizens, LPRs, and Cuban/Haitian entrants from 2026-10-01; the " + "probe restores the pre-narrowing list at 2027 law, which binds " + "only through the REFUGEE/ASYLEE/DEPORTATION_WITHHELD/" + "PAROLED_ONE_YEAR values of immigration_status_str. The source " + "stage draws roughly 160k refugees (74% reporting Medicaid on the " + "2024 ASEC), 160k asylees, and 405k parolees to cited DHS/OHSS " + "stocks, so restoring their eligibility must re-enroll anchored " + "takers and move person-level medicaid dollars by far more than " + "the floor. A ~$0 score means the humanitarian statuses regressed " + "to zero records (microcosm #767's silent-zero failure) or the " + "engine channel broke." + ), + "issue": "PolicyEngine/microcosm#767", + }, + { + "id": "hr1_aca_below_fpl_exception_restoration", + "name": "H.R.1 SS71302 ACA below-FPL immigrant exception restoration", + "parameter_changes": { + "gov.aca.below_fpl_immigration_exception_in_effect": { + "2026-01-01.2026-12-31": True + } + }, + "budget_measure": "aca_ptc", + "period": 2026, + "effect_direction": "reform_minus_baseline", + "expected_sign": "positive", + "binding_inputs": ["immigration_status_str"], + "min_abs_effect": 5_000_000.0, + "reason": ( + "H.R.1 SS71302 repeals the 26 USC 36B(c)(1)(B) below-FPL " + "lawfully-present exception from 2026. Restoring it for 2026 " + "binds, at 2026 law, only through tax units containing a " + "non-citizen who is ACA-lawfully-present yet " + "Medicaid-ineligible by status - on this file exactly the TPS " + "population the source stage draws to the CRS RS20844 per-country " + "stocks (865k weighted persons; the other humanitarian classes " + "remain Medicaid-qualified at 2026 annual parameters). " + "Below-poverty TPS units with marketplace take-up must regain " + "premium tax credits above the floor; ~$0 means the TPS records " + "vanished or the exception channel broke." + ), + "issue": "PolicyEngine/microcosm#767", + }, + { + "id": "hr1_aca_lawful_presence_restoration", + "name": "H.R.1 SS71301 ACA lawful-presence list restoration", + "parameter_changes": { + "gov.aca.ineligible_immigration_statuses": { + "2027-01-01.2100-12-31": ["DACA", "UNDOCUMENTED"] + } + }, + "budget_measure": "aca_ptc", + "period": 2027, + "effect_direction": "reform_minus_baseline", + "expected_sign": "positive", + "binding_inputs": ["immigration_status_str"], + "min_abs_effect": 10_000_000.0, + "reason": ( + "H.R.1 SS71301 adds TPS/REFUGEE/ASYLEE/DEPORTATION_WITHHELD/" + "PAROLED_ONE_YEAR to the ACA ineligible-status list from " + "2027-01-01. The probe restores the pre-H.R.1 list (DACA and " + "UNDOCUMENTED only) at 2027, re-qualifying every humanitarian " + "status the source stage draws (~1.6M weighted persons across " + "parole, refugee, asylee, and TPS). Units among them with " + "marketplace take-up must regain premium tax credits above the " + "floor; ~$0 means the humanitarian statuses regressed or the " + "lawful-presence channel broke." + ), + "issue": "PolicyEngine/microcosm#767", + }, + { + "id": "hr1_snap_humanitarian_eligibility_restoration", + "name": "H.R.1 SS10108 SNAP humanitarian-status restoration", + "parameter_changes": { + "gov.usda.snap.eligibility.eligible_immigration_statuses": { + "2025-07-01.2100-12-31": [ + "CITIZEN", + "LEGAL_PERMANENT_RESIDENT", + "REFUGEE", + "ASYLEE", + "DEPORTATION_WITHHELD", + "CUBAN_HAITIAN_ENTRANT", + "CONDITIONAL_ENTRANT", + "PAROLED_ONE_YEAR", + ] + } + }, + "budget_measure": "snap", + "period": 2026, + "effect_direction": "reform_minus_baseline", + "expected_sign": "positive", + "binding_inputs": ["immigration_status_str"], + "min_abs_effect": 25_000_000.0, + "reason": ( + "H.R.1 SS10108 narrows SNAP alien eligibility to citizens, LPRs, " + "Cuban/Haitian entrants, and COFA citizens from 2025-07-01 " + "(modeled 2025-07). Restoring the pre-H.R.1 list at 2026 " + "re-includes the REFUGEE/ASYLEE/DEPORTATION_WITHHELD/" + "PAROLED_ONE_YEAR members the source stage draws to cited " + "DHS/OHSS stocks; is_snap_excluded_member stops excluding them, " + "so SNAP allotments for their households must rise above the " + "floor. ~$0 means the humanitarian statuses regressed to zero " + "records or the SNAP immigration channel broke." + ), + "issue": "PolicyEngine/microcosm#767", + }, ] From d5def92a1b9aba40b1586c7036b4b028de6a9269 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 25 Aug 2026 22:26:26 +0200 Subject: [PATCH 05/49] Prove the four H.R.1 probe channels on synthetic households and re-pin digests test_us_hr1_immigration_channels builds each probe's reform against the locked engine and shows nonzero, sign-correct movement per channel (medicaid, aca_ptc twice, snap) so the release smoke gate's probes have a PR-CI early warning. The thirteen humanitarian draw sites join EXPECTED_LEGACY_V1_SITES and the audited-source manifest; BE/UK envelope digests re-pin because the shared seed-protocol source is attested. Refs #767 Co-Authored-By: Claude Fable 5 --- .../tests/test_spec_engine_country_bundles.py | 4 +- .../tests/test_spec_engine_seeds.py | 26 +++ .../tests/test_us_hr1_immigration_channels.py | 172 ++++++++++++++++++ 3 files changed, 200 insertions(+), 2 deletions(-) create mode 100644 packages/microcosm-build/tests/test_us_hr1_immigration_channels.py diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index 5ede29d29b..2e045a4093 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -32,7 +32,7 @@ [ ( "be", - "86143c1c3f98980e34490c75706fe7dbf72e96e8accb8d7d4bbd2c1ae1a29b65", + "29e0820ad7779164794f81671a2e51c799ffad601410e7261a59e5d2e0fdcf25", { "household.household_id", "person.person_id", @@ -42,7 +42,7 @@ ), ( "uk", - "e3b171bdaae1bdfe842e61c129fd279a6b29001cdef4c18de51823768911ec8c", + "8a4da147cecd416f9a475b929b80a7010778df9902cd64dd149a5995bc59650d", { "benunit.benunit_id", "household.household_id", diff --git a/packages/microcosm-build/tests/test_spec_engine_seeds.py b/packages/microcosm-build/tests/test_spec_engine_seeds.py index ec1a50ab1c..fb0e9a3cb1 100644 --- a/packages/microcosm-build/tests/test_spec_engine_seeds.py +++ b/packages/microcosm-build/tests/test_spec_engine_seeds.py @@ -43,6 +43,19 @@ "housing_inputs_training_cap", "immigration_ead_students_assignment", "immigration_ead_workers_assignment", + "immigration_humanitarian_asylee_assignment", + "immigration_humanitarian_deportation_withheld_assignment", + "immigration_humanitarian_paroled_one_year_afghanistan_assignment", + "immigration_humanitarian_paroled_one_year_nicaragua_assignment", + "immigration_humanitarian_paroled_one_year_ukraine_assignment", + "immigration_humanitarian_paroled_one_year_venezuela_assignment", + "immigration_humanitarian_refugee_assignment", + "immigration_humanitarian_tps_el_salvador_assignment", + "immigration_humanitarian_tps_honduras_assignment", + "immigration_humanitarian_tps_nepal_assignment", + "immigration_humanitarian_tps_nicaragua_assignment", + "immigration_humanitarian_tps_other_designated_assignment", + "immigration_humanitarian_tps_venezuela_assignment", "legacy_congressional_district_assignment", "legacy_geography_ladder", "legacy_puma_ladder", @@ -140,6 +153,19 @@ ( "immigration_ead_workers_assignment", "immigration_ead_students_assignment", + "immigration_humanitarian_paroled_one_year_afghanistan_assignment", + "immigration_humanitarian_paroled_one_year_ukraine_assignment", + "immigration_humanitarian_paroled_one_year_nicaragua_assignment", + "immigration_humanitarian_paroled_one_year_venezuela_assignment", + "immigration_humanitarian_refugee_assignment", + "immigration_humanitarian_asylee_assignment", + "immigration_humanitarian_deportation_withheld_assignment", + "immigration_humanitarian_tps_venezuela_assignment", + "immigration_humanitarian_tps_el_salvador_assignment", + "immigration_humanitarian_tps_honduras_assignment", + "immigration_humanitarian_tps_nicaragua_assignment", + "immigration_humanitarian_tps_nepal_assignment", + "immigration_humanitarian_tps_other_designated_assignment", ), "packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py", ), diff --git a/packages/microcosm-build/tests/test_us_hr1_immigration_channels.py b/packages/microcosm-build/tests/test_us_hr1_immigration_channels.py new file mode 100644 index 0000000000..a62550aa9e --- /dev/null +++ b/packages/microcosm-build/tests/test_us_hr1_immigration_channels.py @@ -0,0 +1,172 @@ +"""The four H.R.1 immigration probes bind through the engine (microcosm #767). + +The release-blocking reform-coverage smoke gate scores the pinned probe +reforms on the full export; these tests prove the same channels move, +nonzero and sign-correct, on synthetic households whose +``immigration_status_str`` carries the humanitarian values the source stage +now imputes. A probe whose channel dies here would score $0 on every +release, so this is the PR-CI early warning for the engine side of the +silent-zero failure — without needing restricted microdata. + +Each situation pins the take-up flag its program reads (the same +data-seeded surface the release carries) and, for the ACA channels, an +``slcsp`` override so no rating-area geography is required. +""" + +from __future__ import annotations + +import pytest + +pytest.importorskip("policyengine_us") + +from policyengine_us import Simulation # noqa: E402 + +from microcosm.build.us_runtime.release_input_coverage import ( # noqa: E402 + us_release_reform_coverage_probes, +) +from microcosm.build.us_runtime.reform_coverage_smoke import ( # noqa: E402 + _build_reform, +) + +_HR1_PROBE_IDS = ( + "hr1_medicaid_humanitarian_eligibility_restoration", + "hr1_aca_below_fpl_exception_restoration", + "hr1_aca_lawful_presence_restoration", + "hr1_snap_humanitarian_eligibility_restoration", +) + + +@pytest.fixture(scope="module") +def hr1_probes(): + probes = { + probe.id: probe + for probe in us_release_reform_coverage_probes() + if probe.id.startswith("hr1_") + } + assert set(probes) == set(_HR1_PROBE_IDS) + return probes + + +def _household( + status: str, + income: float, + year: int, + *, + person: dict | None = None, + tax_unit: dict | None = None, + spm_unit: dict | None = None, +) -> dict: + people = { + "parent": { + "age": {year: 35}, + "employment_income": {year: income}, + "immigration_status_str": {year: status}, + **(person or {}), + }, + "child": { + "age": {year: 8}, + "immigration_status_str": {year: status}, + **(person or {}), + }, + } + return { + "people": people, + "families": {"f": {"members": ["parent", "child"]}}, + "marital_units": {"m": {"members": ["parent"]}}, + "tax_units": {"t": {"members": ["parent", "child"], **(tax_unit or {})}}, + "spm_units": {"s": {"members": ["parent", "child"], **(spm_unit or {})}}, + "households": { + "h": {"members": ["parent", "child"], "state_name": {year: "TX"}} + }, + } + + +def _effect(probe, situation: dict, year: int) -> tuple[float, float]: + baseline = Simulation(situation=situation) + reformed = Simulation(situation=situation, reform=_build_reform(probe)) + base = float(baseline.calculate(probe.budget_measure, year).sum()) + reform = float(reformed.calculate(probe.budget_measure, year).sum()) + return base, reform + + +def test_medicaid_restoration_reenrolls_refugee_family(hr1_probes) -> None: + # At 2027 law refugees are outside the §71109-narrowed qualified list; + # restoring the pre-H.R.1 list must re-enroll the anchored taker. + probe = hr1_probes["hr1_medicaid_humanitarian_eligibility_restoration"] + situation = _household( + "REFUGEE", + 25_000.0, + 2027, + person={"takes_up_medicaid_if_eligible": {2027: True}}, + ) + base, reform = _effect(probe, situation, 2027) + assert base == 0.0 + assert reform > 0.0 + + +def test_below_fpl_exception_restoration_pays_ptc_to_tps_unit(hr1_probes) -> None: + # A below-poverty TPS unit is lawfully present for the ACA at 2026 law + # yet Medicaid-ineligible by status — exactly the §71302 population. + probe = hr1_probes["hr1_aca_below_fpl_exception_restoration"] + situation = _household( + "TPS", + 12_000.0, + 2026, + tax_unit={ + "takes_up_aca_if_eligible": {2026: True}, + "slcsp": {2026: 8_000.0}, + }, + ) + base, reform = _effect(probe, situation, 2026) + assert base == 0.0 + assert reform > 0.0 + + +def test_lawful_presence_restoration_requalifies_asylee_unit(hr1_probes) -> None: + # At 2027 law §71301 adds asylees to the ACA ineligible list; restoring + # the pre-H.R.1 list (DACA/UNDOCUMENTED only) must re-qualify the unit. + probe = hr1_probes["hr1_aca_lawful_presence_restoration"] + situation = _household( + "ASYLEE", + 35_000.0, + 2027, + tax_unit={ + "takes_up_aca_if_eligible": {2027: True}, + "slcsp": {2027: 8_000.0}, + }, + ) + base, reform = _effect(probe, situation, 2027) + assert base == 0.0 + assert reform > 0.0 + + +def test_snap_restoration_reincludes_refugee_members(hr1_probes) -> None: + # At 2026 law §10108 makes refugees excluded SNAP members; restoring the + # pre-H.R.1 list must raise the household allotment. + probe = hr1_probes["hr1_snap_humanitarian_eligibility_restoration"] + situation = _household( + "REFUGEE", + 20_000.0, + 2026, + spm_unit={"takes_up_snap_if_eligible": {2026: True}}, + ) + base, reform = _effect(probe, situation, 2026) + assert reform > base + + +def test_probe_magnitude_floors_sit_far_below_plausible_scale(hr1_probes) -> None: + # The floors are release-scale guards: far above numerical noise, far + # below the plausible aggregate effect of the imputed stocks (about + # 1.6M weighted persons across the humanitarian categories). + floors = { + "hr1_medicaid_humanitarian_eligibility_restoration": 50_000_000.0, + "hr1_aca_below_fpl_exception_restoration": 5_000_000.0, + "hr1_aca_lawful_presence_restoration": 10_000_000.0, + "hr1_snap_humanitarian_eligibility_restoration": 25_000_000.0, + } + for probe_id, floor in floors.items(): + probe = hr1_probes[probe_id] + assert probe.min_abs_effect == floor + assert probe.expected_sign == "positive" + assert probe.binding_inputs == ("immigration_status_str",) + assert probe.issue == "PolicyEngine/microcosm#767" From 44d6ae486be60c2a967a7979b46f731ffdbd0de0 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 25 Aug 2026 22:30:48 +0200 Subject: [PATCH 06/49] Format the touched tooling and fix the stale site-count docstring Refs #767 Co-Authored-By: Claude Fable 5 --- .../tests/test_us_hr1_immigration_channels.py | 6 +++--- tools/us_bundle_generation/identity_contracts.py | 6 ++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/microcosm-build/tests/test_us_hr1_immigration_channels.py b/packages/microcosm-build/tests/test_us_hr1_immigration_channels.py index a62550aa9e..ef3c2dfab7 100644 --- a/packages/microcosm-build/tests/test_us_hr1_immigration_channels.py +++ b/packages/microcosm-build/tests/test_us_hr1_immigration_channels.py @@ -21,12 +21,12 @@ from policyengine_us import Simulation # noqa: E402 -from microcosm.build.us_runtime.release_input_coverage import ( # noqa: E402 - us_release_reform_coverage_probes, -) from microcosm.build.us_runtime.reform_coverage_smoke import ( # noqa: E402 _build_reform, ) +from microcosm.build.us_runtime.release_input_coverage import ( # noqa: E402 + us_release_reform_coverage_probes, +) _HR1_PROBE_IDS = ( "hr1_medicaid_humanitarian_eligibility_restoration", diff --git a/tools/us_bundle_generation/identity_contracts.py b/tools/us_bundle_generation/identity_contracts.py index 9475b1eee7..8173dabf36 100644 --- a/tools/us_bundle_generation/identity_contracts.py +++ b/tools/us_bundle_generation/identity_contracts.py @@ -102,7 +102,7 @@ def _stages_with_operation( def build_seed_site_bindings( source_document: Mapping[str, Any], ) -> list[dict[str, object]]: - """Bind all 53 legacy-v1 sites to their concrete execution owners.""" + """Bind all 66 legacy-v1 sites to their concrete execution owners.""" source_ids = _source_stage_ids(source_document) transfer_nodes = tuple(group.name for group in CANONICAL_US_LATE_TRANSFER_GROUPS) @@ -174,9 +174,7 @@ def build_seed_site_bindings( "snap_abawd_discretionary_exemption" ), **{ - f"immigration_humanitarian_{label}_assignment": source( - "immigration_status" - ) + f"immigration_humanitarian_{label}_assignment": source("immigration_status") for label in ( "paroled_one_year_afghanistan", "paroled_one_year_ukraine", From 4c6eb44da8a0359740b5d57c7d7732a0c79b7389 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 25 Aug 2026 22:33:55 +0200 Subject: [PATCH 07/49] Re-pin the country envelope digests on the merged tree Main's #743 moved the UK bundle while this branch moves the shared seed protocol; the merged UK digest combines both, BE carries only the protocol move. Refs #767 Co-Authored-By: Claude Fable 5 --- .../microcosm-build/tests/test_spec_engine_country_bundles.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index cc672a1599..de42902d2c 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -32,7 +32,7 @@ [ ( "be", - "86143c1c3f98980e34490c75706fe7dbf72e96e8accb8d7d4bbd2c1ae1a29b65", + "29e0820ad7779164794f81671a2e51c799ffad601410e7261a59e5d2e0fdcf25", { "household.household_id", "person.person_id", @@ -42,7 +42,7 @@ ), ( "uk", - "0c85845b4d463638ae3e5c5a25e17de8b720794e3653c5991dc4f069d95762d3", + "9b00257e0b36c3f34d81581b707c6dd042b786f29851a1bc2399dfc6fbb3b97f", { "benunit.benunit_id", "household.household_id", From cf066ab63a827022b199a38ce24b9d641158c848 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 26 Aug 2026 07:48:00 +0200 Subject: [PATCH 08/49] Re-pin the seed-site, field-count, and claim attestations for the new draws The thirteen humanitarian sites move every closed-world census the fail-closed machinery keeps: protocol site counts (53 -> 66), the owner-link sum (112 -> 125, one owner each, distinct owners unchanged), the configuration field universe (42,120 -> 42,419: +13 legacy-behavior target fields, +286 attestation-only resolved seed fields), four pointer claims (source_stages, spine/resolved seed-site bindings, resolved seed protocol), and the source_stages byte attestation in the bundle tests. Refs #767 Co-Authored-By: Claude Fable 5 --- .../build/spec_engine/field_usage.py | 22 +++++++++---------- .../tests/test_spec_engine_compiler_ir.py | 6 ++--- .../tests/test_spec_engine_field_usage.py | 16 +++++++------- .../test_spec_engine_identity_contracts.py | 2 +- .../tests/test_spec_engine_plan_lock.py | 2 +- .../tests/test_us_bundle_core_contracts.py | 2 +- .../tests/test_us_spec_bundle.py | 4 ++-- 7 files changed, 27 insertions(+), 27 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py index a5dd282ab8..89f230faf9 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py @@ -26,9 +26,9 @@ ) from .schemas import load_schema_registry -EXPECTED_AUTHORED_FIELD_COUNT = 32_351 -EXPECTED_RESOLVED_BINDING_FIELD_COUNT = 9_769 -EXPECTED_CONFIGURATION_FIELD_COUNT = 42_120 +EXPECTED_AUTHORED_FIELD_COUNT = 32_403 +EXPECTED_RESOLVED_BINDING_FIELD_COUNT = 10_016 +EXPECTED_CONFIGURATION_FIELD_COUNT = 42_419 class FieldUsageError(AssertionError): @@ -457,12 +457,12 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "6a781915fd491d2c4b16d2b7d482f69cf362c904130093c59f9629f7a319269b", ), "resolved_seed_protocol": ( - 823, - "c05ebb88538195798e9365f325381c14d92b74007f846f1fef50a08268e6d27d", + 1_031, + "1a8f3e74018287f184cf171cc4e6571490ba1de419cf0121a611fa93a9b7fa17", ), "resolved_seed_site_bindings": ( - 277, - "368fc19a8f07c4abf38ebcf2fbc4414d33c403c0f1f9881c2d2e3f5d6160feb6", + 316, + "199ec98045871070860164b522892327ab840b48c0c6a53972ebc0ff0d7d091f", ), "resolved_vintage_authorities": ( 63, @@ -489,8 +489,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "d6782c5de5bbed1bdc6bf653c4a6d4aadcad4ccc72d35e1092e130fcb04680a3", ), "source_stages": ( - 1_702, - "be3e24b53955b9dabccfc7e779212b1e97daf0c0992be3c183e391bf695ed4ee", + 1_715, + "78227f34c3f4ef946c06693dec9177dd906069f98016854bfd4d39c4cd4c51fa", ), "spine_assembly_household_mass_shares": ( 2, @@ -517,8 +517,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "29c6c1b3243e178783e7ab139993ba3a9b42d62edd4e9e4e1f3b28688daf2c6d", ), "spine_seed_site_bindings": ( - 277, - "dfa7ae701f62f7b1f06954798d29cfba3fc53a0be21e1751646d1f5bafd6071f", + 316, + "bb848f38e068858ac23e964a4ffbbf5542253da50515c72fe7753d8c56912486", ), "spine_support_roles": ( 29, diff --git a/packages/microcosm-build/tests/test_spec_engine_compiler_ir.py b/packages/microcosm-build/tests/test_spec_engine_compiler_ir.py index 393d98dd1d..c669fa342a 100644 --- a/packages/microcosm-build/tests/test_spec_engine_compiler_ir.py +++ b/packages/microcosm-build/tests/test_spec_engine_compiler_ir.py @@ -109,9 +109,9 @@ def test_us_seed_stream_map_is_complete_and_owner_typed( ) -> None: seed_map = compiled_us.seed_stream_map assert seed_map.protocol_id == "legacy-v1" - assert len(seed_map.sites) == 53 + assert len(seed_map.sites) == 66 assert len(seed_map.owners) == 54 - assert sum(len(site.owners) for site in seed_map.sites) == 112 + assert sum(len(site.owners) for site in seed_map.sites) == 125 assert {site.id for site in seed_map.sites} == { site.id for site in resolved_us.seed_protocol.sites } @@ -250,5 +250,5 @@ def test_bundle_without_imputation_compiles_empty_graph( assert compiled.producer_graph.nodes == () assert compiled.stage_dag.nodes == () assert compiled.nodes == () - assert len(compiled.seed_stream_map.sites) == 53 + assert len(compiled.seed_stream_map.sites) == 66 assert compiled.seed_stream_map.owners == () diff --git a/packages/microcosm-build/tests/test_spec_engine_field_usage.py b/packages/microcosm-build/tests/test_spec_engine_field_usage.py index 77fad49422..04e118087e 100644 --- a/packages/microcosm-build/tests/test_spec_engine_field_usage.py +++ b/packages/microcosm-build/tests/test_spec_engine_field_usage.py @@ -92,22 +92,22 @@ def _mutated_bundle( def test_exact_complete_ledger_has_one_primary_mode_per_pointer(field_ledger) -> None: - assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_120 + assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_419 assert field_ledger.source_counts == { - "authored": 32_351, - "resolved_bindings": 9_769, + "authored": 32_403, + "resolved_bindings": 10_016, } assert field_ledger.mode_counts == { - "legacy_behavior": 13_970, - "compiler_semantic": 27_699, + "legacy_behavior": 13_983, + "compiler_semantic": 27_985, "front_end_validation": 348, "identity_only": 103, } assert field_ledger.generation0_effect_counts == { - "legacy_behavior": 38_443, - "no_generation0_effect": 3_677, + "legacy_behavior": 38_456, + "no_generation0_effect": 3_963, } - assert len({field.pointer for field in field_ledger.fields}) == 42_120 + assert len({field.pointer for field in field_ledger.fields}) == 42_419 def test_eligibility_concepts_are_validation_not_generation0_behavior( diff --git a/packages/microcosm-build/tests/test_spec_engine_identity_contracts.py b/packages/microcosm-build/tests/test_spec_engine_identity_contracts.py index a2a682f68b..4849a2bf4a 100644 --- a/packages/microcosm-build/tests/test_spec_engine_identity_contracts.py +++ b/packages/microcosm-build/tests/test_spec_engine_identity_contracts.py @@ -124,7 +124,7 @@ def test_all_53_seed_sites_resolve_to_typed_real_owners( spine, source_stages = identity_documents bindings = _resolve(spine, source_stages) - assert len(bindings) == len(LEGACY_V1_PROTOCOL.sites) == 53 + assert len(bindings) == len(LEGACY_V1_PROTOCOL.sites) == 66 assert [binding.site for binding in bindings] == [ site.id for site in LEGACY_V1_PROTOCOL.sites ] diff --git a/packages/microcosm-build/tests/test_spec_engine_plan_lock.py b/packages/microcosm-build/tests/test_spec_engine_plan_lock.py index a0cd91c4d4..6764e24371 100644 --- a/packages/microcosm-build/tests/test_spec_engine_plan_lock.py +++ b/packages/microcosm-build/tests/test_spec_engine_plan_lock.py @@ -42,7 +42,7 @@ def test_plan_lock_is_complete_and_closed_world(compiled_us: CompiledSpecIR) -> assert payload["spec_binding"]["attestation"] == "mirror-attested" assert len(payload["producer_graph"]["nodes"]) == 38 assert len(payload["producer_graph"]["authored"]["ownership_matrix"]) == 18 - assert len(payload["seed_stream_map"]["sites"]) == 53 + assert len(payload["seed_stream_map"]["sites"]) == 66 assert len(payload["nodes"]) == 38 mutated = copy.deepcopy(payload) diff --git a/packages/microcosm-build/tests/test_us_bundle_core_contracts.py b/packages/microcosm-build/tests/test_us_bundle_core_contracts.py index 1ab0958836..74ad1b2859 100644 --- a/packages/microcosm-build/tests/test_us_bundle_core_contracts.py +++ b/packages/microcosm-build/tests/test_us_bundle_core_contracts.py @@ -129,7 +129,7 @@ def test_source_surface_classification_is_complete() -> None: } assert normative["stage_asset"] == { "id": "source_stages", - "sha256": "dc58a0d700f0add7b658cec774df6e9587303beb58a1f432a35a18dcd1ac4097", + "sha256": "619a974abd80c34201a0ccf813968ebc09ed52e144ff1d39722a4d9be7bd7821", } assert operational["stage_asset"] == { "path": "microcosm.build.us/source_stages.json" diff --git a/packages/microcosm-build/tests/test_us_spec_bundle.py b/packages/microcosm-build/tests/test_us_spec_bundle.py index 586907804b..7db19defbb 100644 --- a/packages/microcosm-build/tests/test_us_spec_bundle.py +++ b/packages/microcosm-build/tests/test_us_spec_bundle.py @@ -190,7 +190,7 @@ def _load_generator_module(): LEGACY_COMPATIBILITY_SHA256 = { "source_stages.json": ( - "dc58a0d700f0add7b658cec774df6e9587303beb58a1f432a35a18dcd1ac4097" + "619a974abd80c34201a0ccf813968ebc09ed52e144ff1d39722a4d9be7bd7821" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" @@ -917,7 +917,7 @@ def test_legacy_seed_vintage_and_publication_grammars_are_pinned( "identity_generation": 1, "seed_protocol": LEGACY_V1_PROTOCOL.id, } - assert len(LEGACY_V1_PROTOCOL.sites) == 53 + assert len(LEGACY_V1_PROTOCOL.sites) == 66 assert len(LEGACY_V1_PROTOCOL.streams) == 14 assert LEGACY_V1_PROTOCOL.site("survey_sample_asec").default == 578 assert LEGACY_V1_PROTOCOL.site("puf_live_aggregate_disaggregation").default == 0 From f6fcf31fc0952439368c3252673f99a46640c0c6 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 28 Aug 2026 12:02:37 +0200 Subject: [PATCH 09/49] Re-pin spec-engine coverage for humanitarian draws --- docs/evidence/spec-engine/us-f0-coverage.json | 99 +++++++++++-------- .../build/spec_engine/inventory_coverage.py | 39 +++++--- .../tests/test_spec_engine_country_bundles.py | 4 +- .../tests/test_spec_engine_coverage_tool.py | 18 ++-- .../test_spec_engine_inventory_coverage.py | 4 +- .../tests/test_spec_engine_loader.py | 2 +- .../tests/test_us_multispine_pool_tool.py | 4 +- tools/spec_engine_coverage.py | 2 +- 8 files changed, 99 insertions(+), 73 deletions(-) diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index 68f70290fb..7635980533 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -4,9 +4,9 @@ "version": 1 }, "country": "us", - "documentation_sha256": "4b39450dbdb8dafb83c3b627123b8026c6f82c660b66fe76f341a67c4f37c77b", + "documentation_sha256": "b595dbfb3cda4ee18f9267f48950e1636f3b98656f2274aa9d45e117494dad26", "field_usage": { - "authored_normative_field_count": 32351, + "authored_normative_field_count": 32403, "claim_count": 49, "claims": [ { @@ -514,8 +514,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 1702, - "pointer_sha256": "be3e24b53955b9dabccfc7e779212b1e97daf0c0992be3c183e391bf695ed4ee", + "pointer_count": 1715, + "pointer_sha256": "78227f34c3f4ef946c06693dec9177dd906069f98016854bfd4d39c4cd4c51fa", "rationale": null, "relative_sink_prefix": "/source_manifest/stages", "source_prefix": "/authored/spec~1sources.yaml/stages", @@ -544,8 +544,8 @@ "legacy_sinks": [], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 277, - "pointer_sha256": "dfa7ae701f62f7b1f06954798d29cfba3fc53a0be21e1751646d1f5bafd6071f", + "pointer_count": 316, + "pointer_sha256": "bb848f38e068858ac23e964a4ffbbf5542253da50515c72fe7753d8c56912486", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/spec~1spine.yaml/seed_site_bindings", @@ -736,8 +736,8 @@ "legacy_sinks": [], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 823, - "pointer_sha256": "c05ebb88538195798e9365f325381c14d92b74007f846f1fef50a08268e6d27d", + "pointer_count": 1031, + "pointer_sha256": "1a8f3e74018287f184cf171cc4e6571490ba1de419cf0121a611fa93a9b7fa17", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/resolved/seed_protocol", @@ -750,8 +750,8 @@ "legacy_sinks": [], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 277, - "pointer_sha256": "368fc19a8f07c4abf38ebcf2fbc4414d33c403c0f1f9881c2d2e3f5d6160feb6", + "pointer_count": 316, + "pointer_sha256": "199ec98045871070860164b522892327ab840b48c0c6a53972ebc0ff0d7d091f", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/resolved/seed_site_bindings", @@ -772,21 +772,21 @@ "verifier": "vintages" } ], - "configuration_field_count": 42120, - "consumed_field_count": 42120, + "configuration_field_count": 42419, + "consumed_field_count": 42419, "generation0_effect_counts": { - "legacy_behavior": 38443, - "no_generation0_effect": 3677 + "legacy_behavior": 38456, + "no_generation0_effect": 3963 }, "mode_counts": { - "compiler_semantic": 27699, + "compiler_semantic": 27985, "front_end_validation": 348, "identity_only": 103, - "legacy_behavior": 13970 + "legacy_behavior": 13983 }, "multiple_primary_use_field_count": 0, - "pointer_inventory_sha256": "bc4a948ab632191954600da8474c5b011f977a65e24c399d126f3dc4a79f23e5", - "resolved_binding_field_count": 9769, + "pointer_inventory_sha256": "879bf3c70ad0d10c7a123a3b3194d3822f3015bed4bfba2b4a6703885a6bc95a", + "resolved_binding_field_count": 10016, "unused_field_count": 0 }, "inventory_coverage": { @@ -815,9 +815,9 @@ "producer_virtual_resources": 75, "release_rungs": 5, "resolved_references": 334, - "seed_owner_bindings": 112, + "seed_owner_bindings": 125, "seed_owner_rows": 54, - "seed_sites": 53, + "seed_sites": 66, "seed_streams": 14, "source_operators": 16, "source_stages": 37, @@ -1361,12 +1361,12 @@ ], "expected": { "producer_count": 38, - "sha256": "3850554eb804fde5e4f86a34ac1bb8a7a07aafff7e8b48396a3d5fca844798e8" + "sha256": "4a74a831c92fc6273a82a22d02c73cca5e228852f4b88e496f83febc861fa222" }, "failures": [], "observed": { "producer_count": 38, - "sha256": "3850554eb804fde5e4f86a34ac1bb8a7a07aafff7e8b48396a3d5fca844798e8" + "sha256": "4a74a831c92fc6273a82a22d02c73cca5e228852f4b88e496f83febc861fa222" }, "status": "covered" }, @@ -1533,7 +1533,7 @@ "expected": { "groups": 12, "partition": "disjoint and exhaustive", - "sites": 53 + "sites": 66 }, "failures": [], "observed": { @@ -1550,6 +1550,19 @@ "pregnancy_assignment", "wic_claim_assignment", "snap_discretionary_exemption_assignment", + "immigration_humanitarian_paroled_one_year_afghanistan_assignment", + "immigration_humanitarian_paroled_one_year_ukraine_assignment", + "immigration_humanitarian_paroled_one_year_nicaragua_assignment", + "immigration_humanitarian_paroled_one_year_venezuela_assignment", + "immigration_humanitarian_refugee_assignment", + "immigration_humanitarian_asylee_assignment", + "immigration_humanitarian_deportation_withheld_assignment", + "immigration_humanitarian_tps_venezuela_assignment", + "immigration_humanitarian_tps_el_salvador_assignment", + "immigration_humanitarian_tps_honduras_assignment", + "immigration_humanitarian_tps_nicaragua_assignment", + "immigration_humanitarian_tps_nepal_assignment", + "immigration_humanitarian_tps_other_designated_assignment", "immigration_ead_workers_assignment", "immigration_ead_students_assignment", "ssi_take_up_assignment", @@ -1616,7 +1629,7 @@ "sipp_tip_training_cap" ] }, - "sites": 53 + "sites": 66 }, "status": "covered" }, @@ -1656,13 +1669,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "36a9d819ef196c312888591936d49c025b0407df9928440cceefabc5458f72af", - "protocol_sha256": "15840b380329410a7094f60b0f1dad453457fc785859f0c372f2c8e2d59b0246" + "map_sha256": "57b5dbd0fa563348885ec4b8d0fdfaace13c101c2fcd60d5620cf19592912f08", + "protocol_sha256": "baee9e47b31c369f1b3c137a97545c59bcb9f33fc96203894d289ff90c62270b" }, "failures": [], "observed": { - "map_sha256": "36a9d819ef196c312888591936d49c025b0407df9928440cceefabc5458f72af", - "protocol_sha256": "15840b380329410a7094f60b0f1dad453457fc785859f0c372f2c8e2d59b0246" + "map_sha256": "57b5dbd0fa563348885ec4b8d0fdfaace13c101c2fcd60d5620cf19592912f08", + "protocol_sha256": "baee9e47b31c369f1b3c137a97545c59bcb9f33fc96203894d289ff90c62270b" }, "status": "covered" }, @@ -1677,7 +1690,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "15840b380329410a7094f60b0f1dad453457fc785859f0c372f2c8e2d59b0246", + "implementation_sha256": "baee9e47b31c369f1b3c137a97545c59bcb9f33fc96203894d289ff90c62270b", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1698,7 +1711,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "15840b380329410a7094f60b0f1dad453457fc785859f0c372f2c8e2d59b0246", + "implementation_sha256": "baee9e47b31c369f1b3c137a97545c59bcb9f33fc96203894d289ff90c62270b", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1732,12 +1745,12 @@ ], "expected": { "relation": "all site fields preserved exactly", - "sites": 53 + "sites": 66 }, "failures": [], "observed": { - "sha256": "99710bf212c3801cfd0434d56d2d1d2e4902a2a958978cdbd91cd9acce81d2fb", - "sites": 53 + "sha256": "4cee90eb62099e64cec47151ac6f9df853c947574c0450c4f7200e9ff297437f", + "sites": 66 }, "status": "covered" }, @@ -1752,12 +1765,12 @@ "compiler_ir.seed_stream_map.sites.owners" ], "expected": { - "bindings": 112, - "coverage": "all 53 sites" + "bindings": 125, + "coverage": "all 66 sites" }, "failures": [], "observed": { - "bindings": 112 + "bindings": 125 }, "status": "covered" }, @@ -1775,12 +1788,12 @@ "legacy_adapter.imputation.late_producer_resource_semantics" ], "expected": { - "sha256": "cd5ba8924d64da5425ee14cca82a774e3f4b2bb5aabe06df291cc3cc457287a9", + "sha256": "5e435fa74ff0f72fa5c8b7cda1ea9ac9cfdf7e114f06df07eece57142374b1ac", "stages": 37 }, "failures": [], "observed": { - "sha256": "cd5ba8924d64da5425ee14cca82a774e3f4b2bb5aabe06df291cc3cc457287a9", + "sha256": "5e435fa74ff0f72fa5c8b7cda1ea9ac9cfdf7e114f06df07eece57142374b1ac", "stages": 37 }, "status": "covered" @@ -1887,7 +1900,7 @@ "alpha", "zeta" ], - "sha256": "a128a85f877fb32def9382b841b8b340f974e8a9148ac029c1f04becdc956c18" + "sha256": "af1b04f0fe03938e826daf666e7242e4b138ebd5d8149abdda447359169ef3d5" }, "failures": [], "observed": { @@ -1911,7 +1924,7 @@ "alpha", "zeta" ], - "sha256": "a128a85f877fb32def9382b841b8b340f974e8a9148ac029c1f04becdc956c18" + "sha256": "af1b04f0fe03938e826daf666e7242e4b138ebd5d8149abdda447359169ef3d5" }, "status": "covered" }, @@ -1975,7 +1988,7 @@ "take_up_contract", "us_qbi_reconciliation_contract" ], - "sha256": "91c65a9ff36839d575036264c4bf57ffde6457e8fd180a0856f9be712ada371d" + "sha256": "acbb196113c7ed0f7fbf2b0c95d5f57fc303d4fec3ef09475f3483745f34ba25" }, "status": "covered" }, @@ -2599,7 +2612,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "5378bb9189aec96f50da22aac71e5bd2c3d919e9795f6ef2147e0bc9c739dd8e" + "spec_sha256": "4c22013c378c881831b07735963b53c68a6f1bc97b17eb5a2e01d91101ee2cfd" } }, "report_schema_version": 3, @@ -2609,7 +2622,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "5378bb9189aec96f50da22aac71e5bd2c3d919e9795f6ef2147e0bc9c739dd8e" + "spec_sha256": "4c22013c378c881831b07735963b53c68a6f1bc97b17eb5a2e01d91101ee2cfd" }, "status": "pass" } diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index b48afbf581..16e337c94e 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -296,6 +296,19 @@ "pregnancy_assignment", "wic_claim_assignment", "snap_discretionary_exemption_assignment", + "immigration_humanitarian_paroled_one_year_afghanistan_assignment", + "immigration_humanitarian_paroled_one_year_ukraine_assignment", + "immigration_humanitarian_paroled_one_year_nicaragua_assignment", + "immigration_humanitarian_paroled_one_year_venezuela_assignment", + "immigration_humanitarian_refugee_assignment", + "immigration_humanitarian_asylee_assignment", + "immigration_humanitarian_deportation_withheld_assignment", + "immigration_humanitarian_tps_venezuela_assignment", + "immigration_humanitarian_tps_el_salvador_assignment", + "immigration_humanitarian_tps_honduras_assignment", + "immigration_humanitarian_tps_nicaragua_assignment", + "immigration_humanitarian_tps_nepal_assignment", + "immigration_humanitarian_tps_other_designated_assignment", "immigration_ead_workers_assignment", "immigration_ead_students_assignment", "ssi_take_up_assignment", @@ -350,18 +363,18 @@ "acs_person_predictors": "878c788a6f037d7aca12b3586ea034eff04f3034ffa11935a736493042551f25", "authority": "3a980927227704d0589f246eef9cd825c2ae84f3a4134ac835e0e5ed39a563ac", "early_families": "4aa9f736fd76e83955477ad1667e58f48f264783f05bdc7f0102cd32d61323bd", - "full_checkpoint": "a128a85f877fb32def9382b841b8b340f974e8a9148ac029c1f04becdc956c18", + "full_checkpoint": "af1b04f0fe03938e826daf666e7242e4b138ebd5d8149abdda447359169ef3d5", "gap_fill_schedule": "1c31f9868f7884347cc19cf1ff65da43f950b9114941a715bab168246db414a7", "graph_nodes": "7125ad28ae2c69f22094a574bbf6ed2ddf1682a2c2c3b416f8f49304b7016ce7", "geography_assignment": "f49425ca8734ac559c73cf44f6458d86d3162a48956b98a27e6e758959361585", "late_families": "d91f9ff0eb52f43e7b6eed3d5c58c37abe1620c3a11021da15dae9c10e16d382", - "late_resource_semantics": "3850554eb804fde5e4f86a34ac1bb8a7a07aafff7e8b48396a3d5fca844798e8", + "late_resource_semantics": "4a74a831c92fc6273a82a22d02c73cca5e228852f4b88e496f83febc861fa222", "late_schedule": "dcf3c6d2eade3449836c49a1dc4d3b8cd395aab9142db700c3c60598fa9c1c79", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "987b501c695e31f45521c4a178528f75ab3df22c09bc407b182213b2de99ee57", - "seed_map": "36a9d819ef196c312888591936d49c025b0407df9928440cceefabc5458f72af", - "seed_protocol": "15840b380329410a7094f60b0f1dad453457fc785859f0c372f2c8e2d59b0246", - "source_manifest": "cd5ba8924d64da5425ee14cca82a774e3f4b2bb5aabe06df291cc3cc457287a9", + "seed_map": "57b5dbd0fa563348885ec4b8d0fdfaace13c101c2fcd60d5620cf19592912f08", + "seed_protocol": "baee9e47b31c369f1b3c137a97545c59bcb9f33fc96203894d289ff90c62270b", + "source_manifest": "5e435fa74ff0f72fa5c8b7cda1ea9ac9cfdf7e114f06df07eece57142374b1ac", "take_up": "fa186daea0f8dd641cc470e41d1a2953f887d45282ec990201298f47bedf8d4d", "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", } @@ -434,9 +447,9 @@ "producer_virtual_resources": 75, "release_rungs": 5, "resolved_references": 334, - "seed_owner_bindings": 112, + "seed_owner_bindings": 125, "seed_owner_rows": 54, - "seed_sites": 53, + "seed_sites": 66, "seed_streams": 14, "source_operators": 16, "source_stages": 37, @@ -1773,7 +1786,7 @@ def add( "seed_site_definitions_exact", clauses={ "seed site definitions differ": site_definitions_exact, - "seed site count differs": len(protocol_sites) == 53, + "seed site count differs": len(protocol_sites) == 66, }, homes=("/bundle/seed_protocol",), consumers=("compiler_ir.seed_stream_map.sites", "compiler_ir.node_slices"), @@ -1781,7 +1794,7 @@ def add( "sites": len(protocol_sites), "sha256": sha256_json([site.to_wire() for site in protocol.sites]), }, - expected={"sites": 53, "relation": "all site fields preserved exactly"}, + expected={"sites": 66, "relation": "all site fields preserved exactly"}, ) binding_by_site = {binding.site: binding for binding in spec.seed_site_bindings} binding_exact = set(binding_by_site) == set(compiled_sites) and all( @@ -1796,7 +1809,7 @@ def add( "owner binding count differs": sum( len(site.owners) for site in compiled_sites.values() ) - == 112, + == 125, "one or more sites have no owner": all( site.owners for site in compiled_sites.values() ), @@ -1806,7 +1819,7 @@ def add( observed={ "bindings": sum(len(site.owners) for site in compiled_sites.values()) }, - expected={"bindings": 112, "coverage": "all 53 sites"}, + expected={"bindings": 125, "coverage": "all 66 sites"}, ) expected_owner_sites: dict[tuple[str, str], list[str]] = {} for site in protocol.sites: @@ -1850,7 +1863,7 @@ def add( "seed groups overlap": groups_disjoint, "seed groups do not cover the protocol exactly": set(grouped_ids) == set(protocol_sites), - "seed group cardinality differs": len(grouped_ids) == 53, + "seed group cardinality differs": len(grouped_ids) == 66, }, homes=("/bundle/seed_protocol", "/spine/seed_site_bindings"), consumers=("compiler_ir.seed_stream_map",), @@ -1860,7 +1873,7 @@ def add( }, expected={ "groups": len(EXPECTED_SEED_GROUPS), - "sites": 53, + "sites": 66, "partition": "disjoint and exhaustive", }, ) diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index 9d028071f2..7765ceaadc 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -32,7 +32,7 @@ [ ( "be", - "86143c1c3f98980e34490c75706fe7dbf72e96e8accb8d7d4bbd2c1ae1a29b65", + "29e0820ad7779164794f81671a2e51c799ffad601410e7261a59e5d2e0fdcf25", { "household.household_id", "person.person_id", @@ -42,7 +42,7 @@ ), ( "uk", - "4793baa59a90930956faf8f5b422ece57da3445c7ed0030622ef62ff35dfb142", + "c364a9426a08151d5ac1d66780f5ca283fcce65b0e3fa461340e06f3547e1ebe", { "benunit.benunit_id", "household.household_id", diff --git a/packages/microcosm-build/tests/test_spec_engine_coverage_tool.py b/packages/microcosm-build/tests/test_spec_engine_coverage_tool.py index 7a4e443bf0..fcfe372058 100644 --- a/packages/microcosm-build/tests/test_spec_engine_coverage_tool.py +++ b/packages/microcosm-build/tests/test_spec_engine_coverage_tool.py @@ -52,22 +52,22 @@ def test_us_coverage_is_exact_complete_and_honest( assert_coverage_complete(coverage_report) assert coverage_report["status"] == "pass" fields = coverage_report["field_usage"] - assert fields["configuration_field_count"] == 42_120 - assert fields["authored_normative_field_count"] == 32_351 - assert fields["resolved_binding_field_count"] == 9_769 - assert fields["consumed_field_count"] == 42_120 + assert fields["configuration_field_count"] == 42_419 + assert fields["authored_normative_field_count"] == 32_403 + assert fields["resolved_binding_field_count"] == 10_016 + assert fields["consumed_field_count"] == 42_419 assert fields["unused_field_count"] == 0 assert fields["multiple_primary_use_field_count"] == 0 assert fields["claim_count"] == 49 assert fields["mode_counts"] == { - "legacy_behavior": 13_970, - "compiler_semantic": 27_699, + "legacy_behavior": 13_983, + "compiler_semantic": 27_985, "front_end_validation": 348, "identity_only": 103, } assert fields["generation0_effect_counts"] == { - "legacy_behavior": 38_443, - "no_generation0_effect": 3_677, + "legacy_behavior": 38_456, + "no_generation0_effect": 3_963, } inventory = coverage_report["inventory_coverage"] @@ -78,7 +78,7 @@ def test_us_coverage_is_exact_complete_and_honest( assert inventory["counts"]["producer_inputs"] == 2_742 assert inventory["counts"]["ownership_rows"] == 18 assert inventory["counts"]["tail_control_fields"] == 934 - assert inventory["counts"]["seed_owner_bindings"] == 112 + assert inventory["counts"]["seed_owner_bindings"] == 125 @pytest.mark.parametrize( diff --git a/packages/microcosm-build/tests/test_spec_engine_inventory_coverage.py b/packages/microcosm-build/tests/test_spec_engine_inventory_coverage.py index 8f4e2673cb..a138f487e4 100644 --- a/packages/microcosm-build/tests/test_spec_engine_inventory_coverage.py +++ b/packages/microcosm-build/tests/test_spec_engine_inventory_coverage.py @@ -84,9 +84,9 @@ "producer_virtual_resources": 75, "release_rungs": 5, "resolved_references": 334, - "seed_owner_bindings": 112, + "seed_owner_bindings": 125, "seed_owner_rows": 54, - "seed_sites": 53, + "seed_sites": 66, "seed_streams": 14, "source_operators": 16, "source_stages": 37, diff --git a/packages/microcosm-build/tests/test_spec_engine_loader.py b/packages/microcosm-build/tests/test_spec_engine_loader.py index a52518ee63..b5e3d699e3 100644 --- a/packages/microcosm-build/tests/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/test_spec_engine_loader.py @@ -235,7 +235,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # Pin the domain separator, normalization rules, schema-set receipt, and # exact normative projection as one reviewable golden vector. assert first.spec_sha256 == ( - "1ff676072985f104b9d80e3e5fa6e2078778969246246cf1371e982c772550cc" + "774754e73efbe4d2e5d33c95c14e17e5f77789ee37a5885c1c2c3116b0dd5be8" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") diff --git a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py index d17303b3b1..462c9927fa 100644 --- a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py @@ -2465,7 +2465,7 @@ def capture_equality(expected: object, actual: object) -> None: "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "5378bb9189aec96f50da22aac71e5bd2c3d919e9795f6ef2147e0bc9c739dd8e", + "spec_sha256": "4c22013c378c881831b07735963b53c68a6f1bc97b17eb5a2e01d91101ee2cfd", }, } @@ -2668,7 +2668,7 @@ def test_constants_adapter_fixture_checkpoints_are_byte_identical_and_only_recei "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "5378bb9189aec96f50da22aac71e5bd2c3d919e9795f6ef2147e0bc9c739dd8e", + "spec_sha256": "4c22013c378c881831b07735963b53c68a6f1bc97b17eb5a2e01d91101ee2cfd", } def run_fixture(root: Path, *, config_authority: str) -> dict[str, object]: diff --git a/tools/spec_engine_coverage.py b/tools/spec_engine_coverage.py index 7a7a1786b2..3fa3373d59 100644 --- a/tools/spec_engine_coverage.py +++ b/tools/spec_engine_coverage.py @@ -41,7 +41,7 @@ REPORT_SCHEMA_VERSION = 3 EXPECTED_POINTER_INVENTORY_SHA256 = ( - "bc4a948ab632191954600da8474c5b011f977a65e24c399d126f3dc4a79f23e5" + "879bf3c70ad0d10c7a123a3b3194d3822f3015bed4bfba2b4a6703885a6bc95a" ) DEFAULT_REPORT_PATH = ( Path(__file__).resolve().parents[1] From 3c3bdbf543143a0f74cd46046f58b60c98a402ef Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 28 Aug 2026 17:26:32 +0200 Subject: [PATCH 10/49] Integrate humanitarian statuses with the US stacked pipeline Add source-aware ASEC and ACS immigration evidence, pooled control scaling, deterministic humanitarian reconciliation, calibration targets, terminal gates, authenticated receipts, and release regressions. Refs #767 --- docs/evidence/spec-engine/us-f0-coverage.json | 82 +- docs/spec-engine.md | 2 +- docs/us-multispine-operator-ordering.md | 18 +- .../build/spec_engine/field_usage.py | 32 +- .../build/spec_engine/inventory_coverage.py | 26 +- .../spec_engine/schema/imputation.schema.json | 75 ++ .../spec_engine/schema/spine.schema.json | 2 +- .../us/release_input_coverage_manifest.json | 2 +- .../src/microcosm/build/us/source_stages.json | 8 +- .../microcosm/build/us/spec/imputation.yaml | 122 +- .../src/microcosm/build/us/spec/sources.yaml | 57 +- .../src/microcosm/build/us/spec/spine.yaml | 3 +- .../microcosm/build/us_runtime/__init__.py | 8 + .../microcosm/build/us_runtime/acs_pums.py | 3 + .../build/us_runtime/acs_transfer.py | 170 ++- .../src/microcosm/build/us_runtime/h5_io.py | 42 +- .../microcosm/build/us_runtime/immigration.py | 1007 +++++++++++++++-- .../build/us_runtime/multispine_pool.py | 74 +- .../build/us_runtime/stacked_spine.py | 417 ++++++- .../us_runtime/us_late_producer_registry.py | 83 +- .../tests/test_spec_engine_compiler_ir.py | 18 +- .../tests/test_spec_engine_coverage_tool.py | 18 +- .../tests/test_spec_engine_field_usage.py | 28 +- .../test_spec_engine_identity_contracts.py | 4 +- .../test_spec_engine_imputation_semantics.py | 2 +- .../test_spec_engine_inventory_coverage.py | 2 +- .../tests/test_spec_engine_legacy_adapter.py | 4 +- ...spec_engine_stacked_authority_semantics.py | 6 +- .../microcosm-build/tests/test_us_acs_pums.py | 25 +- .../tests/test_us_acs_transfer.py | 394 ++++++- .../tests/test_us_bundle_core_contracts.py | 19 +- .../tests/test_us_fiscal_refresh_builder.py | 467 +++++++- .../tests/test_us_immigration.py | 357 ++++++ .../tests/test_us_immigration_pool_scaling.py | 328 ++++++ .../tests/test_us_late_producer_dag.py | 87 +- .../tests/test_us_multispine_pool.py | 17 +- .../tests/test_us_multispine_pool_h5_io.py | 498 +++++++- .../tests/test_us_multispine_pool_tool.py | 541 ++++++++- .../tests/test_us_spec_bundle.py | 23 +- .../tests/test_us_stacked_spine.py | 599 +++++++++- tools/build_us_fiscal_refresh_release.py | 117 +- tools/build_us_multispine_pool.py | 58 +- ...uild_us_release_input_coverage_manifest.py | 2 +- tools/generate_us_bundle_from_constants.py | 2 +- tools/spec_engine_coverage.py | 15 +- tools/us_bundle_generation/imputation.py | 22 +- 46 files changed, 5371 insertions(+), 515 deletions(-) create mode 100644 packages/microcosm-build/tests/test_us_immigration_pool_scaling.py diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index 7635980533..29bf7791a0 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -4,9 +4,9 @@ "version": 1 }, "country": "us", - "documentation_sha256": "b595dbfb3cda4ee18f9267f48950e1636f3b98656f2274aa9d45e117494dad26", + "documentation_sha256": "bc7bf008ee59fcc44890b06cb2a8a085e388a9764d2a5a7aa5e870caeb0112e4", "field_usage": { - "authored_normative_field_count": 32403, + "authored_normative_field_count": 32495, "claim_count": 49, "claims": [ { @@ -220,8 +220,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 79, - "pointer_sha256": "1575513251c72be60edc75f378221fa878ea76e009512f1352a111586d0a60f8", + "pointer_count": 97, + "pointer_sha256": "198bed038f330328119c0e0c65c39a4ec20342f2bd7e01ca60abce33321ff5f4", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/spec~1imputation.yaml/transfer_execution", @@ -274,8 +274,8 @@ ], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 24473, - "pointer_sha256": "1b9f7ffb88d31d2bd365c092d04a96038b2d5a2b03f4f2a79f42e95472b213d6", + "pointer_count": 24546, + "pointer_sha256": "548afd459263a5c0dd0b6b65328bcbe0993f10c7211069260161bc2ab72a29d7", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/spec~1imputation.yaml/producer_graph", @@ -530,8 +530,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 88, - "pointer_sha256": "ae011455154bae0df3913ca9a056058a909d913439b292cb46c11f63c7d0d9a3", + "pointer_count": 89, + "pointer_sha256": "8d281f8f1e2f9914684150b0424ff816d0ec09ed0d9053ea5a64e877a86cc35b", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/spec~1spine.yaml/pipeline_contract", @@ -772,20 +772,20 @@ "verifier": "vintages" } ], - "configuration_field_count": 42419, - "consumed_field_count": 42419, + "configuration_field_count": 42511, + "consumed_field_count": 42511, "generation0_effect_counts": { - "legacy_behavior": 38456, + "legacy_behavior": 38548, "no_generation0_effect": 3963 }, "mode_counts": { - "compiler_semantic": 27985, + "compiler_semantic": 28058, "front_end_validation": 348, "identity_only": 103, - "legacy_behavior": 13983 + "legacy_behavior": 14002 }, "multiple_primary_use_field_count": 0, - "pointer_inventory_sha256": "879bf3c70ad0d10c7a123a3b3194d3822f3015bed4bfba2b4a6703885a6bc95a", + "pointer_inventory_sha256": "db02c16cd852fb80d7316d69990cfeb9e2b2f9dec87ddb18a44178ea57422397", "resolved_binding_field_count": 10016, "unused_field_count": 0 }, @@ -810,7 +810,7 @@ "primary_targets": 65, "producer_authored_outputs": 92, "producer_compiled_outputs": 227, - "producer_inputs": 2742, + "producer_inputs": 2749, "producer_nodes": 38, "producer_virtual_resources": 75, "release_rungs": 5, @@ -1074,11 +1074,11 @@ "legacy_adapter.stacked_checkpoint_static_components" ], "expected": { - "schedule_sha256": "dcf3c6d2eade3449836c49a1dc4d3b8cd395aab9142db700c3c60598fa9c1c79" + "schedule_sha256": "4965c1485c283dec3685f4ca82fa469d8b88a85f82ccd6b39e2adc84bc0e94d6" }, "failures": [], "observed": { - "schedule_sha256": "dcf3c6d2eade3449836c49a1dc4d3b8cd395aab9142db700c3c60598fa9c1c79" + "schedule_sha256": "4965c1485c283dec3685f4ca82fa469d8b88a85f82ccd6b39e2adc84bc0e94d6" }, "status": "covered" }, @@ -1250,14 +1250,14 @@ "expected": { "edges": 71, "nodes": 38, - "schedule_sha256": "dcf3c6d2eade3449836c49a1dc4d3b8cd395aab9142db700c3c60598fa9c1c79", + "schedule_sha256": "4965c1485c283dec3685f4ca82fa469d8b88a85f82ccd6b39e2adc84bc0e94d6", "waves": 6 }, "failures": [], "observed": { "edges": 71, "nodes": 38, - "schedule_sha256": "dcf3c6d2eade3449836c49a1dc4d3b8cd395aab9142db700c3c60598fa9c1c79", + "schedule_sha256": "4965c1485c283dec3685f4ca82fa469d8b88a85f82ccd6b39e2adc84bc0e94d6", "waves": 6 }, "status": "covered" @@ -1275,11 +1275,11 @@ ], "expected": { "relation": "source rows preserved exactly", - "rows": 2742 + "rows": 2749 }, "failures": [], "observed": { - "rows": 2742 + "rows": 2749 }, "status": "covered" }, @@ -1321,7 +1321,7 @@ "expected": "complete execution-row and transition-authority object", "failures": [], "observed": { - "sha256": "503428f6e9d98f19ed3a6ada5bc9883ae44c1b5dc27f09e60b7ac98895a99bc0" + "sha256": "b15bf2dcf709cf6c7922d4b76e1d64c14a6ec2f9aa1185e6770505d798333c76" }, "status": "covered" }, @@ -1338,12 +1338,12 @@ ], "expected": { "nodes": 38, - "sha256": "7125ad28ae2c69f22094a574bbf6ed2ddf1682a2c2c3b416f8f49304b7016ce7" + "sha256": "15406d2ad456a4c09d123471a854194db14cbf8978ccb5235938062464ef18a0" }, "failures": [], "observed": { "nodes": 38, - "sha256": "7125ad28ae2c69f22094a574bbf6ed2ddf1682a2c2c3b416f8f49304b7016ce7" + "sha256": "15406d2ad456a4c09d123471a854194db14cbf8978ccb5235938062464ef18a0" }, "status": "covered" }, @@ -1361,12 +1361,12 @@ ], "expected": { "producer_count": 38, - "sha256": "4a74a831c92fc6273a82a22d02c73cca5e228852f4b88e496f83febc861fa222" + "sha256": "733569c8abc575bcdd72be35da850a7f12f2d0c81fb4745bd56a2c98db2e7079" }, "failures": [], "observed": { "producer_count": 38, - "sha256": "4a74a831c92fc6273a82a22d02c73cca5e228852f4b88e496f83febc861fa222" + "sha256": "733569c8abc575bcdd72be35da850a7f12f2d0c81fb4745bd56a2c98db2e7079" }, "status": "covered" }, @@ -1669,13 +1669,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "57b5dbd0fa563348885ec4b8d0fdfaace13c101c2fcd60d5620cf19592912f08", - "protocol_sha256": "baee9e47b31c369f1b3c137a97545c59bcb9f33fc96203894d289ff90c62270b" + "map_sha256": "7d88058ac155ac50818ccab8010c69352f4eb019181b0ec276cd333240489ef1", + "protocol_sha256": "c1f08bd279cf1b32e47fd00313b702f497cc7899cdbd4e1395fc59b1bcff3429" }, "failures": [], "observed": { - "map_sha256": "57b5dbd0fa563348885ec4b8d0fdfaace13c101c2fcd60d5620cf19592912f08", - "protocol_sha256": "baee9e47b31c369f1b3c137a97545c59bcb9f33fc96203894d289ff90c62270b" + "map_sha256": "7d88058ac155ac50818ccab8010c69352f4eb019181b0ec276cd333240489ef1", + "protocol_sha256": "c1f08bd279cf1b32e47fd00313b702f497cc7899cdbd4e1395fc59b1bcff3429" }, "status": "covered" }, @@ -1690,7 +1690,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "baee9e47b31c369f1b3c137a97545c59bcb9f33fc96203894d289ff90c62270b", + "implementation_sha256": "c1f08bd279cf1b32e47fd00313b702f497cc7899cdbd4e1395fc59b1bcff3429", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1711,7 +1711,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "baee9e47b31c369f1b3c137a97545c59bcb9f33fc96203894d289ff90c62270b", + "implementation_sha256": "c1f08bd279cf1b32e47fd00313b702f497cc7899cdbd4e1395fc59b1bcff3429", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1788,12 +1788,12 @@ "legacy_adapter.imputation.late_producer_resource_semantics" ], "expected": { - "sha256": "5e435fa74ff0f72fa5c8b7cda1ea9ac9cfdf7e114f06df07eece57142374b1ac", + "sha256": "6bf67f40b1afd2c63a7d7fa2c1f448570f4b7ab186475521966af65013de9e71", "stages": 37 }, "failures": [], "observed": { - "sha256": "5e435fa74ff0f72fa5c8b7cda1ea9ac9cfdf7e114f06df07eece57142374b1ac", + "sha256": "6bf67f40b1afd2c63a7d7fa2c1f448570f4b7ab186475521966af65013de9e71", "stages": 37 }, "status": "covered" @@ -1853,11 +1853,11 @@ "legacy_adapter.stacked_checkpoint_static_components" ], "expected": { - "sha256": "3a980927227704d0589f246eef9cd825c2ae84f3a4134ac835e0e5ed39a563ac" + "sha256": "2c01b975ebaeb06bfa666538178e1c4836fe294035c0ff7f2eed6a75e917bba6" }, "failures": [], "observed": { - "sha256": "3a980927227704d0589f246eef9cd825c2ae84f3a4134ac835e0e5ed39a563ac" + "sha256": "2c01b975ebaeb06bfa666538178e1c4836fe294035c0ff7f2eed6a75e917bba6" }, "status": "covered" }, @@ -1900,7 +1900,7 @@ "alpha", "zeta" ], - "sha256": "af1b04f0fe03938e826daf666e7242e4b138ebd5d8149abdda447359169ef3d5" + "sha256": "a7609a6235517555d244f126416ab0e73762e9f3963e42edbef79361193cbe7f" }, "failures": [], "observed": { @@ -1924,7 +1924,7 @@ "alpha", "zeta" ], - "sha256": "af1b04f0fe03938e826daf666e7242e4b138ebd5d8149abdda447359169ef3d5" + "sha256": "a7609a6235517555d244f126416ab0e73762e9f3963e42edbef79361193cbe7f" }, "status": "covered" }, @@ -1988,7 +1988,7 @@ "take_up_contract", "us_qbi_reconciliation_contract" ], - "sha256": "acbb196113c7ed0f7fbf2b0c95d5f57fc303d4fec3ef09475f3483745f34ba25" + "sha256": "0dc5273c438a77398ff9fa8d76a6bfeb38f37ea8e15202ba6760b385f385bec5" }, "status": "covered" }, @@ -2612,7 +2612,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "4c22013c378c881831b07735963b53c68a6f1bc97b17eb5a2e01d91101ee2cfd" + "spec_sha256": "ea202eebdfe30a68323af5cdce6abcc16b94c9f3f18f44ae5369fe7c1d9a5702" } }, "report_schema_version": 3, @@ -2622,7 +2622,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "4c22013c378c881831b07735963b53c68a6f1bc97b17eb5a2e01d91101ee2cfd" + "spec_sha256": "ea202eebdfe30a68323af5cdce6abcc16b94c9f3f18f44ae5369fe7c1d9a5702" }, "status": "pass" } diff --git a/docs/spec-engine.md b/docs/spec-engine.md index 624517390b..4a4de60463 100644 --- a/docs/spec-engine.md +++ b/docs/spec-engine.md @@ -172,7 +172,7 @@ run_provenance_identity: spec_binding: # {country, schema_id, schema_version, # canonicalizer_version, spec_sha256} authority_versions: # semantic contract version per named - # authority (stacked authority v10 is + # authority (stacked authority v11 is # the live example) — field, bump # rules, and precedence below code_inventory_digest # builder_code_identity + kernel set diff --git a/docs/us-multispine-operator-ordering.md b/docs/us-multispine-operator-ordering.md index 9a0054b809..c87623904b 100644 --- a/docs/us-multispine-operator-ordering.md +++ b/docs/us-multispine-operator-ordering.md @@ -246,13 +246,13 @@ by_origin_battery and target checkpoint schema remains version 6. The capital-gains tail manifest uses schema version 2 and binds its support contract and receipt. The canonical stacked authority is version 11, the outer stacked checkpoint - materializer uses version 12, and the stacked pool stage checkpoint - materializer uses version 7. + materializer uses version 13, and the stacked pool stage checkpoint + materializer uses version 8. The outer base identity binds primary-QRF version 6, the ACS universe and QBI reconciliation contracts, the tail schema and support contract, and - late-producer registry schema version 16, including the signed static and + late-producer registry schema version 17, including the signed static and derivation-mode semantics of every virtual DAG resource. The companion pool - manifest uses schema version 9. + manifest uses schema version 10. Older outer authority or materializer payloads are stale; primary-QRF version 6 remains current. @@ -309,7 +309,7 @@ by_origin_battery final publication reject a missing, stale, or reissued authority; NON-CANONICAL test receipts cannot ship. - Outer materializer v10 also embeds one signed resource-semantics row for + Outer materializer v13 also embeds one signed resource-semantics row for every DAG producer. Static configs are exact; donor tables are bound by the declared canonical scalar-content codec; primary and transfer banks name their outer/stage identity derivations; and source receipts name their @@ -879,13 +879,13 @@ The lexically canonical waves have sizes `(1, 1, 17, 14, 3, 2)`: 5. Education; adult-care transfer; WIC transfer. 6. `source_finalizer` and education transfer. -Registry schema version 13 and execution-receipt schema version 3 bind the +Registry schema version 17 and execution-receipt schema version 4 bind the canonical input declarations, outputs, edges, waves, exact kind-specific virtual-resource bindings, content-hashed execution-row schema, and immutable transition authority. The schedule SHA-256 is -`dbae9f945966a58592915780be78137e011d060271af6c933870a55db297baab`; +`4965c1485c283dec3685f4ca82fa469d8b88a85f82ccd6b39e2adc84bc0e94d6`; the full payload SHA-256 is -`95ee19cd1b4d1cf321a32910c234ebc460aa47f9cc30e03fa8560ea6ae5e2eb8`. +`9e72ceed9365ddf8993f68a25037a15cf53375093a60e59d5ce01627a7ceb210`. Reversing registry iteration produces those same bytes. The virtual-resource payload ledger is independently versioned: ACS-universe @@ -911,7 +911,7 @@ and valid. Neither receipt authorizes an upstream null. | PUF raw predictor sources | Every filing-status, count, and income component is observed in its declared source universe. Raw WAGP/SEMP authority is present and agrees with mapped leaves; a cross-grain source collision is rejected. A null on any eligible member fails before coercion. | Structure supplies status/count; ACS-native or ASEC-carried earnings supply earnings; early transfer supplies interest, dividends, and gains. | No. ACS under-15 WAGP/SEMP blanks are an exact source-universe state, not transfer starvation; all other source nulls fail. | | PUF tax-unit features | Every clone-1 recipient has a finite feature vector. Post-aggregation NaN, `+inf`, and `-inf` are counted by named predictor and rejected before fitting; none is coerced or snapped to zero. | Universe-aware person sums plus tax-unit structural inputs. | No. Eligible member values must be complete; the only special case is an all-child unit whose numeric-zero predictor is explicitly owned and counted by the named universe-zero rule. | | Primary QRF banks and chain | Donor/recipient banks are immutable; target order and RNG prefix are contiguous; all targets complete; live recipient identity, source-universe receipt, and feature digest match before finalization. | The processed full PUF donor and strict recipient checkpoint initialized above. | No. Mutation or missing receipt invalidates the bank; it cannot resume under legacy semantics. | -| Outer pool checkpoint identity and resume | Primary-QRF schema v6, tail-manifest schema v2, late-registry schema v14/receipt schema v3, outer stacked materializer v10/authority v9, stacked pool-stage materializer v5, pool manifest schema v7, and the ACS-universe, QBI-mutation, tail-support, late-DAG, and signed virtual-resource-semantics identities must match exactly before any cached stage is discovered. The retiring legacy envelope remains manifest schema v4/materializer v3. | Fresh input pins, live stack receipt, scale controls, code identity, and all semantic contract identities. | No. An older stacked materializer or authority payload is stale; a self-consistent old receipt cannot reopen a checkpoint. Primary-QRF v6 remains current. | +| Outer pool checkpoint identity and resume | Primary-QRF schema v6, tail-manifest schema v2, late-registry schema v17/receipt schema v4, outer stacked materializer v13/authority v11, stacked pool-stage materializer v8, pool manifest schema v10, and the ACS-universe, QBI-mutation, tail-support, late-DAG, and signed virtual-resource-semantics identities must match exactly before any cached stage is discovered. The retiring legacy envelope remains manifest schema v4/materializer v3. | Fresh input pins, live stack receipt, scale controls, code identity, and all semantic contract identities. | No. An older stacked materializer or authority payload is stale; a self-consistent old receipt cannot reopen a checkpoint. Primary-QRF v6 remains current. | | Clone-2 capital-gains tail | Each filing status requires as many eligible recipient households as selected q99.5 donors. Eligibility requires unique single-tax-unit PUF-detail lineage and half-weight capacity for the global maximum assigned donor weight. An adequate status assigns every selected donor once; a thin status skips as a whole with a named, counted `insufficient_support` receipt. | Completed clone-1 QRF output and full PUF tail donors. At 1%, `SINGLE` and `HEAD_OF_HOUSEHOLD` attach, `JOINT` and `SEPARATE` skip, and zero-requirement `SURVIVING_SPOUSE` is `not_applicable`. | No widening or partial attachment is permitted. All 22 AGI bands provide nearest-first fallback only inside a status. Universe-aware PUF recipients remain eligible, including explicitly receipted empty-universe tax units. | | Late producer DAG | Before any callback, all declared inputs are filled on their required scopes or carry an input-specific counted absence receipt; numeric inputs are finite. The exact derived order, readiness rows, once-only source finalizer, and bounded transfer receipts must validate. | ACS earnings-universe materialization, primary PUF/tail, 16 source producers, and 19 bounded transfer groups execute in six derived waves. | No. The refusing producer names the unfilled input and its declared producing stage. A cycle fails at import with its path. | | Late transfer completion | Every declared PUF-clone or ASEC source-producer cell is nonnull; all complementary recipients are filled; the allowed count for both unmodeled and residual rows is zero. | Forty-three PUF and 29 source targets, with two overlaps, supply the 70-target late surface. | No. A missing producer or recipient value is terminal at this boundary. | diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py index 89f230faf9..0daf256e86 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py @@ -26,9 +26,9 @@ ) from .schemas import load_schema_registry -EXPECTED_AUTHORED_FIELD_COUNT = 32_403 +EXPECTED_AUTHORED_FIELD_COUNT = 32_495 EXPECTED_RESOLVED_BINDING_FIELD_COUNT = 10_016 -EXPECTED_CONFIGURATION_FIELD_COUNT = 42_419 +EXPECTED_CONFIGURATION_FIELD_COUNT = 42_511 class FieldUsageError(AssertionError): @@ -249,13 +249,9 @@ def is_stacked_geography_source_identity(pointer: str) -> bool: ) if claim.pointer_class == "stacked_geography_source_identity": - return [ - row for row in rows if is_stacked_geography_source_identity(row[0]) - ] + return [row for row in rows if is_stacked_geography_source_identity(row[0])] if claim.pointer_class == "source_validation": - return [ - row for row in rows if not is_stacked_geography_source_identity(row[0]) - ] + return [row for row in rows if not is_stacked_geography_source_identity(row[0])] raise FieldUsageError(f"{claim.id}: unknown pointer class {claim.pointer_class!r}") @@ -425,12 +421,12 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "e1dd7dc5123ab0f39d08ea4939d98dd09a6fdb8e7449a7ca3125fb1ddbd5b4e9", ), "imputation_producer_graph": ( - 24_473, - "1b9f7ffb88d31d2bd365c092d04a96038b2d5a2b03f4f2a79f42e95472b213d6", + 24_546, + "548afd459263a5c0dd0b6b65328bcbe0993f10c7211069260161bc2ab72a29d7", ), "imputation_transfer_execution": ( - 79, - "1575513251c72be60edc75f378221fa878ea76e009512f1352a111586d0a60f8", + 97, + "198bed038f330328119c0e0c65c39a4ec20342f2bd7e01ca60abce33321ff5f4", ), "imputation_waiver_records": ( 70, @@ -509,8 +505,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "cf0000464013118955571dac2691d4cc1b900c97c6570349b489678cdf937649", ), "spine_pipeline_contract": ( - 88, - "ae011455154bae0df3913ca9a056058a909d913439b292cb46c11f63c7d0d9a3", + 89, + "8d281f8f1e2f9914684150b0424ff816d0ec09ed0d9053ea5a64e877a86cc35b", ), "spine_sampling": ( 17, @@ -1125,7 +1121,9 @@ def _verify_source_pins(context: _VerificationContext, claim: UsageClaim) -> Non if isinstance(row, Mapping) and isinstance(row.get("id"), str) ] if len(ids) != len(rows) or len(ids) != len(set(ids)): - raise FieldUsageError("source_pins: source ids are not an exact unique registry") + raise FieldUsageError( + "source_pins: source ids are not an exact unique registry" + ) expected_refs: set[tuple[str, str, str]] = set() for index, value in enumerate(rows): @@ -1468,9 +1466,7 @@ def _verify_claim(context: _VerificationContext, claim: UsageClaim) -> None: "legacy": lambda: _verify_legacy(context, claim), "source_pins": lambda: _verify_source_pins(context, claim), "spine_channels": lambda: _verify_spine_channels(context, claim), - "spine_assembly_legacy": lambda: _verify_spine_assembly_legacy( - context, claim - ), + "spine_assembly_legacy": lambda: _verify_spine_assembly_legacy(context, claim), "spine_assembly_validation": lambda: _verify_spine_assembly_validation( context, claim ), diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 16e337c94e..036b80c737 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -361,20 +361,20 @@ EXPECTED_HASHES = { "acs_group_predictors": "a927bb7ecf3e84f54c93583ab79318654514ac546aefafba67da5285615fbd60", "acs_person_predictors": "878c788a6f037d7aca12b3586ea034eff04f3034ffa11935a736493042551f25", - "authority": "3a980927227704d0589f246eef9cd825c2ae84f3a4134ac835e0e5ed39a563ac", + "authority": "2c01b975ebaeb06bfa666538178e1c4836fe294035c0ff7f2eed6a75e917bba6", "early_families": "4aa9f736fd76e83955477ad1667e58f48f264783f05bdc7f0102cd32d61323bd", - "full_checkpoint": "af1b04f0fe03938e826daf666e7242e4b138ebd5d8149abdda447359169ef3d5", + "full_checkpoint": "a7609a6235517555d244f126416ab0e73762e9f3963e42edbef79361193cbe7f", "gap_fill_schedule": "1c31f9868f7884347cc19cf1ff65da43f950b9114941a715bab168246db414a7", - "graph_nodes": "7125ad28ae2c69f22094a574bbf6ed2ddf1682a2c2c3b416f8f49304b7016ce7", + "graph_nodes": "15406d2ad456a4c09d123471a854194db14cbf8978ccb5235938062464ef18a0", "geography_assignment": "f49425ca8734ac559c73cf44f6458d86d3162a48956b98a27e6e758959361585", "late_families": "d91f9ff0eb52f43e7b6eed3d5c58c37abe1620c3a11021da15dae9c10e16d382", - "late_resource_semantics": "4a74a831c92fc6273a82a22d02c73cca5e228852f4b88e496f83febc861fa222", - "late_schedule": "dcf3c6d2eade3449836c49a1dc4d3b8cd395aab9142db700c3c60598fa9c1c79", + "late_resource_semantics": "733569c8abc575bcdd72be35da850a7f12f2d0c81fb4745bd56a2c98db2e7079", + "late_schedule": "4965c1485c283dec3685f4ca82fa469d8b88a85f82ccd6b39e2adc84bc0e94d6", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "987b501c695e31f45521c4a178528f75ab3df22c09bc407b182213b2de99ee57", - "seed_map": "57b5dbd0fa563348885ec4b8d0fdfaace13c101c2fcd60d5620cf19592912f08", - "seed_protocol": "baee9e47b31c369f1b3c137a97545c59bcb9f33fc96203894d289ff90c62270b", - "source_manifest": "5e435fa74ff0f72fa5c8b7cda1ea9ac9cfdf7e114f06df07eece57142374b1ac", + "seed_map": "7d88058ac155ac50818ccab8010c69352f4eb019181b0ec276cd333240489ef1", + "seed_protocol": "c1f08bd279cf1b32e47fd00313b702f497cc7899cdbd4e1395fc59b1bcff3429", + "source_manifest": "6bf67f40b1afd2c63a7d7fa2c1f448570f4b7ab186475521966af65013de9e71", "take_up": "fa186daea0f8dd641cc470e41d1a2953f887d45282ec990201298f47bedf8d4d", "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", } @@ -442,7 +442,7 @@ "primary_targets": 65, "producer_authored_outputs": 92, "producer_compiled_outputs": 227, - "producer_inputs": 2_742, + "producer_inputs": 2_749, "producer_nodes": 38, "producer_virtual_resources": 75, "release_rungs": 5, @@ -1018,7 +1018,7 @@ def add( "producer_inputs_exact", clauses={ "producer input rows differ": inputs_exact, - "input row count differs": input_count == 2742, + "input row count differs": input_count == 2749, }, homes=("/imputation/producer_graph/nodes/*/inputs",), consumers=( @@ -1026,7 +1026,7 @@ def add( "compiler_ir.node_slices", ), observed={"rows": input_count}, - expected={"rows": 2742, "relation": "source rows preserved exactly"}, + expected={"rows": 2749, "relation": "source rows preserved exactly"}, ) outputs_exact = set(expected_outputs) == set(compiled_by_id) and all( _json_equal( @@ -1708,9 +1708,7 @@ def add( "sha256": _operational_free_sha256(geography_assignment), "authority_roles": sorted(geography_authorities), "target_vintage": _mapping( - geography_authorities.get( - "congressional_district_vintage_crosswalk" - ), + geography_authorities.get("congressional_district_vintage_crosswalk"), "checkpoint geography crosswalk authority", ).get("target_vintage"), }, diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/imputation.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/imputation.schema.json index 0c2a2548ea..fb618eac32 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/imputation.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/imputation.schema.json @@ -339,6 +339,22 @@ "immigration_status_model_target": { "type": "string" }, + "immigration_required_predictors": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "uniqueItems": true + }, + "immigration_baseline_excluded_statuses": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "uniqueItems": true + }, "immigration_status_targets": { "type": "array", "items": { @@ -397,6 +413,62 @@ ] } }, + "required": [ + "activation", + "contract" + ] + }, + "humanitarian_immigration": { + "type": "object", + "additionalProperties": false, + "properties": { + "activation": { + "type": "object", + "additionalProperties": false, + "properties": { + "all_targets": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "all_targets" + ] + }, + "contract": { + "type": "object", + "additionalProperties": false, + "properties": { + "candidate_shortfall": { + "type": "string" + }, + "mutable_rows": { + "type": "string" + }, + "selection": { + "type": "string" + }, + "target_basis": { + "type": "string" + }, + "targets": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "candidate_shortfall", + "mutable_rows", + "selection", + "target_basis", + "targets" + ] + } + }, "required": [ "activation", "contract" @@ -526,6 +598,7 @@ }, "required": [ "adult_care", + "humanitarian_immigration", "schedule_d_capital_gain_distributions" ] }, @@ -671,6 +744,8 @@ "donor_combined_components", "group_optional_names", "housing", + "immigration_baseline_excluded_statuses", + "immigration_required_predictors", "immigration_status_model_target", "immigration_status_targets", "post_transfer_features", diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/spine.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/spine.schema.json index dc4c7955d6..251360cf95 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/spine.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/spine.schema.json @@ -288,7 +288,7 @@ "const": "populace_us_stacked_pool_checkpoint_identity" }, "schema_version": {"const": 1}, - "materializer_version": {"const": 12}, + "materializer_version": {"const": 13}, "pipeline": {"const": "us-stacked-pool"} } }, diff --git a/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json index d93bdf47d5..7d9ef8cdd4 100644 --- a/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json @@ -1441,7 +1441,7 @@ } }, "period": 2027, - "reason": "H.R.1 SS71109 narrows the Medicaid qualified-immigrant list to citizens, LPRs, and Cuban/Haitian entrants from 2026-10-01; the probe restores the pre-narrowing list at 2027 law, which binds only through the REFUGEE/ASYLEE/DEPORTATION_WITHHELD/PAROLED_ONE_YEAR values of immigration_status_str. The source stage draws roughly 160k refugees (74% reporting Medicaid on the 2024 ASEC), 160k asylees, and 405k parolees to cited DHS/OHSS stocks, so restoring their eligibility must re-enroll anchored takers and move person-level medicaid dollars by far more than the floor. A ~$0 score means the humanitarian statuses regressed to zero records (microcosm #767's silent-zero failure) or the engine channel broke." + "reason": "H.R.1 SS71109 narrows the Medicaid qualified-immigrant list to citizens, LPRs, and Cuban/Haitian entrants from 2026-10-01; the probe restores the pre-narrowing list at 2027 law, which binds only through the REFUGEE/ASYLEE/DEPORTATION_WITHHELD/PAROLED_ONE_YEAR values of immigration_status_str. The source stage draws roughly 160k refugees (74% reporting Medicaid on the 2024 ASEC), 155k asylees, and 405k parolees to cited DHS/OHSS stocks, so restoring their eligibility must re-enroll anchored takers and move person-level medicaid dollars by far more than the floor. A ~$0 score means the humanitarian statuses regressed to zero records (microcosm #767's silent-zero failure) or the engine channel broke." }, { "binding_inputs": [ diff --git a/packages/microcosm-build/src/microcosm/build/us/source_stages.json b/packages/microcosm-build/src/microcosm/build/us/source_stages.json index af0b376f54..d471f38808 100644 --- a/packages/microcosm-build/src/microcosm/build/us/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/us/source_stages.json @@ -1925,11 +1925,11 @@ }, "refugee": { "target": 160000, - "source": "https://ohss.dhs.gov/sites/default/files/2024-11/2024_1108_ohss_refugee_annual_flow_report_2023.pdf" + "source": "https://ohss.dhs.gov/topics/immigration/refugees/annual-flow-report/fy-24-refugees-flow-report" }, "asylee": { - "target": 160000, - "source": "https://ohss.dhs.gov/sites/default/files/2024-10/2024_1002_ohss_asylees_fy2023.pdf" + "target": 155000, + "source": "https://ohss.dhs.gov/topics/immigration/asylees/annual-flow-report/fy24-asylees-flow-report" }, "deportation_withheld": { "target": 0, @@ -1968,7 +1968,7 @@ "ssn_card_type", "immigration_status_str" ], - "notes": "Citizenship is measured (PRCITSHP), not imputed. Non-citizens with any ASEC-UA legal-status indicator (Van Hook et al., SSRN 4662801) become OTHER_NON_CITIZEN. Humanitarian/temporary-protection statuses (microcosm #767) are drawn between the indicators and the EAD spill, sequentially and mutually exclusively in decreasing order of target precision \u2014 PAROLED_ONE_YEAR per program origin (OAW Afghans 73,566 as of 2022-03-31; U4U Ukrainians over 158,000 as of 2023-09-30; CHNV Nicaraguans 93,070 and Venezuelans 117,330 paroled through 2024-12), REFUGEE (FY2023+FY2024 admissions, 60,050+100,060 OHSS \u2014 refugees must apply for LPR adjustment after one year, so the not-yet-adjusted stock is about the trailing two years), ASYLEE (FY2022-FY2024 grants 35,720+54,350+~70,000 OHSS; 88% of 2015-22 affirmative adult grantees had adjusted by end-2024, so about three trailing years), then TPS per country (CRS RS20844 Table 1 as of 2025-03-31, with required-arrival cutoffs binding at PEINUSYR granularity: El Salvador <=2001, Honduras/Nicaragua <=1999, Nepal <=2015). Candidates are restricted by program origin countries (Census Appendix I PENATVTY codes) and arrival windows; Cuba/Haiti-born are excluded everywhere (the CUBAN_HAITIAN_ENTRANT class is the better statutory label and keeps H.R.1 eligibility \u2014 Haiti TPS 330,735 rides that label), the DACA statutory cohort is excluded (dual TPS/DACA holders stay DACA), and Ukraine/Afghanistan TPS registrants (101,150/8,105) are carried by the parole draw since they are overwhelmingly the same U4U/OAW parolees. Parole targets are gross admissions: cohort members who later won asylum keep the parole label, which is channel-equivalent under H.R.1 (both classes lose Medicaid/SNAP/PTC on identical dates). DEPORTATION_WITHHELD carries an explicit zero target: EOIR grants about 2.2k withholdings a year (CRS IN12501: under 1% of 666,177 FY2024 decisions) and publishes no stock, so the accumulated population is below the CPS's resolution for a defensible draw. CONDITIONAL_ENTRANT is deliberately not emitted (the INA 203(a)(7) class closed in 1980). REFUGEE/ASYLEE draw from the indicator-documented pool only; parole/TPS may draw residual-pool people, who then receive NON_CITIZEN_VALID_EAD SSN cards (both programs confer employment authorization), consistent with Pew's own accounting of TPS/parolees inside its unauthorized totals. The EAD worker/student spill then binds the remaining residual pool to Pew's 8.3M workers and the Higher Ed Immigration Portal's ~408k students \u2014 still the only forced margins. The total undocumented population is emergent and gated against Pew's 11.0M 2022 anchor with a coarse band; per-category humanitarian masses are gated against their cited targets with a coarse band (the ASEC undercovers 2022-24 arrivals \u2014 the Ukraine parole pool saturates below its admin count); reconciling levels is the calibration lane's job. DACA (arrival cohort among EAD holders) and CUBAN_HAITIAN_ENTRANT (nativity plus post-1980 arrival) keep their statutory tests; other documented non-citizens stay LEGAL_PERMANENT_RESIDENT, the modal true status." + "notes": "Citizenship is measured (PRCITSHP), not imputed. Non-citizens with any ASEC-UA legal-status indicator (Van Hook et al., SSRN 4662801) become OTHER_NON_CITIZEN. Humanitarian/temporary-protection statuses (microcosm #767) are drawn between the indicators and the EAD spill, sequentially and mutually exclusively in decreasing order of target precision \u2014 PAROLED_ONE_YEAR per program origin (OAW Afghans 73,566 as of 2022-03-31; U4U Ukrainians over 158,000 as of 2023-09-30; CHNV Nicaraguans 93,070 and Venezuelans 117,330 paroled through 2024-12), REFUGEE (FY2023+FY2024 admissions, 60,050+100,060 OHSS \u2014 refugees must apply for LPR adjustment after one year, so the not-yet-adjusted stock is about the trailing two years), ASYLEE (FY2022-FY2024 grants 35,080+51,410+68,060=154,550 OHSS, rounded to 155,000; 88% of 2015-22 affirmative adult grantees had adjusted by end-2024, so about three trailing years), then TPS per country (CRS RS20844 Table 1 as of 2025-03-31, with required-arrival cutoffs binding at PEINUSYR granularity: El Salvador <=2001, Honduras/Nicaragua <=1999, Nepal <=2015). Candidates are restricted by program origin countries (Census Appendix I PENATVTY codes) and arrival windows; Cuba/Haiti-born are excluded everywhere (the CUBAN_HAITIAN_ENTRANT class is the better statutory label and keeps H.R.1 eligibility \u2014 Haiti TPS 330,735 rides that label), the DACA statutory cohort is excluded (dual TPS/DACA holders stay DACA), and Ukraine/Afghanistan TPS registrants (101,150/8,105) are carried by the parole draw since they are overwhelmingly the same U4U/OAW parolees. Parole targets are gross admissions: cohort members who later won asylum keep the parole label, which is channel-equivalent under H.R.1 (both classes lose Medicaid/SNAP/PTC on identical dates). DEPORTATION_WITHHELD carries an explicit zero target: EOIR grants about 2.2k withholdings a year (CRS IN12501: under 1% of 666,177 FY2024 decisions) and publishes no stock, so the accumulated population is below the CPS's resolution for a defensible draw. CONDITIONAL_ENTRANT is deliberately not emitted (the INA 203(a)(7) class closed in 1980). REFUGEE/ASYLEE draw from the indicator-documented pool only; parole/TPS may draw residual-pool people, who then receive NON_CITIZEN_VALID_EAD SSN cards (both programs confer employment authorization), consistent with Pew's own accounting of TPS/parolees inside its unauthorized totals. The EAD worker/student spill then binds the remaining residual pool to Pew's 8.3M workers and the Higher Ed Immigration Portal's ~408k students \u2014 still the only forced margins. The total undocumented population is emergent and gated against Pew's 11.0M 2022 anchor with a coarse band; per-category humanitarian masses are gated against their cited targets with a coarse band (the ASEC undercovers 2022-24 arrivals \u2014 the Ukraine parole pool saturates below its admin count); reconciling levels is the calibration lane's job. DACA (arrival cohort among EAD holders) and CUBAN_HAITIAN_ENTRANT (nativity plus post-1980 arrival) keep their statutory tests; other documented non-citizens stay LEGAL_PERMANENT_RESIDENT, the modal true status." }, { "stage": "hours_worked", diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/imputation.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/imputation.yaml index f0a7ed69c9..167b1b0407 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/imputation.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/imputation.yaml @@ -342,7 +342,11 @@ waiver_records: - dependent_child_status - pregnancy_status transfer_execution: - schema_version: 2 + schema_version: 3 + immigration_required_predictors: + - __acs_transfer_is_us_citizen + - __acs_transfer_birth_country_code + - __acs_transfer_arrival_year group_optional_names: __acs_transfer_employment_income: __acs_transfer_group_employment_income_sum __acs_transfer_interest_dividend_rental_income: __acs_transfer_group_investment_income_sum @@ -405,6 +409,14 @@ transfer_execution: - ssn_card_type - immigration_status_str immigration_status_model_target: __acs_transfer_immigration_status_pair + immigration_baseline_excluded_statuses: + - ASYLEE + - CUBAN_HAITIAN_ENTRANT + - DACA + - DEPORTATION_WITHHELD + - PAROLED_ONE_YEAR + - REFUGEE + - TPS discrete_numeric_targets: - first_home_mortgage_origination_year - second_home_mortgage_origination_year @@ -427,6 +439,19 @@ transfer_execution: tax_unit_role: tax_unit_role_input tax_unit_link: person_tax_unit_id mutable_rows: newly_imputed_expense_cells_only + humanitarian_immigration: + activation: + all_targets: + - ssn_card_type + - immigration_status_str + contract: + targets: + - ssn_card_type + - immigration_status_str + mutable_rows: newly_imputed_paired_cells_only + target_basis: manifest_residual_after_immutable_asec_mass + selection: stable_person_hash_in_manifest_draw_order + candidate_shortfall: error schedule_d_capital_gain_distributions: activation: derive_schedule_d: true @@ -2753,7 +2778,7 @@ families: output_coverage_scope: whole_pool producer_graph: graph_schema_version: 2 - schedule_payload_schema_version: 16 + schedule_payload_schema_version: 17 external_stages: - post_clone_input_surface scope_coverage: @@ -9785,6 +9810,15 @@ producer_graph: - - column: person_id entity: person value_kind: non_null + - - column: source_household_id + entity: person + value_kind: non_null + - column: source_person_id + entity: person + value_kind: non_null + - column: source_year + entity: person + value_kind: non_null - - column: source_person_id entity: person value_kind: non_null @@ -24910,6 +24944,84 @@ producer_graph: producing_stage: post_clone_input_surface required_scope: whole_pool tolerated_absence_receipts: [] + - alternatives: + - - column: YOEP + entity: person + value_kind: column_present + column: '@effective:immigration_acs_arrival' + entity: person + producing_stage: post_clone_input_surface + required_scope: acs_source + tolerated_absence_receipts: [] + - alternatives: + - - column: CIT + entity: person + value_kind: finite_numeric + column: '@effective:immigration_acs_citizenship' + entity: person + producing_stage: post_clone_input_surface + required_scope: acs_source + tolerated_absence_receipts: [] + - alternatives: + - - column: POBP + entity: person + value_kind: finite_numeric + column: '@effective:immigration_acs_origin' + entity: person + producing_stage: post_clone_input_surface + required_scope: acs_source + tolerated_absence_receipts: [] + - alternatives: + - - column: PEINUSYR + entity: person + value_kind: finite_numeric + column: '@effective:immigration_asec_arrival' + entity: person + producing_stage: post_clone_input_surface + required_scope: asec_source + tolerated_absence_receipts: [] + - alternatives: + - - column: PRCITSHP + entity: person + value_kind: finite_numeric + column: '@effective:immigration_asec_citizenship' + entity: person + producing_stage: post_clone_input_surface + required_scope: asec_source + tolerated_absence_receipts: [] + - alternatives: + - - column: PENATVTY + entity: person + value_kind: finite_numeric + column: '@effective:immigration_asec_origin' + entity: person + producing_stage: post_clone_input_surface + required_scope: asec_source + tolerated_absence_receipts: [] + - alternatives: + - - column: person_id + entity: person + value_kind: non_null + - - column: source_household_id + entity: person + value_kind: non_null + - column: source_person_id + entity: person + value_kind: non_null + - column: source_year + entity: person + value_kind: non_null + - - column: source_person_id + entity: person + value_kind: non_null + - column: source_year + entity: person + value_kind: non_null + column: '@effective:immigration_stable_person_lineage' + entity: person + producing_stage: post_clone_input_surface + required_scope: whole_pool + tolerated_absence_receipts: [] - alternatives: - - column: is_female entity: person @@ -25229,7 +25341,7 @@ producer_graph: - alternatives: - - column: person_id entity: person - value_kind: finite_numeric + value_kind: non_null column: person_id entity: person producing_stage: primary_puf_qrf @@ -33333,10 +33445,10 @@ producer_graph: - first_home_mortgage_origination_year - health_savings_account_ald execution_receipt_contract: - version: 3 + version: 4 row_binding: declared_globally_reconciled_input_and_scope_exact_output_source_and_primary_callback_resource_receipt_and_previous_execution_sha256 virtual_resource_binding: exact_kind_specific_semantic_payload_and_sha256 - top_binding: entry_and_output_frame_sha256_execution_chain_source_completion_and_nineteen_transfer_groups + top_binding: entry_and_output_frame_sha256_execution_chain_source_completion_nineteen_transfer_groups_and_constrained_immigration_reconciliation transition_authority: authority_id: us_stacked_late_producer_transition metadata_key: us_late_producer_transition_authority diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml index ba0b87935e..b20d6856e7 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml @@ -99,7 +99,7 @@ sources: stage_asset: id: source_stages path: microcosm.build.us/source_stages.json - sha256: 619a974abd80c34201a0ccf813968ebc09ed52e144ff1d39722a4d9be7bd7821 + sha256: 413174820cfb57f85783831dbbf445f007d06128d31562a1234e998e627bd3aa stage_manifest: version: 1 country: us @@ -1931,10 +1931,10 @@ stages: source: https://www.cbp.gov/newsroom/national-media-release/cbp-releases-december-2024-monthly-update refugee: target: 160000 - source: https://ohss.dhs.gov/sites/default/files/2024-11/2024_1108_ohss_refugee_annual_flow_report_2023.pdf + source: https://ohss.dhs.gov/topics/immigration/refugees/annual-flow-report/fy-24-refugees-flow-report asylee: - target: 160000 - source: https://ohss.dhs.gov/sites/default/files/2024-10/2024_1002_ohss_asylees_fy2023.pdf + target: 155000 + source: https://ohss.dhs.gov/topics/immigration/asylees/annual-flow-report/fy24-asylees-flow-report deportation_withheld: target: 0 source: https://crsreports.congress.gov/product/pdf/IN/IN12501 @@ -1967,30 +1967,31 @@ stages: 73,566 as of 2022-03-31; U4U Ukrainians over 158,000 as of 2023-09-30; CHNV Nicaraguans 93,070 and Venezuelans 117,330 paroled through 2024-12), REFUGEE (FY2023+FY2024 admissions, 60,050+100,060 OHSS — refugees must apply for LPR adjustment after one year, so the not-yet-adjusted stock is about the - trailing two years), ASYLEE (FY2022-FY2024 grants 35,720+54,350+~70,000 OHSS; 88% of 2015-22 affirmative - adult grantees had adjusted by end-2024, so about three trailing years), then TPS per country (CRS - RS20844 Table 1 as of 2025-03-31, with required-arrival cutoffs binding at PEINUSYR granularity: El - Salvador <=2001, Honduras/Nicaragua <=1999, Nepal <=2015). Candidates are restricted by program origin - countries (Census Appendix I PENATVTY codes) and arrival windows; Cuba/Haiti-born are excluded everywhere - (the CUBAN_HAITIAN_ENTRANT class is the better statutory label and keeps H.R.1 eligibility — Haiti - TPS 330,735 rides that label), the DACA statutory cohort is excluded (dual TPS/DACA holders stay DACA), - and Ukraine/Afghanistan TPS registrants (101,150/8,105) are carried by the parole draw since they - are overwhelmingly the same U4U/OAW parolees. Parole targets are gross admissions: cohort members - who later won asylum keep the parole label, which is channel-equivalent under H.R.1 (both classes - lose Medicaid/SNAP/PTC on identical dates). DEPORTATION_WITHHELD carries an explicit zero target: - EOIR grants about 2.2k withholdings a year (CRS IN12501: under 1% of 666,177 FY2024 decisions) and - publishes no stock, so the accumulated population is below the CPS''s resolution for a defensible - draw. CONDITIONAL_ENTRANT is deliberately not emitted (the INA 203(a)(7) class closed in 1980). REFUGEE/ASYLEE - draw from the indicator-documented pool only; parole/TPS may draw residual-pool people, who then receive - NON_CITIZEN_VALID_EAD SSN cards (both programs confer employment authorization), consistent with Pew''s - own accounting of TPS/parolees inside its unauthorized totals. The EAD worker/student spill then binds - the remaining residual pool to Pew''s 8.3M workers and the Higher Ed Immigration Portal''s ~408k students - — still the only forced margins. The total undocumented population is emergent and gated against Pew''s - 11.0M 2022 anchor with a coarse band; per-category humanitarian masses are gated against their cited - targets with a coarse band (the ASEC undercovers 2022-24 arrivals — the Ukraine parole pool saturates - below its admin count); reconciling levels is the calibration lane''s job. DACA (arrival cohort among - EAD holders) and CUBAN_HAITIAN_ENTRANT (nativity plus post-1980 arrival) keep their statutory tests; - other documented non-citizens stay LEGAL_PERMANENT_RESIDENT, the modal true status.' + trailing two years), ASYLEE (FY2022-FY2024 grants 35,080+51,410+68,060=154,550 OHSS, rounded to 155,000; + 88% of 2015-22 affirmative adult grantees had adjusted by end-2024, so about three trailing years), + then TPS per country (CRS RS20844 Table 1 as of 2025-03-31, with required-arrival cutoffs binding + at PEINUSYR granularity: El Salvador <=2001, Honduras/Nicaragua <=1999, Nepal <=2015). Candidates + are restricted by program origin countries (Census Appendix I PENATVTY codes) and arrival windows; + Cuba/Haiti-born are excluded everywhere (the CUBAN_HAITIAN_ENTRANT class is the better statutory label + and keeps H.R.1 eligibility — Haiti TPS 330,735 rides that label), the DACA statutory cohort is excluded + (dual TPS/DACA holders stay DACA), and Ukraine/Afghanistan TPS registrants (101,150/8,105) are carried + by the parole draw since they are overwhelmingly the same U4U/OAW parolees. Parole targets are gross + admissions: cohort members who later won asylum keep the parole label, which is channel-equivalent + under H.R.1 (both classes lose Medicaid/SNAP/PTC on identical dates). DEPORTATION_WITHHELD carries + an explicit zero target: EOIR grants about 2.2k withholdings a year (CRS IN12501: under 1% of 666,177 + FY2024 decisions) and publishes no stock, so the accumulated population is below the CPS''s resolution + for a defensible draw. CONDITIONAL_ENTRANT is deliberately not emitted (the INA 203(a)(7) class closed + in 1980). REFUGEE/ASYLEE draw from the indicator-documented pool only; parole/TPS may draw residual-pool + people, who then receive NON_CITIZEN_VALID_EAD SSN cards (both programs confer employment authorization), + consistent with Pew''s own accounting of TPS/parolees inside its unauthorized totals. The EAD worker/student + spill then binds the remaining residual pool to Pew''s 8.3M workers and the Higher Ed Immigration + Portal''s ~408k students — still the only forced margins. The total undocumented population is emergent + and gated against Pew''s 11.0M 2022 anchor with a coarse band; per-category humanitarian masses are + gated against their cited targets with a coarse band (the ASEC undercovers 2022-24 arrivals — the + Ukraine parole pool saturates below its admin count); reconciling levels is the calibration lane''s + job. DACA (arrival cohort among EAD holders) and CUBAN_HAITIAN_ENTRANT (nativity plus post-1980 arrival) + keep their statutory tests; other documented non-citizens stay LEGAL_PERMANENT_RESIDENT, the modal + true status.' - stage: hours_worked survey: Census CPS ASEC source: https://www.census.gov/programs-surveys/cps.html diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/spine.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/spine.yaml index 42e100f8c2..0fc8b8d348 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/spine.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/spine.yaml @@ -3,7 +3,7 @@ pipeline_contract: artifact_protocol: artifact_kind: populace_us_stacked_pool_checkpoint_identity schema_version: 1 - materializer_version: 12 + materializer_version: 13 pipeline: us-stacked-pool stacked_operator_order: - assemble_stacked_spine @@ -17,6 +17,7 @@ pipeline_contract: - materialize_multispine_agreement_outputs - stacked_completeness_gate - by_origin_battery + - us_immigration_composition_gate pre_clone_source_operator_order: - derive_us_cps_carried_inputs - with_us_hours_worked_inputs diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py index 35dc9b7fa9..e0ecf9f709 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py @@ -382,8 +382,12 @@ ImmigrationControls, UndocumentedControls, derive_us_immigration_status_from_manifest, + reconcile_us_immigration_humanitarian_transfer, us_immigration_composition_gate, us_immigration_composition_summary, + us_immigration_controls, + us_immigration_evidence_features, + us_immigration_humanitarian_draw_mask, us_immigration_stage_spec, with_us_immigration_inputs, ) @@ -1765,8 +1769,12 @@ "us_prior_year_income_summary", "with_us_prior_year_income_inputs", "derive_us_immigration_status_from_manifest", + "reconcile_us_immigration_humanitarian_transfer", "us_immigration_composition_gate", "us_immigration_composition_summary", + "us_immigration_controls", + "us_immigration_evidence_features", + "us_immigration_humanitarian_draw_mask", "us_immigration_stage_spec", "with_us_immigration_inputs", "US_TAKE_UP_SHARE_BAND", diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_pums.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_pums.py index dd0456fca7..423f6a04e2 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_pums.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_pums.py @@ -83,6 +83,9 @@ "SSIP", "RETP", "INTP", + "CIT", + "POBP", + "YOEP", "PWGTP", ) _PERSON_OPTIONAL: tuple[str, ...] = () diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_transfer.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_transfer.py index f791823a0f..b74d7a0b5f 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_transfer.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_transfer.py @@ -31,9 +31,14 @@ from microcosm.build.gates import FitWeightRecord from microcosm.build.serialization_dtypes import canonicalize_frame_string_dtypes +from microcosm.build.us_runtime.acs_pums import ACS_2024_1YR_VINTAGE from microcosm.build.us_runtime.capital_gain_distributions import ( capital_gain_distribution_shares_asset_identity, ) +from microcosm.build.us_runtime.immigration import ( + reconcile_us_immigration_humanitarian_transfer, + us_immigration_evidence_features, +) from microcosm.build.us_runtime.puf_support import ( PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS, PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS, @@ -142,6 +147,32 @@ "immigration_status_str", ) _IMMIGRATION_STATUS_MODEL_TARGET = "__acs_transfer_immigration_status_pair" +_IMMIGRATION_CITIZENSHIP_FEATURE = "__acs_transfer_is_us_citizen" +_IMMIGRATION_ORIGIN_FEATURE = "__acs_transfer_birth_country_code" +_IMMIGRATION_ARRIVAL_FEATURE = "__acs_transfer_arrival_year" +_IMMIGRATION_REQUIRED_FEATURES = ( + _IMMIGRATION_CITIZENSHIP_FEATURE, + _IMMIGRATION_ORIGIN_FEATURE, + _IMMIGRATION_ARRIVAL_FEATURE, +) +_HUMANITARIAN_IMMIGRATION_STATUSES = frozenset( + { + "PAROLED_ONE_YEAR", + "REFUGEE", + "ASYLEE", + "DEPORTATION_WITHHELD", + "TPS", + } +) +_EVIDENCE_DERIVED_IMMIGRATION_STATUSES = frozenset( + { + "CUBAN_HAITIAN_ENTRANT", + "DACA", + } +) +_CONSTRAINED_IMMIGRATION_STATUSES = ( + _HUMANITARIAN_IMMIGRATION_STATUSES | _EVIDENCE_DERIVED_IMMIGRATION_STATUSES +) # RNTP/GRNTP are not pre-subsidy rent, so this leaf remains donor-transferable. _ADDITIONAL_PERSON_TRANSFER_TARGETS = ("pre_subsidy_rent",) @@ -241,8 +272,9 @@ def acs_transfer_execution_contract_identity( ) adult_care_enabled = _ADULT_CARE_EXPENSE in requested_targets payload: dict[str, object] = { - "schema_version": 2, + "schema_version": 3, "person_required_predictors": list(ACS_PERSON_TRANSFER_PREDICTORS), + "immigration_required_predictors": list(_IMMIGRATION_REQUIRED_FEATURES), "person_optional_predictors": list(ACS_OPTIONAL_PERSON_TRANSFER_PREDICTORS), "group_required_predictors": list(ACS_GROUP_TRANSFER_PREDICTORS), "group_optional_names": dict(sorted(_GROUP_OPTIONAL_NAMES.items())), @@ -267,6 +299,9 @@ def acs_transfer_execution_contract_identity( "tenure_codes": dict(sorted(_TENURE_CODES.items())), "immigration_status_targets": list(_IMMIGRATION_STATUS_TARGETS), "immigration_status_model_target": _IMMIGRATION_STATUS_MODEL_TARGET, + "immigration_baseline_excluded_statuses": sorted( + _CONSTRAINED_IMMIGRATION_STATUSES + ), "discrete_numeric_targets": sorted(_DISCRETE_NUMERIC_TARGETS), "post_transfer_structure": { "schedule_d_capital_gain_distributions": { @@ -289,6 +324,14 @@ def acs_transfer_execution_contract_identity( "tax_unit_link": _ADULT_CARE_UNIT, "mutable_rows": "newly_imputed_expense_cells_only", }, + "humanitarian_immigration": { + "enabled": set(_IMMIGRATION_STATUS_TARGETS).issubset(requested_targets), + "targets": list(_IMMIGRATION_STATUS_TARGETS), + "mutable_rows": "newly_imputed_paired_cells_only", + "target_basis": "manifest_residual_after_immutable_asec_mass", + "selection": "stable_person_hash_in_manifest_draw_order", + "candidate_shortfall": "error", + }, }, } payload["sha256"] = hashlib.sha256( @@ -443,7 +486,7 @@ class AcsImputedInput: derivation: str | None = None #: Post-fit structural reconciliation counts, when the column's surface #: was adjusted to a statute contract after prediction. - reconciliation: Mapping[str, int] | None = None + reconciliation: Mapping[str, object] | None = None @dataclass(frozen=True) @@ -814,6 +857,37 @@ def acs_transfer_donor_requirements( if all(component in person.columns for component in components): required[donor.schema.person_entity].update(components) + person_targets = { + target + for targets in target_families.get(donor.schema.person_entity, {}).values() + for target in targets + } + if set(_IMMIGRATION_STATUS_TARGETS).issubset(person_targets): + evidence_triplets = ( + ("PRCITSHP", "PENATVTY", "PEINUSYR"), + ("CIT", "POBP", "YOEP"), + ) + active_triplets: list[tuple[str, str, str]] = [] + for triplet in evidence_triplets: + citizenship = triplet[0] + if citizenship not in person or not person[citizenship].notna().any(): + continue + missing = sorted(set(triplet) - set(person.columns)) + if missing: + raise ValueError( + "paired immigration targets have an incomplete donor " + f"evidence triplet {triplet!r}; missing={missing}." + ) + active_triplets.append(triplet) + if not active_triplets: + raise ValueError( + "paired immigration targets require a live ASEC " + "PRCITSHP/PENATVTY/PEINUSYR or ACS CIT/POBP/YOEP donor " + "evidence triplet." + ) + for triplet in active_triplets: + required[donor.schema.person_entity].update(triplet) + housing = bool(_HOUSING_TRANSFER_TARGETS.intersection(target_names)) head_source = next( ( @@ -1137,10 +1211,12 @@ def transfer_acs_inputs( _apply_post_transfer_structure( output_tables, provenance, + recipient=recipient, imputed_masks=imputed_masks, donor_spine=donor_spine, resolved_channel=resolved_channel, execution_contract=resolved_execution_contract, + seed=seed, ) tables: dict[str, pd.DataFrame] = dict(output_tables) @@ -1174,26 +1250,67 @@ def _apply_post_transfer_structure( output_tables: dict[str, pd.DataFrame], provenance: list[AcsImputedInput], *, + recipient: Frame, imputed_masks: Mapping[tuple[str, str], np.ndarray], donor_spine: str, resolved_channel: str | None, execution_contract: Mapping[str, object], + seed: int, ) -> None: """Apply the deterministic post-fit steps the base's construction implies. - Both steps key off cells THIS transfer filled, so custom test plans that + All steps key off cells THIS transfer filled, so custom test plans that never touch these families are unaffected: - The Schedule D CGD memo leg is derived from the two transferred capital-gain parents at the packaged share with route exclusivity. - The adult-care expense surface is reconciled to the statute structure (qualifying carriers only, at most one per tax unit). + - The paired immigration baseline is reconciled to manifest humanitarian + residual targets after the immutable ASEC contribution. """ person = output_tables.get("person") if person is None: return + immigration_masks = [ + imputed_masks.get(("person", target)) for target in _IMMIGRATION_STATUS_TARGETS + ] + if all(mask is not None for mask in immigration_masks): + assert immigration_masks[0] is not None + assert immigration_masks[1] is not None + if not np.array_equal(immigration_masks[0], immigration_masks[1]): + raise ValueError( + "ACS immigration transfer must impute its paired target cells " + "on exactly the same recipient rows." + ) + mutable_immigration = immigration_masks[0] + immigration_provenance = [ + (index, item) + for index, item in enumerate(provenance) + if item.entity == "person" and item.column in _IMMIGRATION_STATUS_TARGETS + ] + source_operator_family = any( + item.family.split("__batch_", 1)[0] == "source_operator_immigration" + for _, item in immigration_provenance + ) + if mutable_immigration.any() and source_operator_family: + reconciled, receipt = reconcile_us_immigration_humanitarian_transfer( + person, + weights=np.asarray( + recipient.resolve_weights("person").values, + dtype=np.float64, + ), + mutable_rows=mutable_immigration, + seed=seed, + time_period=ACS_2024_1YR_VINTAGE, + ) + for target in _IMMIGRATION_STATUS_TARGETS: + person[target] = reconciled[target] + for index, item in immigration_provenance: + provenance[index] = replace(item, reconciliation=receipt) + post_transfer_contract = execution_contract["post_transfer_structure"] assert isinstance(post_transfer_contract, Mapping) schedule_d_contract = post_transfer_contract[ @@ -2070,6 +2187,38 @@ def _transfer_feature_surface( else: surface = _group_feature_surface(donor, recipient, entity=entity) + if set(_IMMIGRATION_STATUS_TARGETS).issubset(targets): + if entity != donor.schema.person_entity: + raise ValueError( + "The paired immigration targets must be transferred on person." + ) + donor_evidence = us_immigration_evidence_features( + donor, + time_period=ACS_2024_1YR_VINTAGE, + ).rename( + columns={ + "is_us_citizen": _IMMIGRATION_CITIZENSHIP_FEATURE, + "birth_country_code": _IMMIGRATION_ORIGIN_FEATURE, + "arrival_year": _IMMIGRATION_ARRIVAL_FEATURE, + } + ) + recipient_evidence = us_immigration_evidence_features( + recipient, + time_period=ACS_2024_1YR_VINTAGE, + ).rename( + columns={ + "is_us_citizen": _IMMIGRATION_CITIZENSHIP_FEATURE, + "birth_country_code": _IMMIGRATION_ORIGIN_FEATURE, + "arrival_year": _IMMIGRATION_ARRIVAL_FEATURE, + } + ) + surface = _FeatureSurface( + donor=pd.concat([surface.donor, donor_evidence], axis=1), + recipient=pd.concat([surface.recipient, recipient_evidence], axis=1), + required=(*surface.required, *_IMMIGRATION_REQUIRED_FEATURES), + optional=surface.optional, + ) + housing = bool(_HOUSING_TRANSFER_TARGETS.intersection(targets)) if not housing: return surface @@ -2963,7 +3112,8 @@ def _target_encodings( result: dict[str, _TargetEncoding] = {} if immigration: pairs = list( - zip( + _baseline_immigration_pair(ssn, status) + for ssn, status in zip( table[_IMMIGRATION_STATUS_TARGETS[0]].to_numpy(dtype=object), table[_IMMIGRATION_STATUS_TARGETS[1]].to_numpy(dtype=object), strict=True, @@ -3010,6 +3160,18 @@ def _target_encodings( return result +def _baseline_immigration_pair(ssn: object, status: object) -> tuple[object, object]: + """Remove evidence-constrained labels from the unconstrained codec.""" + + if status not in _CONSTRAINED_IMMIGRATION_STATUSES: + return ssn, status + if ssn == "CITIZEN": + return "CITIZEN", "CITIZEN" + if ssn == "NONE": + return "NONE", "UNDOCUMENTED" + return ssn, "LEGAL_PERMANENT_RESIDENT" + + def _complete_case_target_encodings( table: pd.DataFrame, *, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py b/packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py index 516dead3c5..707b8ed00a 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py @@ -76,6 +76,8 @@ US_MULTISPINE_AGREEMENT_DIAGNOSTICS_ARTIFACT_KIND = ( "populace_us_multispine_agreement_diagnostics" ) +# 10 binds the humanitarian-immigration composition gate into the stacked +# operator order and authenticated terminal-gate surface. # 9 binds the post-assembly household-geography assignment receipt and its # authenticated release-vintage authorities. # 8 binds the nullable-boolean-capable physical H5 materializer in both the @@ -86,7 +88,7 @@ # authority and restores its immutable Frame-metadata anchor on H5 load. # Schema 5 can authenticate the DAG receipt's structure, but cannot prove that # the published receipt is the one authorized by the generating transition. -US_MULTISPINE_POOL_MANIFEST_SCHEMA_VERSION = 9 +US_MULTISPINE_POOL_MANIFEST_SCHEMA_VERSION = 10 US_MULTISPINE_POOL_H5_MATERIALIZER_VERSION = 3 """Version 3 atomically binds release CD provenance attrs; v2 added BooleanDtype.""" _LEGACY_MULTISPINE_POOL_MANIFEST_SCHEMA_VERSION = 4 @@ -106,6 +108,14 @@ "materialize_multispine_agreement_outputs", "stacked_completeness_gate", "by_origin_battery", + "us_immigration_composition_gate", +) +_STACKED_TERMINAL_GATE_NAMES = frozenset( + { + "us_stacked_completeness", + "us_by_origin_battery", + "immigration_composition", + } ) _LEGACY_POOL_OPERATOR_ORDER = ( "assemble", @@ -461,10 +471,10 @@ def us_multispine_pool_release_receipt( ) gate_passed = gate.get("passed") gate_failures = gate.get("failures") - if type(gate_passed) is not bool or not isinstance( - gate_failures, list - ) or not all( - isinstance(failure, str) for failure in gate_failures + if ( + type(gate_passed) is not bool + or not isinstance(gate_failures, list) + or not all(isinstance(failure, str) for failure in gate_failures) ): raise ValueError( f"Authenticated US multispine pool gate {gate_name!r} has an " @@ -803,7 +813,7 @@ def _validate_stacked_late_dag_manifest_binding( *, manifest_path: Path, ) -> None: - """Make schema-9 consumers authenticate geography and late-DAG proofs.""" + """Make current-schema consumers authenticate geography and late-DAG proofs.""" if manifest.get("pipeline") != "us-stacked-pool": return @@ -1228,7 +1238,7 @@ def _validate_stacked_geography_h5_binding( manifest_path: Path, pool_path: Path, ) -> None: - """Bind schema-9 manifest geography claims to the authenticated H5.""" + """Bind current-schema manifest geography claims to the authenticated H5.""" if manifest.get("pipeline") != _STACKED_PIPELINE: return @@ -2059,6 +2069,24 @@ def _require_matching_terminal_gate_aliases( f"US stacked pool diagnostics {diagnostics_path} terminal_gates do " "not match agreement_gate." ) + manifest_gates = _mapping( + manifest_terminal_gates.get("gates"), + label=f"US stacked pool manifest {manifest_path}.terminal_gates.gates", + ) + diagnostics_gates = _mapping( + diagnostics_terminal_gates.get("gates"), + label=(f"US stacked pool diagnostics {diagnostics_path}.terminal_gates.gates"), + ) + for label, gates in ( + (f"US stacked pool manifest {manifest_path}", manifest_gates), + (f"US stacked pool diagnostics {diagnostics_path}", diagnostics_gates), + ): + if set(gates) != _STACKED_TERMINAL_GATE_NAMES: + raise ValueError( + f"{label} does not carry the canonical terminal gate set; " + f"expected={sorted(_STACKED_TERMINAL_GATE_NAMES)}, " + f"observed={sorted(gates)}." + ) def _publication_run_id(value: object, *, label: str) -> str: diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py b/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py index 151ad3ebc7..3ad55226d2 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py @@ -70,7 +70,8 @@ class is the better statutory label and keeps H.R.1 eligibility), as is status). Selection draws are seeded blake2b hashes keyed by the person's stable source -identity (``source_year``/``source_person_id`` when present), so +identity (``source_year``/``source_household_id``/``source_person_id`` when +present), so support-channel clones of one source person always receive the same status and reruns are bit-reproducible without global RNG state. @@ -117,8 +118,13 @@ class is the better statutory label and keeps H.R.1 eligibility), as is "ImmigrationControls", "UndocumentedControls", "derive_us_immigration_status_from_manifest", + "reconcile_us_immigration_humanitarian_transfer", "us_immigration_composition_gate", "us_immigration_composition_summary", + "us_immigration_controls", + "us_immigration_evidence_features", + "us_immigration_humanitarian_draw_mask", + "us_immigration_humanitarian_transfer_selection_masks", "us_immigration_stage_spec", "with_us_immigration_inputs", ] @@ -239,7 +245,6 @@ class is the better statutory label and keeps H.R.1 eligibility), as is 26: 2019, 27: 2021, 28: 2023, - 29: 2024, } #: PENATVTY codes for Cuba and Haiti; the Cuban/Haitian entrant class exists @@ -272,11 +277,23 @@ class is the better statutory label and keeps H.R.1 eligibility), as is "tps": "TPS", } -#: PEINUSYR codes for 2020+ arrivals (2024 ASEC data dictionary: 27 = -#: 2020-2021, 28 = 2022-2024) — the humanitarian-parole and recent-refugee -#: cohorts. The Census codebook rebins this variable across vintages, so -#: these windows are survey-vintage facts, not statutory dates. +#: PEINUSYR codes for 2020+ arrivals. The 2024 ASEC distinguishes 2020-2021 +#: (27) from 2022-2024 (28). Program-specific rules below do not treat those +#: windows as interchangeable: U4U and CHNV cannot draw from code 27, while +#: Operation Allies Welcome can. _RECENT_ARRIVAL_CODES = (27, 28) +_PAROLE_ASEC_ARRIVAL_CODES: Mapping[str, tuple[int, ...]] = { + "afghanistan": _RECENT_ARRIVAL_CODES, + "ukraine": (28,), + "nicaragua": (28,), + "venezuela": (28,), +} +_PAROLE_ACS_MIN_ARRIVAL_YEAR: Mapping[str, int] = { + "afghanistan": 2021, + "ukraine": 2022, + "nicaragua": 2023, + "venezuela": 2022, +} #: Asylum grants lag arrival by filing queues plus court backlog; grantees #: in 2022-2024 overwhelmingly arrived 2016+ (OHSS asylee flow reports). _ASYLEE_ARRIVAL_CODES = (25, 26, 27, 28) @@ -356,6 +373,27 @@ class is the better statutory label and keeps H.R.1 eligibility), as is "other_designated": ((205, 239, 248, 224, 407, 416, 448, 451), 28), } +#: ACS YOEP preserves an exact year, unlike the ASEC arrival bins. Use that +#: extra information for continuous-residence compatibility rather than +#: deliberately coarsening YOEP back to PEINUSYR. The per-country cutoffs are +#: calendar-year approximations of the designation dates; ASEC retains the +#: published coarse-bin contract above. +_TPS_ACS_MAX_ARRIVAL_YEAR_BY_BIRTH: Mapping[int, int] = { + 373: 2023, # Venezuela (2021 and 2023 designations) + 312: 2001, # El Salvador + 314: 1998, # Honduras + 315: 1998, # Nicaragua + 229: 2015, # Nepal + 205: 2024, # Burma + 239: 2024, # Syria + 248: 2024, # Yemen + 224: 2024, # Lebanon + 407: 2023, # Cameroon + 416: 2024, # Ethiopia + 448: 2024, # Somalia + 451: 2023, # Sudan / South Sudan (shared survey code) +} + #: Categories whose flat manifest block carries one national target. _FLAT_HUMANITARIAN_CATEGORIES = ("refugee", "asylee", "deportation_withheld") #: Categories whose manifest block carries per-origin targets, and the @@ -483,6 +521,20 @@ def humanitarian_target(self, category: str) -> float: ) +@dataclass(frozen=True) +class _ImmigrationEvidenceProfile: + """Canonical immigration evidence shared by ASEC and ACS rows.""" + + source_is_acs: np.ndarray + is_citizen: np.ndarray + birth_country: np.ndarray + arrival_code: np.ndarray + arrival_year: np.ndarray + age: np.ndarray + age_at_entry: np.ndarray + tps_acs_max_arrival_year: np.ndarray + + def us_immigration_stage_spec() -> SourceStageSpec: """Load the packaged ``immigration_status`` source-stage manifest entry.""" @@ -716,6 +768,22 @@ def _controls_from_parameters(params: Mapping[str, object]) -> ImmigrationContro ) +def us_immigration_controls() -> ImmigrationControls: + """Return the controls bound to the packaged immigration manifest stage.""" + + derive = [ + operation + for operation in us_immigration_stage_spec().operations + if operation.kind == "derive_immigration_status" + ] + if len(derive) != 1: + raise ValueError( + "US immigration stage must declare exactly one " + "derive_immigration_status operation." + ) + return _controls_from_parameters(derive[0].parameters) + + def _integer_column(person: pd.DataFrame, column: str) -> np.ndarray: return ( pd.to_numeric(person[column], errors="coerce") @@ -732,23 +800,199 @@ def _float_column(person: pd.DataFrame, column: str) -> np.ndarray: ) +def _numeric_evidence_column(person: pd.DataFrame, column: str) -> np.ndarray: + if column not in person.columns: + return np.full(len(person), np.nan, dtype=np.float64) + return pd.to_numeric(person[column], errors="coerce").to_numpy( + dtype=np.float64, + na_value=np.nan, + ) + + +def _source_aware_immigration_profile( + person: pd.DataFrame, + *, + time_period: int, +) -> _ImmigrationEvidenceProfile: + """Normalize ASEC PRCITSHP/PENATVTY/PEINUSYR and ACS CIT/POBP/YOEP. + + A stacked recipient carries the union of both raw survey schemas, with the + non-owning source null on each row. Exactly one citizenship source must be + present. Relevant POBP country codes share the Census country codebook with + PENATVTY, so the normalized origin is lossless; ACS YOEP retains its exact + year while ASEC necessarily uses the documented interval midpoint. + """ + + if isinstance(time_period, bool) or int(time_period) != time_period: + raise ValueError( + f"US immigration time_period must be an integer, got {time_period!r}." + ) + time_period = int(time_period) + asec_citizenship = _numeric_evidence_column(person, "PRCITSHP") + acs_citizenship = _numeric_evidence_column(person, "CIT") + has_asec = np.isfinite(asec_citizenship) + has_acs = np.isfinite(acs_citizenship) + ambiguous = has_asec & has_acs + missing = ~has_asec & ~has_acs + if ambiguous.any() or missing.any(): + raise SourceRuntimeError( + "US immigration evidence requires exactly one row-level citizenship " + "source (ASEC PRCITSHP or ACS CIT); " + f"ambiguous_rows={int(ambiguous.sum())}, missing_rows={int(missing.sum())}." + ) + + for label, values, present in ( + ("PRCITSHP", asec_citizenship, has_asec), + ("CIT", acs_citizenship, has_acs), + ): + invalid = present & ( + ~np.equal(values, np.rint(values)) | ~np.isin(values, [1, 2, 3, 4, 5]) + ) + if invalid.any(): + bad = sorted(set(values[invalid].tolist()))[:5] + raise SourceRuntimeError( + f"{label} carries value(s) outside the Census domain 1..5: {bad}." + ) + + citizenship = np.where(has_acs, acs_citizenship, asec_citizenship).astype(np.int64) + is_citizen = np.isin(citizenship, [1, 2, 3, 4]) + + asec_birth = _numeric_evidence_column(person, "PENATVTY") + acs_birth = _numeric_evidence_column(person, "POBP") + birth = np.where(has_acs, acs_birth, asec_birth) + invalid_birth = ~np.isfinite(birth) | ~np.equal(birth, np.rint(birth)) | (birth < 1) + if invalid_birth.any(): + raise SourceRuntimeError( + "US immigration evidence requires a positive integral PENATVTY/POBP " + f"country code on every row; invalid_rows={int(invalid_birth.sum())}." + ) + birth_country = birth.astype(np.int64) + + arrival_code_values = _numeric_evidence_column(person, "PEINUSYR") + invalid_asec_arrival = has_asec & ( + ~np.isfinite(arrival_code_values) + | ~np.equal(arrival_code_values, np.rint(arrival_code_values)) + | (arrival_code_values < 0) + | (arrival_code_values > max(_ARRIVAL_YEAR_MIDPOINTS)) + ) + if invalid_asec_arrival.any(): + bad = sorted(set(arrival_code_values[invalid_asec_arrival].tolist()))[:5] + raise SourceRuntimeError( + "PEINUSYR carries value(s) outside the 2024 ASEC domain " + f"0..{max(_ARRIVAL_YEAR_MIDPOINTS)}: {bad}." + ) + arrival_code = np.zeros(len(person), dtype=np.int64) + arrival_code[has_asec] = arrival_code_values[has_asec].astype(np.int64) + arrival_year = np.full(len(person), time_period, dtype=np.int64) + for code, midpoint in _ARRIVAL_YEAR_MIDPOINTS.items(): + arrival_year[has_asec & (arrival_code == code)] = midpoint + + acs_arrival = _numeric_evidence_column(person, "YOEP") + acs_foreign_born = has_acs & np.isin(citizenship, [4, 5]) + invalid_acs_arrival = acs_foreign_born & ( + ~np.isfinite(acs_arrival) + | ~np.equal(acs_arrival, np.rint(acs_arrival)) + | (acs_arrival < 1900) + | (acs_arrival > time_period) + ) + if invalid_acs_arrival.any(): + raise SourceRuntimeError( + "ACS YOEP must be an integral 1900..time_period year for every " + f"foreign-born CIT=4/5 row; invalid_rows={int(invalid_acs_arrival.sum())}." + ) + observed_acs_arrival = has_acs & np.isfinite(acs_arrival) + arrival_year[observed_acs_arrival] = acs_arrival[observed_acs_arrival].astype( + np.int64 + ) + + age_values = _numeric_evidence_column(person, "A_AGE") + if "age" in person.columns: + mapped_age = _numeric_evidence_column(person, "age") + age_values = np.where(np.isfinite(age_values), age_values, mapped_age) + invalid_age = ( + ~np.isfinite(age_values) + | ~np.equal(age_values, np.rint(age_values)) + | (age_values < 0) + ) + if invalid_age.any(): + raise SourceRuntimeError( + "US immigration evidence requires a finite non-negative integral " + f"A_AGE/age on every row; invalid_rows={int(invalid_age.sum())}." + ) + age = age_values.astype(np.int64) + age_at_entry = np.maximum(0, age - (time_period - arrival_year)) + tps_acs_max_arrival_year = np.full(len(person), -1, dtype=np.int64) + for birth_code, cutoff in _TPS_ACS_MAX_ARRIVAL_YEAR_BY_BIRTH.items(): + tps_acs_max_arrival_year[birth_country == birth_code] = cutoff + return _ImmigrationEvidenceProfile( + source_is_acs=has_acs, + is_citizen=is_citizen, + birth_country=birth_country, + arrival_code=arrival_code, + arrival_year=arrival_year, + age=age, + age_at_entry=age_at_entry, + tps_acs_max_arrival_year=tps_acs_max_arrival_year, + ) + + +def us_immigration_evidence_features( + frame: Frame, + *, + time_period: int = 2024, +) -> pd.DataFrame: + """Return finite, source-harmonized immigration predictors per person.""" + + person = frame.table(frame.schema.person_entity) + profile = _source_aware_immigration_profile(person, time_period=time_period) + return pd.DataFrame( + { + "is_us_citizen": profile.is_citizen.astype(np.float64), + "birth_country_code": profile.birth_country.astype(np.float64), + "arrival_year": profile.arrival_year.astype(np.float64), + }, + index=person.index, + ) + + def _stable_person_draws(person: pd.DataFrame, *, seed: int, salt: str) -> np.ndarray: """Deterministic uniform draws keyed by stable person identity. Support-channel clones carry their source person's ``source_year`` / - ``source_person_id``, so keying on those gives every clone of one source - person the same draw; frames without source ids fall back to - ``person_id``. + ``source_household_id`` / ``source_person_id``, so keying on those gives + every clone of one source person the same draw; frames without full source + ids fall back to the legacy source pair or ``person_id``. """ - if {"source_year", "source_person_id"}.issubset(person.columns): + def complete(columns: tuple[str, ...]) -> bool: + return set(columns).issubset(person.columns) and bool( + person.loc[:, list(columns)].notna().to_numpy(dtype=bool).all() + ) + + full_lineage = ("source_year", "source_household_id", "source_person_id") + legacy_lineage = ("source_year", "source_person_id") + if complete(full_lineage): keys = ( person["source_year"].astype(str) + ":" + + person["source_household_id"].astype(str) + + ":" + person["source_person_id"].astype(str) ) - else: + elif complete(legacy_lineage): + keys = ( + person["source_year"].astype(str) + + ":" + + person["source_person_id"].astype(str) + ) + elif complete(("person_id",)): keys = person["person_id"].astype(str) + else: + raise SourceRuntimeError( + "US immigration deterministic draws require one complete stable " + "person-lineage alternative: source_year/source_household_id/" + "source_person_id, source_year/source_person_id, or person_id." + ) denominator = float(2**64) return np.fromiter( ( @@ -829,12 +1073,42 @@ def _daca_statutory_cohort( ) +def _special_status_masks( + *, + ssn_codes: np.ndarray, + birth_country: np.ndarray, + arrival_year: np.ndarray, + age_at_entry: np.ndarray, + age: np.ndarray, +) -> tuple[np.ndarray, np.ndarray]: + """Return final Cuban/Haitian-entrant and DACA cohort masks. + + The source derivation applies DACA after the Cuban/Haitian entrant rule, + so a qualifying EAD holder in both cohorts retains DACA. Keeping that + precedence in one helper lets source derivation, recipient reconciliation, + and release validation share the same exact contract. + """ + + documented_noncitizen = np.isin(ssn_codes, [2, 3]) + daca = (ssn_codes == 2) & _daca_statutory_cohort( + arrival_year, + age_at_entry, + age, + ) + cuban_haitian = ( + documented_noncitizen + & np.isin(birth_country, _CUBAN_HAITIAN_BIRTH_CODES) + & (arrival_year >= _CUBAN_HAITIAN_ARRIVAL_CUTOFF) + & ~daca + ) + return cuban_haitian, daca + + def _humanitarian_draw_candidates( draw: HumanitarianDraw, *, ssn_codes: np.ndarray, - birth_country: np.ndarray, - arrival_code: np.ndarray, + profile: _ImmigrationEvidenceProfile, ) -> np.ndarray: """Candidate mask for one draw, before exclusions shared by all draws. @@ -845,45 +1119,560 @@ def _humanitarian_draw_candidates( (code 0), whose members then receive EAD-backed SSN cards. """ + noncitizen = ~profile.is_citizen + excluded = np.isin(profile.birth_country, _CUBAN_HAITIAN_BIRTH_CODES) + excluded |= _daca_statutory_cohort( + profile.arrival_year, + profile.age_at_entry, + profile.age, + ) + if draw.category == "paroled_one_year": - assert draw.origin is not None - return ( - np.isin(ssn_codes, (0, 3)) - & np.isin(birth_country, _PAROLE_ORIGIN_CODES[draw.origin]) - & np.isin(arrival_code, _RECENT_ARRIVAL_CODES) + origins = ( + (draw.origin,) if draw.origin is not None else tuple(_PAROLE_ORIGIN_CODES) ) + cohort = np.zeros(len(ssn_codes), dtype=bool) + for origin in origins: + if origin not in _PAROLE_ORIGIN_CODES: + raise SourceRuntimeError( + f"US immigration has no parole origin rule for {origin!r}." + ) + origin_match = np.isin( + profile.birth_country, + _PAROLE_ORIGIN_CODES[origin], + ) + asec_window = (~profile.source_is_acs) & np.isin( + profile.arrival_code, + _PAROLE_ASEC_ARRIVAL_CODES[origin], + ) + acs_window = profile.source_is_acs & ( + profile.arrival_year >= _PAROLE_ACS_MIN_ARRIVAL_YEAR[origin] + ) + cohort |= origin_match & (asec_window | acs_window) + return noncitizen & np.isin(ssn_codes, (0, 3)) & cohort & ~excluded if draw.category == "refugee": return ( - (ssn_codes == 3) - & np.isin(birth_country, _REFUGEE_ORIGIN_CODES) - & np.isin(arrival_code, _RECENT_ARRIVAL_CODES) + noncitizen + & (ssn_codes == 3) + & np.isin(profile.birth_country, _REFUGEE_ORIGIN_CODES) + & ( + ((~profile.source_is_acs) & (profile.arrival_code == 28)) + | (profile.source_is_acs & (profile.arrival_year >= 2022)) + ) + & ~excluded ) if draw.category == "asylee": return ( - (ssn_codes == 3) - & np.isin(birth_country, _ASYLEE_ORIGIN_CODES) - & np.isin(arrival_code, _ASYLEE_ARRIVAL_CODES) + noncitizen + & (ssn_codes == 3) + & np.isin(profile.birth_country, _ASYLEE_ORIGIN_CODES) + & ( + ( + (~profile.source_is_acs) + & np.isin(profile.arrival_code, _ASYLEE_ARRIVAL_CODES) + ) + | (profile.source_is_acs & (profile.arrival_year >= 2016)) + ) + & ~excluded ) if draw.category == "deportation_withheld": return ( - (ssn_codes == 3) - & (arrival_code >= 1) - & (arrival_code <= _WITHHELD_MAX_ARRIVAL_CODE) + noncitizen + & (ssn_codes == 3) + & ( + ( + (~profile.source_is_acs) + & (profile.arrival_code >= 1) + & (profile.arrival_code <= _WITHHELD_MAX_ARRIVAL_CODE) + ) + | (profile.source_is_acs & (profile.arrival_year <= 2019)) + ) + & ~excluded ) if draw.category == "tps": - assert draw.origin is not None - codes, max_arrival_code = _TPS_ORIGIN_CODES[draw.origin] - return ( - np.isin(ssn_codes, (0, 3)) - & np.isin(birth_country, codes) - & (arrival_code >= 1) - & (arrival_code <= max_arrival_code) + origins = ( + (draw.origin,) if draw.origin is not None else tuple(_TPS_ORIGIN_CODES) ) + cohort = np.zeros(len(ssn_codes), dtype=bool) + for origin in origins: + if origin not in _TPS_ORIGIN_CODES: + raise SourceRuntimeError( + f"US immigration has no TPS origin rule for {origin!r}." + ) + codes, max_arrival_code = _TPS_ORIGIN_CODES[origin] + origin_match = np.isin(profile.birth_country, codes) + asec_window = ( + (~profile.source_is_acs) + & (profile.arrival_code >= 1) + & (profile.arrival_code <= max_arrival_code) + ) + acs_window = ( + profile.source_is_acs + & (profile.tps_acs_max_arrival_year >= 0) + & (profile.arrival_year <= profile.tps_acs_max_arrival_year) + ) + cohort |= origin_match & (asec_window | acs_window) + return noncitizen & np.isin(ssn_codes, (0, 3)) & cohort & ~excluded raise SourceRuntimeError( f"US immigration derivation has no candidate rule for draw {draw.label!r}." ) +def us_immigration_humanitarian_draw_mask( + frame: Frame, + draw: HumanitarianDraw, + *, + time_period: int = 2024, +) -> np.ndarray: + """Rows emitted as ``draw`` whose source evidence supports that label. + + This is the shared hard-cohort contract for transfer reconciliation, + calibration, and the release gate. It intentionally includes the emitted + status test: callers receive the achieved population for one manifest + draw, not the larger pool of possible candidates. + """ + + person = frame.table(frame.schema.person_entity) + missing = sorted(set(US_IMMIGRATION_OUTPUT_COLUMNS) - set(person.columns)) + if missing: + raise ValueError( + f"US humanitarian draw compatibility requires output column(s) {missing}." + ) + profile = _source_aware_immigration_profile(person, time_period=time_period) + return _humanitarian_emitted_mask(person, draw=draw, profile=profile) + + +def us_immigration_humanitarian_transfer_selection_masks( + frame: Frame, + *, + mutable_rows: np.ndarray, + seed: int, + time_period: int = 2024, + controls: ImmigrationControls | None = None, +) -> dict[str, np.ndarray]: + """Replay the exact deterministic mutable-row selection for every draw. + + The post-transfer frame retains everything needed to reconstruct the + reconciliation decision: source evidence, final SSN codes, stable person + lineage, resolved person weights, manifest controls, and draw order. For + parole and TPS, final EAD codes are mapped back to the residual-pool code + exactly as they are during reconciliation. Earlier expected selections, + rather than observed status labels, exclude rows from later draws, so a + downstream equal-mass status swap cannot redefine the replayed candidate + surface. + """ + + person = frame.table(frame.schema.person_entity) + missing = sorted(set(US_IMMIGRATION_OUTPUT_COLUMNS) - set(person.columns)) + if missing: + raise ValueError( + "Humanitarian transfer selection replay requires person column(s) " + f"{missing}." + ) + mutable = np.asarray(mutable_rows, dtype=bool) + if mutable.shape != (len(person),): + raise ValueError( + "Humanitarian transfer selection replay mutable_rows must align " + "one-to-one with the person table." + ) + weights = np.asarray( + frame.resolve_weights(frame.schema.person_entity).values, + dtype=np.float64, + ) + if ( + weights.shape != (len(person),) + or not np.isfinite(weights).all() + or (weights < 0).any() + ): + raise ValueError( + "Humanitarian transfer selection replay requires finite " + "non-negative person weights." + ) + controls = us_immigration_controls() if controls is None else controls + profile = _source_aware_immigration_profile(person, time_period=time_period) + ssn_codes = _ssn_name_codes(person) + immutable = ~mutable + selected_once = np.zeros(len(person), dtype=bool) + selections: dict[str, np.ndarray] = {} + for draw in controls.humanitarian: + immutable_draw = ( + _humanitarian_emitted_mask( + person, + draw=draw, + profile=profile, + ) + & immutable + ) + residual_target = max( + 0.0, + float(draw.target) - float(weights[immutable_draw].sum()), + ) + candidate_ssn = ssn_codes + if draw.category in {"paroled_one_year", "tps"}: + candidate_ssn = np.where(candidate_ssn == 2, 0, candidate_ssn) + candidates = ( + _humanitarian_draw_candidates( + draw, + ssn_codes=candidate_ssn, + profile=profile, + ) + & mutable + & ~selected_once + ) + stable_draws = _stable_person_draws( + person, + seed=seed, + salt=f"acs_transfer:{draw.salt}", + ) + selected = _select_weight_to_target( + candidates, + weights, + stable_draws, + residual_target, + ) + selections[draw.label] = selected + selected_once |= selected + return selections + + +def _ssn_name_codes(person: pd.DataFrame) -> np.ndarray: + ssn_lookup = { + "NONE": 0, + "CITIZEN": 1, + "NON_CITIZEN_VALID_EAD": 2, + "OTHER_NON_CITIZEN": 3, + } + return ( + person["ssn_card_type"] + .astype(str) + .map(ssn_lookup) + .fillna(-1) + .to_numpy(dtype=np.int64) + ) + + +def _humanitarian_emitted_mask( + person: pd.DataFrame, + *, + draw: HumanitarianDraw, + profile: _ImmigrationEvidenceProfile, +) -> np.ndarray: + ssn_codes = _ssn_name_codes(person) + candidates = _humanitarian_draw_candidates( + draw, + ssn_codes=ssn_codes, + profile=profile, + ) + # Parole and TPS selections from the residual pool are upgraded from NONE + # to EAD after candidacy is evaluated. Their emitted rows may therefore + # carry code 2 even though only code 0/3 was eligible before selection. + if draw.category in {"paroled_one_year", "tps"}: + candidates |= _humanitarian_draw_candidates( + draw, + ssn_codes=np.where(ssn_codes == 2, 0, ssn_codes), + profile=profile, + ) & (ssn_codes == 2) + status = person["immigration_status_str"].astype(str).to_numpy() + return np.asarray(candidates & (status == draw.status), dtype=bool) + + +def reconcile_us_immigration_humanitarian_transfer( + person: pd.DataFrame, + *, + weights: np.ndarray, + mutable_rows: np.ndarray, + seed: int, + time_period: int = 2024, + controls: ImmigrationControls | None = None, +) -> tuple[pd.DataFrame, dict[str, object]]: + """Reconcile a transferred immigration pair to manifest draw targets. + + Immutable rows are the ASEC source contribution. Only rows whose paired + target cells were imputed may be repaired or selected. Each manifest draw + receives the residual of its national target after its compatible, + immutable contribution, in manifest order; candidate exhaustion is a hard + error rather than permission to ship a wrong-origin label. + """ + + result = person.copy() + missing = sorted(set(US_IMMIGRATION_OUTPUT_COLUMNS) - set(result.columns)) + if missing: + raise ValueError( + f"Humanitarian transfer reconciliation requires person column(s) {missing}." + ) + weights = np.asarray(weights, dtype=np.float64) + mutable = np.asarray(mutable_rows, dtype=bool) + if weights.shape != (len(result),) or mutable.shape != (len(result),): + raise ValueError( + "Humanitarian transfer weights and mutable_rows must align one-to-one " + "with the person table." + ) + if not np.isfinite(weights).all() or (weights < 0).any(): + raise ValueError( + "Humanitarian transfer reconciliation requires finite non-negative " + "person weights." + ) + controls = us_immigration_controls() if controls is None else controls + profile = _source_aware_immigration_profile(result, time_period=time_period) + original_ssn = result["ssn_card_type"].astype(str).to_numpy(copy=True) + original_status = result["immigration_status_str"].astype(str).to_numpy(copy=True) + ssn = original_ssn.copy() + status = original_status.copy() + valid_ssn = np.isin(ssn, SSN_CARD_TYPE_VALUES) + valid_status = np.isin(status, IMMIGRATION_STATUS_VALUES) + if (~valid_ssn).any() or (~valid_status).any(): + raise ValueError( + "Humanitarian transfer reconciliation received values outside the " + "PolicyEngine SSN/immigration enum domains." + ) + + humanitarian_statuses = set(_HUMANITARIAN_STATUS_BY_CATEGORY.values()) + evidence_derived_statuses = {"CUBAN_HAITIAN_ENTRANT", "DACA"} + constrained_statuses = humanitarian_statuses | evidence_derived_statuses + baseline_constrained = mutable & np.isin(status, list(constrained_statuses)) + if baseline_constrained.any(): + raise ValueError( + "ACS joint-QRF baseline emitted evidence-constrained labels before " + "reconciliation." + ) + + # Source citizenship is observed on both surveys. Normalize only mutable + # pairs; immutable ASEC disagreements remain an error below. + citizen_mutable = mutable & profile.is_citizen + ssn[citizen_mutable] = "CITIZEN" + status[citizen_mutable] = "CITIZEN" + noncitizen_mutable = mutable & ~profile.is_citizen + ssn[noncitizen_mutable & (ssn == "CITIZEN")] = "OTHER_NON_CITIZEN" + status[noncitizen_mutable & (status == "CITIZEN")] = "LEGAL_PERMANENT_RESIDENT" + status[noncitizen_mutable & (ssn == "NONE")] = "UNDOCUMENTED" + status[noncitizen_mutable & (ssn != "NONE") & (status == "UNDOCUMENTED")] = ( + "LEGAL_PERMANENT_RESIDENT" + ) + citizenship_repairs = int( + ( + mutable + & ((ssn != original_ssn) | (status != original_status)) + & ( + profile.is_citizen + | (original_ssn == "CITIZEN") + | (original_status == "CITIZEN") + ) + ).sum() + ) + pair_repairs = int( + (mutable & ((ssn != original_ssn) | (status != original_status))).sum() + ) + result["ssn_card_type"] = ssn + result["immigration_status_str"] = status + + # CHE and DACA have hard source-evidence rules just like the manifest + # draws, but no stock target. Rebuild them deterministically after the + # unconstrained codec and citizenship/pair repair rather than accepting a + # donor label on an incompatible ACS cohort. + special_ssn_codes = _ssn_name_codes(result) + cuban_haitian, daca = _special_status_masks( + ssn_codes=special_ssn_codes, + birth_country=profile.birth_country, + arrival_year=profile.arrival_year, + age_at_entry=profile.age_at_entry, + age=profile.age, + ) + special_before = status.copy() + status[mutable & cuban_haitian] = "CUBAN_HAITIAN_ENTRANT" + status[mutable & daca] = "DACA" + special_status_assignments = int((mutable & (status != special_before)).sum()) + result["immigration_status_str"] = status + + immutable = ~mutable + immutable_pair_invalid = immutable & ( + ((ssn == "CITIZEN") != (status == "CITIZEN")) + | ((ssn == "NONE") != (status == "UNDOCUMENTED")) + | ((ssn == "CITIZEN") != profile.is_citizen) + ) + if immutable_pair_invalid.any(): + raise ValueError( + "Immutable ASEC immigration rows violate citizenship/paired-status " + f"invariants; invalid_rows={int(immutable_pair_invalid.sum())}." + ) + immutable_special_invalid = immutable & ( + ((status == "CUBAN_HAITIAN_ENTRANT") != cuban_haitian) + | ((status == "DACA") != daca) + ) + if immutable_special_invalid.any(): + raise ValueError( + "Immutable ASEC immigration rows violate Cuban/Haitian entrant or " + f"DACA cohort invariants; invalid_rows={int(immutable_special_invalid.sum())}." + ) + + selected_once = np.zeros(len(result), dtype=bool) + compatible_humanitarian = np.zeros(len(result), dtype=bool) + draw_receipts: dict[str, object] = {} + largest_target = max((draw.target for draw in controls.humanitarian), default=0.0) + tolerance = max(1e-6, largest_target * 1e-12) + for draw in controls.humanitarian: + immutable_draw = ( + _humanitarian_emitted_mask( + result, + draw=draw, + profile=profile, + ) + & immutable + ) + compatible_humanitarian |= immutable_draw + immutable_population = float(weights[immutable_draw].sum()) + if draw.target <= 0 and immutable_population > tolerance: + raise ValueError( + f"Immutable ASEC rows emit {immutable_population:,.6f} weighted " + f"persons for explicit-zero humanitarian draw {draw.label!r}." + ) + residual_target = max(0.0, float(draw.target) - immutable_population) + + candidate_ssn = _ssn_name_codes(result) + if draw.category in {"paroled_one_year", "tps"}: + candidate_ssn = np.where(candidate_ssn == 2, 0, candidate_ssn) + candidates = ( + _humanitarian_draw_candidates( + draw, + ssn_codes=candidate_ssn, + profile=profile, + ) + & mutable + & ~selected_once + ) + available_population = float(weights[candidates].sum()) + if available_population + tolerance < residual_target: + raise ValueError( + "Humanitarian transfer candidate shortfall for " + f"{draw.label!r}: residual_target={residual_target:,.6f}, " + f"eligible_recipient_population={available_population:,.6f}, " + f"immutable_population={immutable_population:,.6f}." + ) + stable_draws = _stable_person_draws( + result, + seed=seed, + salt=f"acs_transfer:{draw.salt}", + ) + selected = _select_weight_to_target( + candidates, + weights, + stable_draws, + residual_target, + ) + selected_population = float(weights[selected].sum()) + if selected_population + tolerance < residual_target: + raise RuntimeError( + f"Humanitarian transfer selection underfilled {draw.label!r}." + ) + status[selected] = draw.status + if draw.category in {"paroled_one_year", "tps"}: + ssn[selected & (ssn == "NONE")] = "NON_CITIZEN_VALID_EAD" + selected_once |= selected + result["ssn_card_type"] = ssn + result["immigration_status_str"] = status + achieved_mask = _humanitarian_emitted_mask( + result, + draw=draw, + profile=profile, + ) + compatible_humanitarian |= achieved_mask + achieved_population = float(weights[achieved_mask].sum()) + threshold_tie_population = 0.0 + if selected.any(): + threshold = float(np.max(stable_draws[selected])) + threshold_tie_population = float( + weights[candidates & (stable_draws == threshold)].sum() + ) + residual_selection_error = abs(selected_population - residual_target) + if residual_selection_error > threshold_tie_population + tolerance: + raise RuntimeError( + f"Humanitarian transfer residual selection for {draw.label!r} " + "missed its target by more than the threshold tie mass." + ) + absolute_error = abs(achieved_population - float(draw.target)) + immutable_overshoot = max(0.0, immutable_population - float(draw.target)) + discrete_bound = immutable_overshoot + threshold_tie_population + tolerance + draw_receipts[draw.label] = { + "target": float(draw.target), + "immutable_population": immutable_population, + "residual_target": residual_target, + "eligible_recipient_population": available_population, + "selected_recipient_population": selected_population, + "achieved_population": achieved_population, + "absolute_error": absolute_error, + "selection_threshold_tie_population": threshold_tie_population, + "residual_selection_error": residual_selection_error, + "within_residual_discrete_weight_bound": True, + "immutable_overshoot": immutable_overshoot, + "within_discrete_weight_bound": absolute_error <= discrete_bound, + } + if absolute_error > discrete_bound: + raise RuntimeError( + f"Humanitarian transfer reconciliation for {draw.label!r} " + "missed its target by more than the discrete selection bound." + ) + + result["ssn_card_type"] = ssn + result["immigration_status_str"] = status + all_humanitarian = np.isin(status, list(humanitarian_statuses)) + incompatible_humanitarian = all_humanitarian & ~compatible_humanitarian + if incompatible_humanitarian.any(): + raise ValueError( + "Humanitarian transfer output contains status/origin/arrival/SSN " + f"incompatible rows: {int(incompatible_humanitarian.sum())}." + ) + final_ssn_codes = _ssn_name_codes(result) + final_cuban_haitian, final_daca = _special_status_masks( + ssn_codes=final_ssn_codes, + birth_country=profile.birth_country, + arrival_year=profile.arrival_year, + age_at_entry=profile.age_at_entry, + age=profile.age, + ) + special_status_invalid = ( + (status == "CUBAN_HAITIAN_ENTRANT") != final_cuban_haitian + ) | ((status == "DACA") != final_daca) + if special_status_invalid.any(): + raise RuntimeError( + "Humanitarian transfer reconciliation left Cuban/Haitian entrant " + "or DACA cohort invariants invalid on " + f"{int(special_status_invalid.sum())} row(s)." + ) + final_pair_invalid = ( + ((ssn == "CITIZEN") != (status == "CITIZEN")) + | ((ssn == "NONE") != (status == "UNDOCUMENTED")) + | ((ssn == "CITIZEN") != profile.is_citizen) + ) + if final_pair_invalid.any(): + raise RuntimeError( + "Humanitarian transfer reconciliation left citizenship/paired-status " + f"invariants invalid on {int(final_pair_invalid.sum())} row(s)." + ) + if not np.array_equal( + result.loc[immutable, "ssn_card_type"].astype(str).to_numpy(), + original_ssn[immutable], + ) or not np.array_equal( + result.loc[immutable, "immigration_status_str"].astype(str).to_numpy(), + original_status[immutable], + ): + raise RuntimeError( + "Humanitarian transfer reconciliation changed immutable ASEC rows." + ) + receipt: dict[str, object] = { + "kind": "deterministic_humanitarian_residual_target", + "seed": int(seed), + "time_period": int(time_period), + "mutable_rows": int(mutable.sum()), + "immutable_rows": int(immutable.sum()), + "citizenship_repairs": citizenship_repairs, + "pair_repairs": pair_repairs, + "special_status_assignments": special_status_assignments, + "floating_tolerance": tolerance, + "selection_order": [draw.label for draw in controls.humanitarian], + "draws": draw_receipts, + } + return result, receipt + + def _assign_humanitarian_statuses( person: pd.DataFrame, ssn_codes: np.ndarray, @@ -905,28 +1694,15 @@ def _assign_humanitarian_statuses( """ marks = np.full(len(person), "", dtype="U24") - arrival_code, arrival_year, age_at_entry = _arrival_profile( - person, time_period=time_period - ) - age = _integer_column(person, "A_AGE") - birth_country = _integer_column(person, "PENATVTY") - # Cuba/Haiti-born keep the statutorily-favorable entrant class; the DACA - # cohort keeps its statutory tag (dual TPS/DACA holders stay DACA). - excluded = np.isin(birth_country, _CUBAN_HAITIAN_BIRTH_CODES) - excluded |= _daca_statutory_cohort(arrival_year, age_at_entry, age) + profile = _source_aware_immigration_profile(person, time_period=time_period) for draw in controls.humanitarian: if draw.target <= 0: continue - candidates = ( - _humanitarian_draw_candidates( - draw, - ssn_codes=ssn_codes, - birth_country=birth_country, - arrival_code=arrival_code, - ) - & ~excluded - & (marks == "") - ) + candidates = _humanitarian_draw_candidates( + draw, + ssn_codes=ssn_codes, + profile=profile, + ) & (marks == "") draws = _stable_person_draws(person, seed=seed, salt=draw.salt) selected = _select_weight_to_target(candidates, weights, draws, draw.target) marks[selected] = draw.status @@ -1072,15 +1848,14 @@ def _derive_immigration_status( status[ssn_codes == 1] = "CITIZEN" status[ssn_codes == 0] = "UNDOCUMENTED" - documented_noncitizen = np.isin(ssn_codes, [2, 3]) - cuban_haitian = ( - documented_noncitizen - & np.isin(birth_country, _CUBAN_HAITIAN_BIRTH_CODES) - & (arrival_year >= _CUBAN_HAITIAN_ARRIVAL_CUTOFF) + cuban_haitian, daca = _special_status_masks( + ssn_codes=ssn_codes, + birth_country=birth_country, + arrival_year=arrival_year, + age_at_entry=age_at_entry, + age=age, ) status[cuban_haitian] = "CUBAN_HAITIAN_ENTRANT" - - daca = (ssn_codes == 2) & _daca_statutory_cohort(arrival_year, age_at_entry, age) status[daca] = "DACA" # Humanitarian draws exclude Cuba/Haiti-born and the DACA cohort, so @@ -1096,6 +1871,7 @@ def with_us_immigration_inputs( *, seed: int, time_period: int, + person_weight_scale: float = 1.0, ) -> Frame: """Run the ``immigration_status`` manifest stage over a US frame. @@ -1107,6 +1883,11 @@ def with_us_immigration_inputs( CPS ASEC source columns. seed: Build-wide imputation seed. time_period: The dataset's time period (arrival-year arithmetic). + person_weight_scale: Stage-only multiplier for person design weights. + Pooled source projections use the inverse of their share of the + full pool's person mass so absolute national controls contribute + only that source's mass share. Standalone frames keep the default + multiplier of one. Returns: A new frame whose person table carries ``ssn_card_type`` and @@ -1121,6 +1902,15 @@ def with_us_immigration_inputs( if frame.schema != US_SCHEMA: raise ValueError("US immigration inputs require the US schema.") + if ( + isinstance(person_weight_scale, bool) + or not np.isfinite(person_weight_scale) + or person_weight_scale <= 0 + ): + raise ValueError( + "US immigration person_weight_scale must be a finite positive " + f"number, got {person_weight_scale!r}." + ) person = frame.table("person") present = [ column for column in US_IMMIGRATION_OUTPUT_COLUMNS if column in person.columns @@ -1135,7 +1925,9 @@ def with_us_immigration_inputs( ) stage_person = person.copy(deep=True) - stage_person[_PERSON_WEIGHT_COLUMN] = frame.resolve_weights("person").values + stage_person[_PERSON_WEIGHT_COLUMN] = np.asarray( + frame.resolve_weights("person").values, dtype=np.float64 + ) * float(person_weight_scale) output = run_source_stage( us_immigration_stage_spec(), tables={"person": stage_person}, @@ -1211,18 +2003,7 @@ def us_immigration_composition_gate( """ if controls is None: - stage = us_immigration_stage_spec() - derive = [ - operation - for operation in stage.operations - if operation.kind == "derive_immigration_status" - ] - if len(derive) != 1: - raise ValueError( - "US immigration stage must declare exactly one " - "derive_immigration_status operation." - ) - controls = _controls_from_parameters(derive[0].parameters) + controls = us_immigration_controls() person = frame.table("person") weights = np.asarray(frame.resolve_weights("person").values, dtype=np.float64) @@ -1264,6 +2045,11 @@ def us_immigration_composition_gate( ssn = person["ssn_card_type"].astype(str) status = person["immigration_status_str"].astype(str) + try: + evidence = _source_aware_immigration_profile(person, time_period=2024) + except (SourceRuntimeError, ValueError) as exc: + evidence = None + failures.append(f"source-aware immigration evidence is invalid: {exc}") for column, values, domain in ( ("ssn_card_type", ssn, SSN_CARD_TYPE_VALUES), ("immigration_status_str", status, IMMIGRATION_STATUS_VALUES), @@ -1295,6 +2081,33 @@ def us_immigration_composition_gate( f"{undocumented_disagreements} person(s) have ssn_card_type NONE " "and immigration_status_str UNDOCUMENTED disagreeing." ) + if evidence is not None: + evidence_citizen_disagreements = int((ssn_citizen != evidence.is_citizen).sum()) + if evidence_citizen_disagreements: + failures.append( + f"{evidence_citizen_disagreements} person(s) have emitted " + "citizenship disagreeing with their source PRCITSHP/CIT evidence." + ) + expected_cuban_haitian, expected_daca = _special_status_masks( + ssn_codes=_ssn_name_codes(person), + birth_country=evidence.birth_country, + arrival_year=evidence.arrival_year, + age_at_entry=evidence.age_at_entry, + age=evidence.age, + ) + special_status_invalid = ( + (status.to_numpy() == "CUBAN_HAITIAN_ENTRANT") != expected_cuban_haitian + ) | ((status.to_numpy() == "DACA") != expected_daca) + details["evidence_derived_status_compatibility"] = { + "cuban_haitian_entrant_rows": int(expected_cuban_haitian.sum()), + "daca_rows": int(expected_daca.sum()), + "invalid_rows": int(special_status_invalid.sum()), + } + if special_status_invalid.any(): + failures.append( + f"{int(special_status_invalid.sum())} person(s) violate the " + "source-aware Cuban/Haitian entrant or DACA cohort contract." + ) non_citizen_share = ( float(weights[~ssn_citizen].sum()) / total if total else float("nan") @@ -1320,6 +2133,54 @@ def us_immigration_composition_gate( status_values = status.to_numpy() hum_low, hum_high = _HUMANITARIAN_TARGET_RELATIVE_BAND + humanitarian_statuses = set(_HUMANITARIAN_STATUS_BY_CATEGORY.values()) + humanitarian_rows = np.isin(status_values, list(humanitarian_statuses)) + draw_achieved: dict[str, dict[str, object]] = {} + compatible_humanitarian = np.zeros(len(person), dtype=bool) + if evidence is not None: + for draw in controls.humanitarian: + emitted_mask = _humanitarian_emitted_mask( + person, + draw=draw, + profile=evidence, + ) + compatible_humanitarian |= emitted_mask + emitted = float(weights[emitted_mask].sum()) + relative = (emitted / draw.target) if draw.target > 0 else None + draw_achieved[draw.label] = { + "category": draw.category, + "origin": draw.origin, + "status": draw.status, + "target": draw.target, + "population": emitted, + "relative": relative, + "source": draw.source, + } + if draw.target <= 0: + if emitted > 0: + failures.append( + f"{draw.label}: {emitted:,.0f} compatible weighted " + "persons emitted against an explicit zero target." + ) + continue + if relative is None or not (hum_low <= relative <= hum_high): + failures.append( + f"{draw.label}: compatible emitted population {emitted:,.0f} " + f"is {relative:.2f}x the cited stock target {draw.target:,.0f} " + f"(band [{hum_low}, {hum_high}])." + ) + incompatible = humanitarian_rows & ~compatible_humanitarian + if incompatible.any(): + incompatible_population = float(weights[incompatible].sum()) + examples = sorted(set(status_values[incompatible].tolist())) + failures.append( + f"{int(incompatible.sum())} humanitarian row(s) " + f"({incompatible_population:,.0f} weighted persons) violate " + "their source-aware status/origin/arrival/SSN cohort contract; " + f"statuses={examples}." + ) + details["humanitarian_draw_achieved"] = draw_achieved + achieved_by_category: dict[str, dict[str, object]] = {} for category in HUMANITARIAN_STATUS_CATEGORIES: status_name = _HUMANITARIAN_STATUS_BY_CATEGORY[category] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/multispine_pool.py b/packages/microcosm-build/src/microcosm/build/us_runtime/multispine_pool.py index 8f2ea06b98..dbeeac6a2d 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/multispine_pool.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/multispine_pool.py @@ -469,7 +469,7 @@ def simulation_ready(self) -> bool: type PoolOperator = Callable[[Frame], PoolStageOutput] type AgreementGate = Callable[[Frame], GateResult] -type SourceFrameOperator = Callable[[Frame], Frame | PoolStageOutput] +type SourceFrameOperator = Callable[..., Frame | PoolStageOutput] @dataclass(frozen=True) @@ -861,9 +861,8 @@ def _resolve_take_up_program_bindings( for program in load_take_up_contract().programs ) for index, binding in enumerate(bindings): - if ( - len(binding) != 3 - or not all(isinstance(value, str) and value for value in binding) + if len(binding) != 3 or not all( + isinstance(value, str) and value for value in binding ): raise ValueError( "Take-up manifest program binding must contain three non-empty " @@ -1333,9 +1332,7 @@ def surface_provision(variable: str) -> str: variable, execution_scope="whole_pool", provision=provision, - available_by=( - "transferred" if variable in transfer_owned else "seeded" - ), + available_by=("transferred" if variable in transfer_owned else "seeded"), fallback=fallback, ) @@ -1941,10 +1938,13 @@ def _post_clone_source_operators() -> Mapping[str, SourceFrameOperator]: force_puf_imputation=True, ) ), - "with_us_immigration_inputs": lambda current: with_us_immigration_inputs( - current, - seed=POOL_RANDOM_SEED, - time_period=POOL_TIME_PERIOD, + "with_us_immigration_inputs": lambda current, *, person_weight_scale=1.0: ( + with_us_immigration_inputs( + current, + seed=POOL_RANDOM_SEED, + time_period=POOL_TIME_PERIOD, + person_weight_scale=person_weight_scale, + ) ), "with_us_education_inputs": lambda current: with_us_education_inputs( current, @@ -2150,8 +2150,42 @@ def _run_source_operator_chain( and contract.execution_scope == _CPS_SOURCE_EXECUTION_SCOPE ): declared_outputs = _persisted_source_outputs(declared_outputs) + person_design_weight_scaling: dict[str, float] | None = None if contract.execution_scope == _CPS_SOURCE_EXECUTION_SCOPE: available_mask = _cps_source_evidence_mask(current, phase=phase) + if operator_name == "with_us_immigration_inputs": + full_person_weights = np.asarray( + current.resolve_weights(current.schema.person_entity).values, + dtype=np.float64, + ) + cps_person_weights = full_person_weights[ + available_mask.to_numpy(dtype=bool) + ] + if ( + not np.isfinite(full_person_weights).all() + or (full_person_weights < 0).any() + or not np.isfinite(cps_person_weights).all() + or (cps_person_weights < 0).any() + ): + raise ValueError( + "Pooled immigration controls require finite non-negative " + "person design weights." + ) + full_person_mass = float(full_person_weights.sum()) + cps_person_mass = float(cps_person_weights.sum()) + if full_person_mass <= 0 or cps_person_mass <= 0: + raise ValueError( + "Pooled immigration controls require positive full-pool " + "and CPS-projection person design-weight mass; " + f"got full={full_person_mass!r}, cps={cps_person_mass!r}." + ) + person_weight_scale = full_person_mass / cps_person_mass + person_design_weight_scaling = { + "full_pool_person_design_weight_mass": full_person_mass, + "cps_projection_person_design_weight_mass": cps_person_mass, + "person_weight_scale": person_weight_scale, + "cps_person_mass_share": cps_person_mass / full_person_mass, + } available = _source_available_projection( current, available_mask, @@ -2170,13 +2204,29 @@ def _run_source_operator_chain( ) before_rows = _frame_row_counts(current) available_rows = _frame_row_counts(available) - kernel_outcome = operators[operator_name](available) + if person_design_weight_scaling is None: + kernel_outcome = operators[operator_name](available) + else: + kernel_outcome = operators[operator_name]( + available, + person_weight_scale=person_design_weight_scaling["person_weight_scale"], + ) kernel_receipt: Mapping[str, object] = {} if isinstance(kernel_outcome, PoolStageOutput): outcome = kernel_outcome.frame kernel_receipt = kernel_outcome.receipt else: outcome = kernel_outcome + if person_design_weight_scaling is not None: + if "person_design_weight_scaling" in kernel_receipt: + raise ValueError( + "Pooled immigration kernel receipt may not override the " + "orchestrator-owned person-design-weight scaling contract." + ) + kernel_receipt = { + **dict(kernel_receipt), + "person_design_weight_scaling": person_design_weight_scaling, + } if not isinstance(outcome, Frame): raise TypeError( f"Multispine source operator {operator_name!r} must return Frame, " diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py b/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py index 879445e0bd..e4f80cb24d 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py @@ -99,6 +99,11 @@ TargetFamilies, transfer_acs_inputs, ) +from microcosm.build.us_runtime.immigration import ( + us_immigration_controls, + us_immigration_humanitarian_draw_mask, + us_immigration_humanitarian_transfer_selection_masks, +) from microcosm.build.us_runtime.late_producer_dag import ( ProducerContract, ProducerInput, @@ -4144,6 +4149,54 @@ def _validate_post_transfer_live_output_binding( "unmodeled_rows", "residual_null_rows", ) +_IMMIGRATION_TRANSFER_FAMILY = "source_operator_immigration" +_IMMIGRATION_TRANSFER_TARGETS = ("ssn_card_type", "immigration_status_str") +_IMMIGRATION_RECONCILIATION_KEY = "post_transfer_reconciliation" +_IMMIGRATION_RECONCILIATION_DRAW_FIELDS = frozenset( + { + "target", + "immutable_population", + "residual_target", + "eligible_recipient_population", + "selected_recipient_population", + "achieved_population", + "absolute_error", + "selection_threshold_tie_population", + "residual_selection_error", + "within_residual_discrete_weight_bound", + "immutable_overshoot", + "within_discrete_weight_bound", + } +) + + +def _is_immigration_transfer_target( + *, + entity: str, + family: str, + target: str, +) -> bool: + return ( + entity == "person" + and family.split("__batch_", 1)[0] == _IMMIGRATION_TRANSFER_FAMILY + and target in _IMMIGRATION_TRANSFER_TARGETS + ) + + +def _immigration_qrf_evidence_targets( + surface: TargetFamilies, +) -> tuple[tuple[str, str], ...]: + return tuple( + (entity, target) + for entity, families in surface.items() + for family, targets in families.items() + for target in targets + if _is_immigration_transfer_target( + entity=entity, + family=family, + target=target, + ) + ) def _validate_acs_transfer_row_counts( @@ -4179,6 +4232,260 @@ def _validate_acs_transfer_row_counts( return typed_counts +def _validate_immigration_post_transfer_reconciliation( + target_receipt: Mapping[str, object], + *, + boundary: str, + frame: Frame | None = None, +) -> Mapping[str, object]: + """Validate the persisted constrained-recipient reconciliation evidence.""" + + reconciliation = target_receipt.get(_IMMIGRATION_RECONCILIATION_KEY) + if not isinstance(reconciliation, Mapping): + raise ValueError( + f"{boundary}: immigration post-transfer reconciliation is absent." + ) + expected_keys = { + "kind", + "seed", + "time_period", + "mutable_rows", + "immutable_rows", + "citizenship_repairs", + "pair_repairs", + "special_status_assignments", + "floating_tolerance", + "selection_order", + "draws", + } + if set(reconciliation) != expected_keys or reconciliation.get("kind") != ( + "deterministic_humanitarian_residual_target" + ): + raise ValueError( + f"{boundary}: immigration post-transfer reconciliation schema is invalid." + ) + integer_fields = ( + "seed", + "time_period", + "mutable_rows", + "immutable_rows", + "citizenship_repairs", + "pair_repairs", + "special_status_assignments", + ) + if any( + not isinstance(reconciliation.get(field), int) + or isinstance(reconciliation[field], bool) + or reconciliation[field] < 0 + for field in integer_fields + ): + raise ValueError( + f"{boundary}: immigration reconciliation integer evidence is invalid." + ) + counts = _validate_acs_transfer_row_counts( + target_receipt, + boundary=boundary, + required=True, + ) + if ( + reconciliation["mutable_rows"] != counts["imputed_rows"] + or reconciliation["citizenship_repairs"] > reconciliation["mutable_rows"] + or reconciliation["pair_repairs"] > reconciliation["mutable_rows"] + or reconciliation["special_status_assignments"] > reconciliation["mutable_rows"] + or reconciliation["citizenship_repairs"] > reconciliation["pair_repairs"] + ): + raise ValueError( + f"{boundary}: immigration reconciliation row accounting is invalid." + ) + producer_rows = target_receipt.get("producer_rows") + if producer_rows is not None and ( + not isinstance(producer_rows, int) + or isinstance(producer_rows, bool) + or producer_rows < 0 + or reconciliation["immutable_rows"] != producer_rows + ): + raise ValueError( + f"{boundary}: immigration reconciliation immutable-row binding is invalid." + ) + controls = us_immigration_controls() + canonical_tolerance = max( + 1e-6, + max((draw.target for draw in controls.humanitarian), default=0.0) * 1e-12, + ) + tolerance = reconciliation.get("floating_tolerance") + if ( + not isinstance(tolerance, (int, float)) + or isinstance(tolerance, bool) + or not np.isfinite(float(tolerance)) + or tolerance <= 0 + or float(tolerance) != canonical_tolerance + ): + raise ValueError( + f"{boundary}: immigration reconciliation tolerance is non-canonical." + ) + tolerance = float(tolerance) + live_draw_populations: dict[str, tuple[float, float, float]] = {} + live_mutable_draw_masks: dict[str, np.ndarray] = {} + expected_mutable_draw_masks: dict[str, np.ndarray] = {} + if frame is not None: + person_entity = frame.schema.person_entity + person = frame.table(person_entity) + channel = person[support_channel_column(person_entity)].astype(str) + immutable_rows = channel.eq(BASE_ASEC_SUPPORT_CHANNEL).to_numpy(dtype=bool) + mutable_rows = channel.eq(ACS_STACKED_SUPPORT_CHANNEL).to_numpy(dtype=bool) + if ( + not np.logical_or(immutable_rows, mutable_rows).all() + or int(immutable_rows.sum()) != reconciliation["immutable_rows"] + or int(mutable_rows.sum()) != reconciliation["mutable_rows"] + ): + raise ValueError( + f"{boundary}: immigration reconciliation row counts differ " + "from the live ASEC/ACS frame." + ) + weights = np.asarray( + frame.resolve_weights(person_entity).values, + dtype=np.float64, + ) + if ( + weights.shape != (len(person),) + or not np.isfinite(weights).all() + or (weights < 0).any() + ): + raise ValueError( + f"{boundary}: immigration reconciliation live person weights " + "are invalid." + ) + for control in controls.humanitarian: + emitted = us_immigration_humanitarian_draw_mask( + frame, + control, + time_period=int(reconciliation["time_period"]), + ) + live_draw_populations[control.label] = ( + float(weights[emitted & immutable_rows].sum()), + float(weights[emitted & mutable_rows].sum()), + float(weights[emitted].sum()), + ) + live_mutable_draw_masks[control.label] = emitted & mutable_rows + expected_mutable_draw_masks = ( + us_immigration_humanitarian_transfer_selection_masks( + frame, + mutable_rows=mutable_rows, + seed=int(reconciliation["seed"]), + time_period=int(reconciliation["time_period"]), + controls=controls, + ) + ) + selection_order = reconciliation.get("selection_order") + draws = reconciliation.get("draws") + expected_order = [draw.label for draw in controls.humanitarian] + targets_by_label = { + draw.label: float(draw.target) for draw in controls.humanitarian + } + if ( + not isinstance(selection_order, list) + or selection_order != expected_order + or not isinstance(draws, Mapping) + or set(draws) != set(expected_order) + ): + raise ValueError( + f"{boundary}: immigration reconciliation draw order is non-canonical." + ) + numeric_fields = _IMMIGRATION_RECONCILIATION_DRAW_FIELDS - { + "within_residual_discrete_weight_bound", + "within_discrete_weight_bound", + } + for label in expected_order: + draw = draws[label] + if not isinstance(draw, Mapping) or set(draw) != ( + _IMMIGRATION_RECONCILIATION_DRAW_FIELDS + ): + raise ValueError( + f"{boundary}: immigration reconciliation draw {label!r} schema " + "is invalid." + ) + if any( + not isinstance(draw.get(field), (int, float)) + or isinstance(draw[field], bool) + or not np.isfinite(float(draw[field])) + or draw[field] < 0 + for field in numeric_fields + ) or any( + draw.get(field) is not True + for field in ( + "within_residual_discrete_weight_bound", + "within_discrete_weight_bound", + ) + ): + raise ValueError( + f"{boundary}: immigration reconciliation draw {label!r} values " + "are invalid." + ) + target = targets_by_label[label] + immutable = float(draw["immutable_population"]) + residual = float(draw["residual_target"]) + eligible = float(draw["eligible_recipient_population"]) + selected = float(draw["selected_recipient_population"]) + achieved = float(draw["achieved_population"]) + tie = float(draw["selection_threshold_tie_population"]) + expected_residual = max(0.0, target - immutable) + residual_error = abs(selected - residual) + absolute_error = abs(achieved - target) + immutable_overshoot = max(0.0, immutable - target) + if frame is not None: + live_immutable, live_selected, live_achieved = live_draw_populations[label] + if ( + abs(immutable - live_immutable) > tolerance + or abs(selected - live_selected) > tolerance + or abs(achieved - live_achieved) > tolerance + ): + raise ValueError( + f"{boundary}: immigration reconciliation draw {label!r} " + "differs from the live ASEC/ACS weighted population." + ) + if not np.array_equal( + live_mutable_draw_masks[label], + expected_mutable_draw_masks[label], + ): + raise ValueError( + f"{boundary}: immigration reconciliation draw {label!r} " + "selected mutable-row identities differ from the canonical " + "seeded selection." + ) + if target <= 0 and immutable > tolerance: + raise ValueError( + f"{boundary}: immigration reconciliation draw {label!r} has " + "immutable mass against an explicit-zero target." + ) + if eligible + tolerance < residual: + raise ValueError( + f"{boundary}: immigration reconciliation draw {label!r} " + "conceals a recipient candidate shortfall." + ) + if selected + tolerance < residual: + raise ValueError( + f"{boundary}: immigration reconciliation draw {label!r} " + "underfills its residual target." + ) + if ( + abs(float(draw["target"]) - target) > tolerance + or abs(residual - expected_residual) > tolerance + or selected > eligible + tolerance + or tie > eligible + tolerance + or abs(achieved - (immutable + selected)) > tolerance + or abs(float(draw["residual_selection_error"]) - residual_error) > tolerance + or residual_error > tie + tolerance + or abs(float(draw["absolute_error"]) - absolute_error) > tolerance + or abs(float(draw["immutable_overshoot"]) - immutable_overshoot) > tolerance + or absolute_error > immutable_overshoot + tie + tolerance + ): + raise ValueError( + f"{boundary}: immigration reconciliation draw {label!r} " + "accounting is invalid." + ) + return reconciliation + + def _acs_imputed_pattern_evidence(record: AcsImputedInput) -> dict[str, object]: """Return deterministic JSON-ready evidence for one ACS transfer record.""" @@ -4242,6 +4549,11 @@ def _acs_pattern_predictor_authority( mandatory = tuple(housing["mandatory_features"]) required = (*required, *mandatory) optional = tuple(item for item in optional if item not in mandatory) + immigration_targets = set(contract["immigration_status_targets"]) + if immigration_targets.issubset(family_targets): + mandatory = tuple(contract["immigration_required_predictors"]) + required = (*required, *mandatory) + optional = tuple(item for item in optional if item not in mandatory) return tuple(required), tuple(optional) optional_names = acs_transfer_runtime._GROUP_OPTIONAL_NAMES return ( @@ -4780,6 +5092,7 @@ def validate_stacked_post_puf_transfer_receipt( validated_calibration_keys.update( spec.key for spec in expected_calibrations.values() ) + group_immigration_reconciliations: list[Mapping[str, object]] = [] for target_key, target_receipt in group_targets.items(): if not isinstance(target_receipt, Mapping): raise ValueError( @@ -4793,8 +5106,36 @@ def validate_stacked_post_puf_transfer_receipt( ) owner_receipt = target_receipt.get("post_transfer_calibration") spec = expected_calibrations.get(target_key) + target = target_key.rsplit("/", 1)[1] + is_immigration = _is_immigration_transfer_target( + entity=group.entity, + family=group.family, + target=target, + ) + if is_immigration: + group_immigration_reconciliations.append( + _validate_immigration_post_transfer_reconciliation( + target_receipt, + boundary=f"{boundary} target {target_key}", + frame=frame, + ) + ) + _validate_acs_imputed_pattern_evidence( + target_receipt, + expected_entity=group.entity, + expected_family=group.family, + expected_target=target, + expected_family_targets=group.targets, + expected_regime_targets=group.targets, + boundary=f"{boundary} target {target_key}", + ) + elif _IMMIGRATION_RECONCILIATION_KEY in target_receipt: + raise ValueError( + f"{boundary}: undeclared immigration reconciliation is " + f"attached to {target_key!r}." + ) if spec is None: - if "qrf_pattern_evidence" in target_receipt: + if "qrf_pattern_evidence" in target_receipt and not is_immigration: raise ValueError( f"{boundary}: undeclared ACS QRF pattern evidence is " f"attached to {target_key!r}." @@ -4814,7 +5155,7 @@ def validate_stacked_post_puf_transfer_receipt( target_receipt, expected_entity=group.entity, expected_family=group.family, - expected_target=target_key.rsplit("/", 1)[1], + expected_target=target, expected_family_targets=group.targets, expected_regime_targets=expected_regime_targets, boundary=f"{boundary} target {target_key}", @@ -4918,6 +5259,17 @@ def validate_stacked_post_puf_transfer_receipt( spec=spec, boundary=f"{boundary} target {target_key}", ) + if group_immigration_reconciliations: + if len(group_immigration_reconciliations) != len( + _IMMIGRATION_TRANSFER_TARGETS + ) or any( + _json_ready(item) != _json_ready(group_immigration_reconciliations[0]) + for item in group_immigration_reconciliations[1:] + ): + raise ValueError( + f"{boundary}: paired immigration reconciliation evidence " + f"is incomplete or inconsistent for group {name!r}." + ) if validated_calibration_keys != set(late_calibration_specs): raise ValueError( f"{boundary}: post-transfer calibration coverage does not match " @@ -7978,6 +8330,17 @@ def validate_stacked_late_producer_transition_authority( """Validate the anchor after declared downstream operators have run.""" validate_stacked_late_producer_receipt(receipt, boundary=boundary) + transfer = receipt.get("post_puf_transfer") + assert isinstance(transfer, Mapping) + # Downstream fiscal and gate operators intentionally change the complete + # late-output digest, but they do not own source channels, resolved + # weights, or immigration outputs. Replay that reconstructible terminal + # evidence before authenticating the immutable transition carrier. + validate_stacked_post_puf_transfer_receipt( + transfer, + boundary=boundary, + frame=frame, + ) _validate_late_transition_authority( frame, receipt, @@ -10171,10 +10534,17 @@ def _transfer_stacked_post_puf_inputs_evaluate( derive_schedule_d=derive_schedule_d, execution_contract=execution_contract, regime_evidence_targets=tuple( - (spec.entity, spec.target) - for spec in _stacked_post_transfer_calibration_specs( - surface, - stage="late_transfer", + dict.fromkeys( + ( + *( + (spec.entity, spec.target) + for spec in _stacked_post_transfer_calibration_specs( + surface, + stage="late_transfer", + ) + ), + *_immigration_qrf_evidence_targets(surface), + ) ) ), ) @@ -10419,7 +10789,8 @@ def _verify_post_puf_transfer_outcome( target_families, stage="late_transfer", ) - } + } | set(_immigration_qrf_evidence_targets(target_families)) + immigration_reconciliations: dict[str, Mapping[str, object]] = {} for entity, families in target_families.items(): table = frame.table(entity) for family, family_targets in families.items(): @@ -10499,7 +10870,39 @@ def _verify_post_puf_transfer_outcome( target_receipt["qrf_pattern_evidence"] = ( _acs_imputed_pattern_evidence(record) ) + if _is_immigration_transfer_target( + entity=entity, + family=family, + target=target, + ): + if record is None or not isinstance(record.reconciliation, Mapping): + failures.append( + f"{label}: constrained immigration reconciliation " + "evidence is absent." + ) + else: + evidence = _json_ready(record.reconciliation) + assert isinstance(evidence, Mapping) + target_receipt[_IMMIGRATION_RECONCILIATION_KEY] = evidence + immigration_reconciliations[target] = evidence target_receipts[target_receipt_key] = target_receipt + if set(immigration_reconciliations) not in ( + set(), + set(_IMMIGRATION_TRANSFER_TARGETS), + ): + failures.append( + "post_puf_transfer/person/source_operator_immigration: paired " + "reconciliation evidence is incomplete." + ) + elif immigration_reconciliations and any( + _json_ready(evidence) + != _json_ready(immigration_reconciliations[_IMMIGRATION_TRANSFER_TARGETS[0]]) + for evidence in immigration_reconciliations.values() + ): + failures.append( + "post_puf_transfer/person/source_operator_immigration: paired " + "reconciliation evidence disagrees." + ) if failures: raise ValueError( "Stacked post-PUF transfer outcome verification failed:\n " diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_producer_registry.py b/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_producer_registry.py index 1b0f0cc6cf..0f8445595b 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_producer_registry.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_producer_registry.py @@ -99,7 +99,9 @@ "us_late_producer_schedule_receipt", ] -# v16 declares person.s_corp_income as a whole-pool primary-PUF output: its +# v17 binds the ASEC and ACS source-scoped evidence read by constrained +# immigration transfer and reconciliation. v16 declares person.s_corp_income +# as a whole-pool primary-PUF output: its # certified combined-source semantics are carried by partnership_income, while # the separate S-corporation leaf is an exact-zero universe. v15 content-binds # the complete late dual-producer ownership matrix and @@ -122,9 +124,11 @@ # implicit. Receipt v3 reconciles repeated physical evidence and scope # cardinalities across each execution row, binds source-receipt outputs to the # callback receipt, and requires the primary callback to report the exact -# resources it consumed. Receipt v2 introduced exact virtual-resource payloads. -US_LATE_PRODUCER_REGISTRY_SCHEMA_VERSION = 16 -US_LATE_PRODUCER_RECEIPT_SCHEMA_VERSION = 3 +# resources it consumed. Receipt v4 additionally persists and validates the +# paired constrained-immigration reconciliation and QRF evidence. Receipt v2 +# introduced exact virtual-resource payloads. +US_LATE_PRODUCER_REGISTRY_SCHEMA_VERSION = 17 +US_LATE_PRODUCER_RECEIPT_SCHEMA_VERSION = 4 US_LATE_PRODUCER_TRANSITION_AUTHORITY_VERSION = 1 US_LATE_PRODUCER_TRANSITION_AUTHORITY_KEY = "us_late_producer_transition_authority" US_LATE_PRODUCER_TRANSITION_AUTHORITY_ID = "us_stacked_late_producer_transition" @@ -970,6 +974,11 @@ def _inventory( _single("resolved_person_weight", "person", "@resolved_weight"), _requirement( "stable_source_identity", + ( + _column("person", "source_year"), + _column("person", "source_household_id"), + _column("person", "source_person_id"), + ), ( _column("person", "source_year"), _column("person", "source_person_id"), @@ -1188,6 +1197,69 @@ def _transfer_input_inventory(group: TransferProducerGroup) -> SourceInputInvent _ADULT_CARE_ROLE_INPUT, ) ) + if group.name == transfer_producer_name("person", "source_operator_immigration"): + post_transfer_structure.extend( + ( + _requirement( + "immigration_stable_person_lineage", + ( + _column("person", "source_year"), + _column("person", "source_household_id"), + _column("person", "source_person_id"), + ), + ( + _column("person", "source_year"), + _column("person", "source_person_id"), + ), + (_column("person", "person_id"),), + ), + _single( + "immigration_asec_citizenship", + "person", + "PRCITSHP", + value_kind="finite_numeric", + required_scope=_ASEC_SOURCE_SCOPE, + ), + _single( + "immigration_asec_origin", + "person", + "PENATVTY", + value_kind="finite_numeric", + required_scope=_ASEC_SOURCE_SCOPE, + ), + _single( + "immigration_asec_arrival", + "person", + "PEINUSYR", + value_kind="finite_numeric", + required_scope=_ASEC_SOURCE_SCOPE, + ), + _single( + "immigration_acs_citizenship", + "person", + "CIT", + value_kind="finite_numeric", + required_scope=_ACS_SOURCE_SCOPE, + ), + _single( + "immigration_acs_origin", + "person", + "POBP", + value_kind="finite_numeric", + required_scope=_ACS_SOURCE_SCOPE, + ), + _single( + "immigration_acs_arrival", + "person", + "YOEP", + # Native-born ACS people carry a structural blank. The + # source-aware runtime requires a finite year only for + # foreign-born CIT=4/5 rows. + value_kind="column_present", + required_scope=_ACS_SOURCE_SCOPE, + ), + ) + ) return _inventory( group.name, *structural, @@ -2063,7 +2135,8 @@ def us_late_producer_schedule_payload() -> dict[str, object]: ), "top_binding": ( "entry_and_output_frame_sha256_execution_chain_source_" - "completion_and_nineteen_transfer_groups" + "completion_nineteen_transfer_groups_and_constrained_" + "immigration_reconciliation" ), "transition_authority": { "authority_id": US_LATE_PRODUCER_TRANSITION_AUTHORITY_ID, diff --git a/packages/microcosm-build/tests/test_spec_engine_compiler_ir.py b/packages/microcosm-build/tests/test_spec_engine_compiler_ir.py index c669fa342a..6de9a45582 100644 --- a/packages/microcosm-build/tests/test_spec_engine_compiler_ir.py +++ b/packages/microcosm-build/tests/test_spec_engine_compiler_ir.py @@ -22,9 +22,7 @@ ) from microcosm.build.spec_engine.resolver import F0_CONTRACT_ONLY_KERNEL_IDS -US_SCHEDULE_SHA256 = ( - "dcf3c6d2eade3449836c49a1dc4d3b8cd395aab9142db700c3c60598fa9c1c79" -) +US_SCHEDULE_SHA256 = "4965c1485c283dec3685f4ca82fa469d8b88a85f82ccd6b39e2adc84bc0e94d6" @pytest.fixture(scope="module") @@ -180,9 +178,7 @@ def test_normative_node_mutation_changes_its_slice_and_descendant_keys( def mutation(value: dict[str, Any]) -> None: node = next( - row - for row in value["producer_graph"]["nodes"] - if row["id"] == leaf_id + row for row in value["producer_graph"]["nodes"] if row["id"] == leaf_id ) node["capabilities"]["retry_safety"] = "nonretryable" @@ -196,9 +192,9 @@ def mutation(value: dict[str, Any]) -> None: def test_dangling_compiler_dependency_refuses(resolved_us: ResolvedSpec) -> None: def mutation(value: dict[str, Any]) -> None: - value["producer_graph"]["nodes"][0]["inputs"][0][ - "producing_stage" - ] = "missing_producer" + value["producer_graph"]["nodes"][0]["inputs"][0]["producing_stage"] = ( + "missing_producer" + ) mutated = _mutate_domain(resolved_us, ResourceKind.IMPUTATION, mutation) with pytest.raises(CompilerIRError, match="dangling producer"): @@ -211,9 +207,7 @@ def test_contract_only_kernel_cannot_back_a_producer( contract_only_kernel = min(F0_CONTRACT_ONLY_KERNEL_IDS) def mutation(value: dict[str, Any]) -> None: - value["producer_graph"]["nodes"][0]["kernel"] = ( - f"kernel:{contract_only_kernel}" - ) + value["producer_graph"]["nodes"][0]["kernel"] = f"kernel:{contract_only_kernel}" mutated = _mutate_domain(resolved_us, ResourceKind.IMPUTATION, mutation) with pytest.raises( diff --git a/packages/microcosm-build/tests/test_spec_engine_coverage_tool.py b/packages/microcosm-build/tests/test_spec_engine_coverage_tool.py index fcfe372058..3ad0a4afb9 100644 --- a/packages/microcosm-build/tests/test_spec_engine_coverage_tool.py +++ b/packages/microcosm-build/tests/test_spec_engine_coverage_tool.py @@ -52,21 +52,21 @@ def test_us_coverage_is_exact_complete_and_honest( assert_coverage_complete(coverage_report) assert coverage_report["status"] == "pass" fields = coverage_report["field_usage"] - assert fields["configuration_field_count"] == 42_419 - assert fields["authored_normative_field_count"] == 32_403 + assert fields["configuration_field_count"] == 42_511 + assert fields["authored_normative_field_count"] == 32_495 assert fields["resolved_binding_field_count"] == 10_016 - assert fields["consumed_field_count"] == 42_419 + assert fields["consumed_field_count"] == 42_511 assert fields["unused_field_count"] == 0 assert fields["multiple_primary_use_field_count"] == 0 assert fields["claim_count"] == 49 assert fields["mode_counts"] == { - "legacy_behavior": 13_983, - "compiler_semantic": 27_985, + "legacy_behavior": 14_002, + "compiler_semantic": 28_058, "front_end_validation": 348, "identity_only": 103, } assert fields["generation0_effect_counts"] == { - "legacy_behavior": 38_456, + "legacy_behavior": 38_548, "no_generation0_effect": 3_963, } @@ -75,7 +75,7 @@ def test_us_coverage_is_exact_complete_and_honest( assert inventory["covered_item_count"] == 41 assert inventory["missing_item_count"] == 0 assert inventory["missing_items"] == [] - assert inventory["counts"]["producer_inputs"] == 2_742 + assert inventory["counts"]["producer_inputs"] == 2_749 assert inventory["counts"]["ownership_rows"] == 18 assert inventory["counts"]["tail_control_fields"] == 934 assert inventory["counts"]["seed_owner_bindings"] == 125 @@ -89,9 +89,7 @@ def test_us_coverage_is_exact_complete_and_honest( "report schema version differs", ), ( - lambda report: report["spec_binding"].__setitem__( - "schema_version", 99 - ), + lambda report: report["spec_binding"].__setitem__("schema_version", 99), "spec_binding contract differs", ), ( diff --git a/packages/microcosm-build/tests/test_spec_engine_field_usage.py b/packages/microcosm-build/tests/test_spec_engine_field_usage.py index 04e118087e..5258d3a9a8 100644 --- a/packages/microcosm-build/tests/test_spec_engine_field_usage.py +++ b/packages/microcosm-build/tests/test_spec_engine_field_usage.py @@ -92,22 +92,22 @@ def _mutated_bundle( def test_exact_complete_ledger_has_one_primary_mode_per_pointer(field_ledger) -> None: - assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_419 + assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_511 assert field_ledger.source_counts == { - "authored": 32_403, + "authored": 32_495, "resolved_bindings": 10_016, } assert field_ledger.mode_counts == { - "legacy_behavior": 13_983, - "compiler_semantic": 27_985, + "legacy_behavior": 14_002, + "compiler_semantic": 28_058, "front_end_validation": 348, "identity_only": 103, } assert field_ledger.generation0_effect_counts == { - "legacy_behavior": 38_456, + "legacy_behavior": 38_548, "no_generation0_effect": 3_963, } - assert len({field.pointer for field in field_ledger.fields}) == 42_419 + assert len({field.pointer for field in field_ledger.fields}) == 42_511 def test_eligibility_concepts_are_validation_not_generation0_behavior( @@ -162,15 +162,11 @@ def test_spine_assembly_mass_share_fields_name_exact_adapter_sinks( ) -> None: for channel in ("acs", "asec"): field = field_ledger.field( - "/authored/spec~1spine.yaml/assembly/household_mass_shares/" - f"{channel}" + f"/authored/spec~1spine.yaml/assembly/household_mass_shares/{channel}" ) assert field.mode is UsageMode.LEGACY_BEHAVIOR assert field.generation0_effect is Generation0Effect.LEGACY_BEHAVIOR - assert ( - f"/spine_assembly/household_mass_shares/{channel}" - in field.sink_pointers - ) + assert f"/spine_assembly/household_mass_shares/{channel}" in field.sink_pointers def test_copied_surfaces_cannot_rescue_a_missing_calibration_sink( @@ -358,14 +354,10 @@ def mutate(document: dict[str, object]) -> None: ledger = build_field_usage_ledger(mutated, legacy_payload=mutated_legacy) for channel in ("acs", "asec"): field = ledger.field( - "/authored/spec~1spine.yaml/assembly/household_mass_shares/" - f"{channel}" + f"/authored/spec~1spine.yaml/assembly/household_mass_shares/{channel}" ) assert field.claim_id == "spine_assembly_household_mass_shares" - assert ( - f"/spine_assembly/household_mass_shares/{channel}" - in field.sink_pointers - ) + assert f"/spine_assembly/household_mass_shares/{channel}" in field.sink_pointers def test_geography_declaration_mutation_changes_checkpoint_identity( diff --git a/packages/microcosm-build/tests/test_spec_engine_identity_contracts.py b/packages/microcosm-build/tests/test_spec_engine_identity_contracts.py index 4849a2bf4a..8d1f13e22a 100644 --- a/packages/microcosm-build/tests/test_spec_engine_identity_contracts.py +++ b/packages/microcosm-build/tests/test_spec_engine_identity_contracts.py @@ -63,7 +63,7 @@ def test_pipeline_contract_is_an_exact_generation_zero_projection() -> None: == { "artifact_kind": "populace_us_stacked_pool_checkpoint_identity", "schema_version": 1, - "materializer_version": 12, + "materializer_version": 13, "pipeline": "us-stacked-pool", } ) @@ -118,7 +118,7 @@ def test_identity_contract_objects_are_closed_world( load_schema_registry().validate(mutated, "spine.schema.json") -def test_all_53_seed_sites_resolve_to_typed_real_owners( +def test_all_seed_sites_resolve_to_typed_real_owners( identity_documents: tuple[dict[str, object], list[dict[str, object]]], ) -> None: spine, source_stages = identity_documents diff --git a/packages/microcosm-build/tests/test_spec_engine_imputation_semantics.py b/packages/microcosm-build/tests/test_spec_engine_imputation_semantics.py index 749a793b26..de68f686d7 100644 --- a/packages/microcosm-build/tests/test_spec_engine_imputation_semantics.py +++ b/packages/microcosm-build/tests/test_spec_engine_imputation_semantics.py @@ -98,7 +98,7 @@ def test_imputation_projector_matches_live_plans_and_graph_receipts( for key, expected in live.items(): assert canonical_json_bytes(projected[key]) == canonical_json_bytes(expected) assert projected["late_producer_schedule_receipt"]["schedule_sha256"] == ( - "dcf3c6d2eade3449836c49a1dc4d3b8cd395aab9142db700c3c60598fa9c1c79" + "4965c1485c283dec3685f4ca82fa469d8b88a85f82ccd6b39e2adc84bc0e94d6" ) assert projected["overlap_ownership"]["sha256"] == ( "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356" diff --git a/packages/microcosm-build/tests/test_spec_engine_inventory_coverage.py b/packages/microcosm-build/tests/test_spec_engine_inventory_coverage.py index a138f487e4..b38bcade48 100644 --- a/packages/microcosm-build/tests/test_spec_engine_inventory_coverage.py +++ b/packages/microcosm-build/tests/test_spec_engine_inventory_coverage.py @@ -79,7 +79,7 @@ "primary_targets": 65, "producer_authored_outputs": 92, "producer_compiled_outputs": 227, - "producer_inputs": 2_742, + "producer_inputs": 2_749, "producer_nodes": 38, "producer_virtual_resources": 75, "release_rungs": 5, diff --git a/packages/microcosm-build/tests/test_spec_engine_legacy_adapter.py b/packages/microcosm-build/tests/test_spec_engine_legacy_adapter.py index 53f05be22b..6e9ce6c6ec 100644 --- a/packages/microcosm-build/tests/test_spec_engine_legacy_adapter.py +++ b/packages/microcosm-build/tests/test_spec_engine_legacy_adapter.py @@ -195,10 +195,10 @@ def test_adapter_preserves_generation_zero_identity_components( imputation = legacy_payload["imputation"] assert isinstance(imputation, dict) assert legacy_payload["stacked_authority_receipt"]["sha256"] == ( - "3a980927227704d0589f246eef9cd825c2ae84f3a4134ac835e0e5ed39a563ac" + "2c01b975ebaeb06bfa666538178e1c4836fe294035c0ff7f2eed6a75e917bba6" ) assert imputation["late_producer_schedule_receipt"]["schedule_sha256"] == ( - "dcf3c6d2eade3449836c49a1dc4d3b8cd395aab9142db700c3c60598fa9c1c79" + "4965c1485c283dec3685f4ca82fa469d8b88a85f82ccd6b39e2adc84bc0e94d6" ) assert imputation["overlap_ownership"]["sha256"] == ( "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356" diff --git a/packages/microcosm-build/tests/test_spec_engine_stacked_authority_semantics.py b/packages/microcosm-build/tests/test_spec_engine_stacked_authority_semantics.py index c378f3dbdb..e506b7fe48 100644 --- a/packages/microcosm-build/tests/test_spec_engine_stacked_authority_semantics.py +++ b/packages/microcosm-build/tests/test_spec_engine_stacked_authority_semantics.py @@ -73,7 +73,7 @@ def test_authority_projection_is_field_and_byte_identical_to_live_generation_zer assert projected == live assert stacked_identity_bytes(projected) == _canonical_bytes(live) assert projected["sha256"] == ( - "3a980927227704d0589f246eef9cd825c2ae84f3a4134ac835e0e5ed39a563ac" + "2c01b975ebaeb06bfa666538178e1c4836fe294035c0ff7f2eed6a75e917bba6" ) assert { name: component["sha256"] for name, component in projected["components"].items() @@ -88,7 +88,7 @@ def test_authority_projection_is_field_and_byte_identical_to_live_generation_zer "cacc6c11e114dbae3aaa2761cc6b3fcb1191cd9b689b1c2bd096614c51ebff8b" ), "late_producer_schedule": ( - "9b15db577b85c796944e8eb267500d5d662f2a0eee77b25c1e4241c7d9620473" + "59b65046ee0f200b6227c58cb4d43043f7353a4ae7cd181848217514be58c06c" ), "metric_registry": ( "d75cb9b29f8b0a9a085471a11f4c19c32ba04cbe5419053df94ea81cbe6125a9" @@ -191,7 +191,7 @@ def test_checkpoint_projection_is_field_and_byte_identical_to_live_oracle( ] assert ( projected["pool_code"]["late_producer_schedule"]["schedule_sha256"] - == "dcf3c6d2eade3449836c49a1dc4d3b8cd395aab9142db700c3c60598fa9c1c79" + == "4965c1485c283dec3685f4ca82fa469d8b88a85f82ccd6b39e2adc84bc0e94d6" ) diff --git a/packages/microcosm-build/tests/test_us_acs_pums.py b/packages/microcosm-build/tests/test_us_acs_pums.py index 48b656d965..897a4b8801 100644 --- a/packages/microcosm-build/tests/test_us_acs_pums.py +++ b/packages/microcosm-build/tests/test_us_acs_pums.py @@ -65,6 +65,9 @@ def _person( "SSIP": 0, "RETP": 0, "INTP": 0, + "CIT": 1, + "POBP": 6, + "YOEP": None, "PWGTP": 10, } row.update(overrides) @@ -94,7 +97,16 @@ def _source(tmp_path: Path) -> AcsPumsSource: person_zip, { "psam_pusa.csv": [ - _person("2024HU0000002", 1, 20, MAR=4, WAGP=30_000), + _person( + "2024HU0000002", + 1, + 20, + MAR=4, + WAGP=30_000, + CIT=5, + POBP=164, + YOEP=2021, + ), _person( "2024HU0000001", 1, @@ -193,6 +205,14 @@ def test_load_acs_pums_tables_streams_all_csv_members_and_keeps_native_blanks( "2024HU0000002", ] assert pd.isna(tables["person"].loc[2, "WAGP"]) + foreign_born = ( + tables["person"].loc[tables["person"]["SERIALNO"].eq("2024HU0000002")].iloc[0] + ) + assert (foreign_born["CIT"], foreign_born["POBP"], foreign_born["YOEP"]) == ( + 5, + 164, + 2021, + ) assert metadata["vacant_household_rows_dropped"] == 1 assert metadata["household_csv_members"] == ["psam_husa.csv", "psam_husb.csv"] assert metadata["person_csv_members"] == ["psam_pusa.csv", "psam_pusb.csv"] @@ -226,6 +246,9 @@ def test_build_acs_pums_unit_frame_preserves_lineage_geography_and_weights( assert person["source_household_id"].dtype == np.dtype(np.int64) assert pd.api.types.is_string_dtype(person["source_person_id"].dtype) assert person["source_row_id"].dtype == np.dtype(np.int64) + assert person.loc[person["source_household_id"].eq(2), "CIT"].item() == 5 + assert person.loc[person["source_household_id"].eq(2), "POBP"].item() == 164 + assert person.loc[person["source_household_id"].eq(2), "YOEP"].item() == 2021 assert metadata["weighted_household_population"] == pytest.approx(30.0) diff --git a/packages/microcosm-build/tests/test_us_acs_transfer.py b/packages/microcosm-build/tests/test_us_acs_transfer.py index a436f50a86..e8332d734c 100644 --- a/packages/microcosm-build/tests/test_us_acs_transfer.py +++ b/packages/microcosm-build/tests/test_us_acs_transfer.py @@ -11,6 +11,7 @@ import microcosm.build.us_runtime.acs_transfer as acs_transfer_module import microcosm.build.us_runtime.acs_transfer_bank as acs_transfer_bank_module +import microcosm.build.us_runtime.immigration as immigration_module from microcosm.build.frame_checkpoint import write_frame_checkpoint from microcosm.build.serialization_dtypes import CANONICAL_STRING_DTYPE from microcosm.build.us_runtime.acs_transfer import ( @@ -22,6 +23,7 @@ ACS_PERSON_TRANSFER_PREDICTORS, AcsTransferResult, acs_adult_care_qualifying_rows, + acs_transfer_donor_requirements, declared_acs_transfer_target_families, default_acs_transfer_target_families, transfer_acs_inputs, @@ -29,6 +31,11 @@ from microcosm.build.us_runtime.acs_transfer_bank import ( AcsTransferTargetBankStore, ) +from microcosm.build.us_runtime.immigration import ( + HumanitarianDraw, + ImmigrationControls, + UndocumentedControls, +) from microcosm.build.us_runtime.puf_support import clone_us_frame_for_puf_support from microcosm.build.us_runtime.spine_assembly import assemble_spines from microcosm.fit import Regime @@ -84,6 +91,9 @@ def _donor_frame() -> Frame: ), "age": [45.0, 43.0, 30.0, 8.0, 68.0, 66.0, 39.0, 17.0], "is_female": [False, True, True, False, False, True, True, False], + "PRCITSHP": [1, 1, 1, 1, 1, 1, 5, 5], + "PENATVTY": [57, 57, 57, 57, 57, 57, 164, 373], + "PEINUSYR": [0, 0, 0, 0, 0, 0, 28, 24], "is_household_head": [True, False, True, False, True, False, True, False], "employment_income_before_lsr": [ 80_000.0, @@ -208,6 +218,9 @@ def _recipient_frame() -> Frame: ), "age": [41.0, 40.0, 27.0, 72.0, 69.0, 15.0], "is_female": [False, True, True, False, True, False], + "CIT": [1, 1, 5, 1, 1, 5], + "POBP": [6, 6, 164, 36, 36, 373], + "YOEP": [np.nan, np.nan, 2023, np.nan, np.nan, 2015], "is_household_head": [True, False, True, True, False, False], "employment_income_before_lsr": [ 65_000.0, @@ -263,6 +276,90 @@ def _recipient_frame() -> Frame: ) +def _mixed_immigration_recipient() -> Frame: + """One immutable ASEC row followed by ACS rows with paired null targets.""" + + base = _recipient_frame() + tables = {entity: base.table(entity).copy() for entity in base.entities} + person = tables["person"] + for column in ("PRCITSHP", "PENATVTY", "PEINUSYR"): + person[column] = np.nan + immutable_index = person.index[0] + person.loc[immutable_index, ["CIT", "POBP", "YOEP"]] = np.nan + person.loc[immutable_index, ["PRCITSHP", "PENATVTY", "PEINUSYR"]] = [ + 5, + 200, + 27, + ] + person["ssn_card_type"] = pd.Series( + ["OTHER_NON_CITIZEN", *([np.nan] * (len(person) - 1))], + index=person.index, + dtype=object, + ) + person["immigration_status_str"] = pd.Series( + ["PAROLED_ONE_YEAR", *([np.nan] * (len(person) - 1))], + index=person.index, + dtype=object, + ) + return Frame( + tables, + base.schema, + {entity: base.weights_for(entity) for entity in base.weighted_entities}, + base.strata, + mass_log=base.mass_log, + ) + + +def _small_immigration_controls( + *, + ukraine_target: float = 60.0, +) -> ImmigrationControls: + source = "https://example.com/immigration-control" + return ImmigrationControls( + undocumented=UndocumentedControls( + workers=1.0, + students=1.0, + population_anchor=1.0, + sources={ + "undocumented_workers": source, + "undocumented_students": source, + "undocumented_population_anchor": source, + }, + ), + humanitarian=( + HumanitarianDraw( + category="paroled_one_year", + origin="afghanistan", + status="PAROLED_ONE_YEAR", + target=50.0, + source=source, + ), + HumanitarianDraw( + category="paroled_one_year", + origin="ukraine", + status="PAROLED_ONE_YEAR", + target=ukraine_target, + source=source, + ), + HumanitarianDraw( + category="tps", + origin="venezuela", + status="TPS", + target=70.0, + source=source, + ), + ), + ) + + +def _no_humanitarian_controls() -> ImmigrationControls: + controls = _small_immigration_controls() + return ImmigrationControls( + undocumented=controls.undocumented, + humanitarian=(), + ) + + def _replace_column( frame: Frame, entity: str, @@ -677,7 +774,6 @@ def test_explicit_declared_plan_preserves_deferred_geography( lambda: plan, ) monkeypatch.setattr(acs_transfer_module, "QRF", _MeanQRF) - result = transfer_acs_inputs( _recipient_frame(), _donor_frame(), @@ -745,6 +841,59 @@ def test_declared_families_are_independent_of_release_coverage_surface() -> None assert "receives_wic" in production_declared["person"]["model_required_boolean"] +def test_donor_requirements_include_immigration_source_evidence_triplet() -> None: + donor = _with_columns( + _donor_frame(), + "person", + { + "ssn_card_type": ["CITIZEN"] * 6 + ["NONE", "NONE"], + "immigration_status_str": ["CITIZEN"] * 6 + + ["UNDOCUMENTED", "UNDOCUMENTED"], + }, + ) + plan = { + "person": { + "source_operator_immigration": ( + "ssn_card_type", + "immigration_status_str", + ) + } + } + + requirements = acs_transfer_donor_requirements(donor, plan) + + assert {"PRCITSHP", "PENATVTY", "PEINUSYR"}.issubset(requirements["person"]) + + acs_tables = {entity: donor.table(entity).copy() for entity in donor.entities} + acs_person = acs_tables["person"].rename( + columns={"PRCITSHP": "CIT", "PENATVTY": "POBP"} + ) + acs_person = acs_person.drop(columns=["PEINUSYR"]) + acs_person["YOEP"] = [np.nan] * 6 + [2023, 2015] + acs_tables["person"] = acs_person + acs_donor = Frame( + acs_tables, + donor.schema, + {entity: donor.weights_for(entity) for entity in donor.weighted_entities}, + donor.strata, + mass_log=donor.mass_log, + ) + acs_requirements = acs_transfer_donor_requirements(acs_donor, plan) + assert {"CIT", "POBP", "YOEP"}.issubset(acs_requirements["person"]) + + tables = {entity: donor.table(entity).copy() for entity in donor.entities} + tables["person"] = tables["person"].drop(columns=["PEINUSYR"]) + incomplete = Frame( + tables, + donor.schema, + {entity: donor.weights_for(entity) for entity in donor.weighted_entities}, + donor.strata, + mass_log=donor.mass_log, + ) + with pytest.raises(ValueError, match="incomplete donor evidence triplet"): + acs_transfer_donor_requirements(incomplete, plan) + + def test_declared_plan_carries_the_23_stage_base_surface() -> None: """The M/N/O donor repairs must reach the ACS spine through the plan.""" @@ -871,6 +1020,244 @@ def test_explicit_transfer_adds_requested_model_inputs( and "immigration_status_str" not in call["targets"] for call in joint_calls ) + assert { + "__acs_transfer_is_us_citizen", + "__acs_transfer_birth_country_code", + "__acs_transfer_arrival_year", + }.issubset(joint_calls[0]["predictors"]) + + +def test_joint_immigration_codec_never_emits_constrained_baseline( + monkeypatch: pytest.MonkeyPatch, +) -> None: + constrained = [ + "PAROLED_ONE_YEAR", + "REFUGEE", + "ASYLEE", + "DEPORTATION_WITHHELD", + "TPS", + "CUBAN_HAITIAN_ENTRANT", + "DACA", + "REFUGEE", + ] + donor = _with_columns( + _donor_frame(), + "person", + { + "ssn_card_type": ["OTHER_NON_CITIZEN"] * 8, + "immigration_status_str": constrained, + }, + ) + monkeypatch.setattr(acs_transfer_module, "QRF", _MeanQRF) + monkeypatch.setattr( + immigration_module, + "us_immigration_controls", + _no_humanitarian_controls, + ) + + result = transfer_acs_inputs( + _recipient_frame(), + donor, + target_families={ + "person": { + "source_operator_immigration": ( + "ssn_card_type", + "immigration_status_str", + ) + } + }, + donor_channel=None, + n_estimators=1, + ) + + assert set(result.frame.person["immigration_status_str"]) <= { + "CITIZEN", + "LEGAL_PERMANENT_RESIDENT", + } + encoding = acs_transfer_module._target_encodings( + donor.person, + targets=("ssn_card_type", "immigration_status_str"), + )["immigration_status_str"] + assert not {pair[1] for pair in encoding.categories}.intersection(constrained) + + +def test_transfer_rederives_entrant_and_daca_only_on_compatible_acs_cohorts( + monkeypatch: pytest.MonkeyPatch, +) -> None: + donor = _with_columns( + _donor_frame(), + "person", + { + "ssn_card_type": ["NON_CITIZEN_VALID_EAD"] * 8, + "immigration_status_str": [ + "CUBAN_HAITIAN_ENTRANT", + "DACA", + ] + * 4, + }, + ) + recipient = _with_columns( + _recipient_frame(), + "person", + { + "age": [50.0, 50.0, 29.0, 50.0, 40.0, 70.0], + "CIT": [5, 5, 5, 5, 1, 5], + "POBP": [327, 164, 312, 312, 6, 332], + "YOEP": [2000, 2020, 2005, 2020, np.nan, 1970], + }, + ) + monkeypatch.setattr(acs_transfer_module, "QRF", _MeanQRF) + monkeypatch.setattr( + immigration_module, + "us_immigration_controls", + _no_humanitarian_controls, + ) + + result = transfer_acs_inputs( + recipient, + donor, + target_families={ + "person": { + "source_operator_immigration": ( + "ssn_card_type", + "immigration_status_str", + ) + } + }, + donor_channel=None, + n_estimators=1, + ) + + assert result.frame.person["immigration_status_str"].tolist() == [ + "CUBAN_HAITIAN_ENTRANT", + "LEGAL_PERMANENT_RESIDENT", + "DACA", + "LEGAL_PERMANENT_RESIDENT", + "CITIZEN", + "LEGAL_PERMANENT_RESIDENT", + ] + receipt = next( + item.reconciliation + for item in result.imputed_inputs + if item.column == "immigration_status_str" + ) + assert receipt is not None + assert receipt["special_status_assignments"] == 2 + + +def test_humanitarian_reconciliation_uses_residual_targets_and_records_receipt( + monkeypatch: pytest.MonkeyPatch, +) -> None: + donor = _with_columns( + _donor_frame(), + "person", + { + "ssn_card_type": ["OTHER_NON_CITIZEN"] * 8, + "immigration_status_str": ["LEGAL_PERMANENT_RESIDENT"] * 8, + }, + ) + monkeypatch.setattr(acs_transfer_module, "QRF", _MeanQRF) + monkeypatch.setattr( + immigration_module, + "us_immigration_controls", + _small_immigration_controls, + ) + + result = transfer_acs_inputs( + _mixed_immigration_recipient(), + donor, + target_families={ + "person": { + "source_operator_immigration": ( + "ssn_card_type", + "immigration_status_str", + ) + } + }, + donor_channel=None, + seed=19, + n_estimators=1, + ) + + person = result.frame.person + assert person["immigration_status_str"].tolist() == [ + "PAROLED_ONE_YEAR", # immutable Afghanistan contribution + "CITIZEN", # source CIT repairs the imputed baseline pair + "PAROLED_ONE_YEAR", # compatible Ukrainian residual + "CITIZEN", + "CITIZEN", + "TPS", # compatible Venezuelan residual + ] + assert person.loc[person.index[0], "ssn_card_type"] == "OTHER_NON_CITIZEN" + reconciled = [ + item + for item in result.imputed_inputs + if item.column in {"ssn_card_type", "immigration_status_str"} + ] + assert len(reconciled) == 2 + assert reconciled[0].reconciliation == reconciled[1].reconciliation + receipt = reconciled[0].reconciliation + assert receipt is not None + assert receipt["mutable_rows"] == 5 + assert receipt["immutable_rows"] == 1 + draws = receipt["draws"] + assert draws["paroled_one_year:afghanistan"] == { + "target": 50.0, + "immutable_population": 50.0, + "residual_target": 0.0, + "eligible_recipient_population": 0.0, + "selected_recipient_population": 0.0, + "achieved_population": 50.0, + "absolute_error": 0.0, + "selection_threshold_tie_population": 0.0, + "residual_selection_error": 0.0, + "within_residual_discrete_weight_bound": True, + "immutable_overshoot": 0.0, + "within_discrete_weight_bound": True, + } + assert draws["paroled_one_year:ukraine"]["residual_target"] == 60.0 + assert draws["paroled_one_year:ukraine"]["achieved_population"] == 60.0 + assert draws["tps:venezuela"]["achieved_population"] == 70.0 + assert all(draw["within_discrete_weight_bound"] for draw in draws.values()) + + +def test_humanitarian_reconciliation_fails_on_candidate_shortfall( + monkeypatch: pytest.MonkeyPatch, +) -> None: + donor = _with_columns( + _donor_frame(), + "person", + { + "ssn_card_type": ["OTHER_NON_CITIZEN"] * 8, + "immigration_status_str": ["LEGAL_PERMANENT_RESIDENT"] * 8, + }, + ) + monkeypatch.setattr(acs_transfer_module, "QRF", _MeanQRF) + monkeypatch.setattr( + immigration_module, + "us_immigration_controls", + lambda: _small_immigration_controls(ukraine_target=61.0), + ) + + with pytest.raises( + ValueError, + match="candidate shortfall.*paroled_one_year:ukraine", + ): + transfer_acs_inputs( + _mixed_immigration_recipient(), + donor, + target_families={ + "person": { + "source_operator_immigration": ( + "ssn_card_type", + "immigration_status_str", + ) + } + }, + donor_channel=None, + seed=19, + n_estimators=1, + ) def test_large_target_family_is_split_to_bound_retained_qrf_forests( @@ -1158,6 +1545,11 @@ def test_target_bank_resumes_joint_immigration_codec_as_one_model_target( tmp_path: Path, ) -> None: _lock_bank_fixture_threads(monkeypatch) + monkeypatch.setattr( + immigration_module, + "us_immigration_controls", + _no_humanitarian_controls, + ) donor = _with_columns( _donor_frame(), "person", diff --git a/packages/microcosm-build/tests/test_us_bundle_core_contracts.py b/packages/microcosm-build/tests/test_us_bundle_core_contracts.py index 74ad1b2859..a4a83b7646 100644 --- a/packages/microcosm-build/tests/test_us_bundle_core_contracts.py +++ b/packages/microcosm-build/tests/test_us_bundle_core_contracts.py @@ -129,7 +129,7 @@ def test_source_surface_classification_is_complete() -> None: } assert normative["stage_asset"] == { "id": "source_stages", - "sha256": "619a974abd80c34201a0ccf813968ebc09ed52e144ff1d39722a4d9be7bd7821", + "sha256": "413174820cfb57f85783831dbbf445f007d06128d31562a1234e998e627bd3aa", } assert operational["stage_asset"] == { "path": "microcosm.build.us/source_stages.json" @@ -180,15 +180,12 @@ def test_cd_vintage_crosswalk_source_and_geography_authority_are_pinned() -> Non crosswalk_source = next( row for row in build_sources()["sources"] - if row["id"] - == "us_congressional_district_vintage_crosswalk_117_to_119" + if row["id"] == "us_congressional_district_vintage_crosswalk_117_to_119" ) assert crosswalk_source == { "id": "us_congressional_district_vintage_crosswalk_117_to_119", "role": "congressional_district_vintage_crosswalk", - "sha256": ( - "c7cb040b1f57ca2ea2adcbfe60cc2b250ca23acbc4b640cd421e766fa54c1aec" - ), + "sha256": ("c7cb040b1f57ca2ea2adcbfe60cc2b250ca23acbc4b640cd421e766fa54c1aec"), "byte_size": 77_935, "loader": "kernel:load_congressional_district_vintage_crosswalk", "vintages": ["vintage:cd_117", "vintage:cd_119"], @@ -204,9 +201,7 @@ def test_cd_vintage_crosswalk_source_and_geography_authority_are_pinned() -> Non assignment = build_geography()["assignment"] assert assignment["order"] == "before_gap_fill" assert assignment["congressional_district_vintage_crosswalk"] == { - "source_ref": ( - "source:us_congressional_district_vintage_crosswalk_117_to_119" - ), + "source_ref": ("source:us_congressional_district_vintage_crosswalk_117_to_119"), "source_vintage": "vintage:cd_117", "target_vintage": "vintage:cd_119", } @@ -217,9 +212,9 @@ def test_cd_vintage_crosswalk_source_reference_is_resolved() -> None: resources = {**_vintage_resources(), "geography": build_geography()} _resolve_vintages(resources) - resources["geography"]["assignment"][ - "congressional_district_vintage_crosswalk" - ]["source_ref"] = "source:missing_crosswalk" + resources["geography"]["assignment"]["congressional_district_vintage_crosswalk"][ + "source_ref" + ] = "source:missing_crosswalk" with pytest.raises(SpecResolutionError, match="dangling source reference"): _resolve_vintages(resources) diff --git a/packages/microcosm-build/tests/test_us_fiscal_refresh_builder.py b/packages/microcosm-build/tests/test_us_fiscal_refresh_builder.py index 3c76b46204..af8d1a3774 100644 --- a/packages/microcosm-build/tests/test_us_fiscal_refresh_builder.py +++ b/packages/microcosm-build/tests/test_us_fiscal_refresh_builder.py @@ -508,10 +508,10 @@ def test__given_target_frame_checkpoint__then_builder_round_trips_frame( ssi_take_up_assignment_sha256="ssi-flags-sha", selection_identities_sha256=None, ) - # 11 = target checkpoints preserve nullable booleans explicitly; schema 2 - # distinguishes the new values+mask codec from schema-1 checkpoints. + # 12 = target checkpoints include source-aware humanitarian-stock columns; + # schema 2 distinguishes the values+mask codec from schema-1 checkpoints. assert identity["schema_version"] == 2 - assert identity["materializer_version"] == 11 + assert identity["materializer_version"] == 12 # The SSI prior-weight basis is identity-bearing (microcosm#543 instance # 2): unflagged runs carry the key as None. assert identity["ssi_take_up_prior_weight_basis_sha256"] is None @@ -723,9 +723,9 @@ def test__given_stale_materializer_version_checkpoint__then_builder_rejects_it( ) -> None: """A checkpoint stored under a superseded materializer version must not load. - Version 11 adds the lossless nullable-boolean checkpoint codec. The version - constant participates in the identity comparison; this pins stored-10 - versus current-11 rejection directly. + Version 12 adds source-aware humanitarian-stock target columns. The version + constant participates in the identity comparison; this pins stored-11 + versus current-12 rejection directly. """ builder = _load_builder_module() monkeypatch.setattr(builder, "US_SCHEMA", small_frame.schema) @@ -759,10 +759,10 @@ def test__given_stale_materializer_version_checkpoint__then_builder_rejects_it( ssi_take_up_assignment_sha256="ssi-flags-sha", selection_identities_sha256=None, ) - # 10 = the pre-nullable-boolean-codec world; 9 = the still-older pre-#557 - # release-refit world. Both must miss against expected version 11. - stale_identity = {**dict(identity), "materializer_version": 10} - older_identity = {**dict(identity), "materializer_version": 9} + # 11 = the pre-humanitarian-target world; 10 = the still-older + # pre-nullable-boolean-codec world. Both must miss against version 12. + stale_identity = {**dict(identity), "materializer_version": 11} + older_identity = {**dict(identity), "materializer_version": 10} path = tmp_path / "target_frame_checkpoint.h5" builder._write_target_frame_checkpoint( path, @@ -1743,20 +1743,16 @@ def selected(path, *, allow_terminal_gate_failure): ) return - loaded_frame, receipt, loaded_identity = ( - builder._load_base_pool_if_identified( - pool_h5, - allow_gate_failed_base_pool=allow_gate_failed, - ) + loaded_frame, receipt, loaded_identity = builder._load_base_pool_if_identified( + pool_h5, + allow_gate_failed_base_pool=allow_gate_failed, ) assert loaded_frame is frame assert loaded_identity is authenticated assert receipt["status"] == status assert receipt["allow_gate_failed_base_pool"] is allow_gate_failed - assert receipt["agreement_gate_reference"]["failure_count"] == len( - gate_failures - ) + assert receipt["agreement_gate_reference"]["failure_count"] == len(gate_failures) def test_builder_refuses_actual_red_base_h5_pool_sidecar_without_opt_in( @@ -1777,7 +1773,9 @@ def test_builder_refuses_actual_red_base_h5_pool_sidecar_without_opt_in( ) out = tmp_path / "out" monkeypatch.setattr(builder, "_git_dirty", lambda: False) - monkeypatch.setattr(builder, "_refuse_certified_release_dir_reuse", lambda path: None) + monkeypatch.setattr( + builder, "_refuse_certified_release_dir_reuse", lambda path: None + ) monkeypatch.setattr( builder, "_load_frame", @@ -1826,7 +1824,9 @@ def test_builder_refuses_bare_stamped_pool_h5_before_generic_load( ) out = tmp_path / "out" monkeypatch.setattr(builder, "_git_dirty", lambda: False) - monkeypatch.setattr(builder, "_refuse_certified_release_dir_reuse", lambda path: None) + monkeypatch.setattr( + builder, "_refuse_certified_release_dir_reuse", lambda path: None + ) monkeypatch.setattr( builder, "_load_frame", @@ -4113,7 +4113,15 @@ def test_release_calibration_diagnostics_writes_nan_final_loss_as_null( @pytest.mark.parametrize( "terminal_mode", - ["merge", "integrity", "retirement", "crash", "telemetry", "puf_tail"], + [ + "merge", + "integrity", + "retirement", + "crash", + "telemetry", + "puf_tail", + "immigration_drift", + ], ) def test_main_writes_diagnostics_before_post_calibration_gate_failure( monkeypatch, tmp_path, terminal_mode @@ -4141,6 +4149,10 @@ def test_main_writes_diagnostics_before_post_calibration_gate_failure( ``puf_tail``: exact-k selection loses the original PUF capital-gains tail; the failure is batched while diagnostics and final-weight evidence remain, every later terminal group runs, and release artifacts stay suppressed. + ``immigration_drift``: the source-frame immigration composition passes, then + calibrated weights make the same selected status mix fail on the final + export-frame recheck; that final verdict reaches diagnostics and aborts + publication. """ builder = _load_builder_module() release_id = ( @@ -4193,9 +4205,23 @@ def n(self, entity): return 2 if terminal_mode == "puf_tail" else 4 def table(self, entity): - assert entity == "household" size = self.n("household") - return pd.DataFrame({"household_id": np.arange(1, size + 1, dtype="int64")}) + if entity == "household": + return pd.DataFrame( + {"household_id": np.arange(1, size + 1, dtype="int64")} + ) + assert entity == "person" + return pd.DataFrame( + { + "household_id": np.arange(1, size + 1, dtype="int64"), + "immigration_status_str": [ + "REFUGEE", + "LEGAL_PERMANENT_RESIDENT", + "ASYLEE", + "LEGAL_PERMANENT_RESIDENT", + ][:size], + } + ) def weights_for(self, entity): assert entity == "household" @@ -4217,8 +4243,20 @@ def weights_for(self, entity): ) def table(self, entity): - assert entity == "household" - return pd.DataFrame({"household_id": np.asarray([10, 20], dtype="int64")}) + if entity == "household": + return pd.DataFrame( + {"household_id": np.asarray([10, 20], dtype="int64")} + ) + assert entity == "person" + return pd.DataFrame( + { + "household_id": np.asarray([10, 20], dtype="int64"), + "immigration_status_str": [ + "REFUGEE", + "LEGAL_PERMANENT_RESIDENT", + ], + } + ) if terminal_mode == "puf_tail": loss_basis = builder._fiscal_target_loss_basis(registry, np.ones(1)) @@ -4356,7 +4394,13 @@ def complete(self): "_staging_telemetry", lambda *args, **kwargs: live_telemetry, ) - if terminal_mode in {"integrity", "retirement", "telemetry", "puf_tail"}: + if terminal_mode in { + "integrity", + "retirement", + "telemetry", + "puf_tail", + "immigration_drift", + }: monkeypatch.setattr( builder, "PolicyEngineUSEngine", @@ -4817,14 +4861,50 @@ def fake_energy_subsidy_signal_gate(frame): "with_us_immigration_inputs", lambda frame, *, seed, time_period: frame, ) + + def fake_immigration_composition_gate(frame): + statuses = frame.table("person")["immigration_status_str"].astype(str) + household_weights = frame.weights_for("household") + weights = np.asarray(household_weights.values, dtype=np.float64) + assert len(statuses) == len(weights) + humanitarian = statuses.isin( + {"REFUGEE", "ASYLEE", "DEPORTATION_WITHHELD", "CUBAN_HAITIAN_ENTRANT"} + ).to_numpy(dtype=bool) + humanitarian_share = float(np.average(humanitarian, weights=weights)) + calls = captured.setdefault("immigration_gate_calls", []) + is_final_drift = ( + terminal_mode == "immigration_drift" + and household_weights.kind == WeightKind.CALIBRATED + ) + calls.append( + { + "weight_kind": household_weights.kind.value, + "weights": weights.tolist(), + "statuses": statuses.tolist(), + "humanitarian_share": humanitarian_share, + "passed": not is_final_drift, + } + ) + failure = ( + "weighted humanitarian status share drifted from 0.500000 to " + f"{humanitarian_share:.6f} after calibration " + "[final-immigration-sentinel]" + ) + return builder.GateResult( + name="immigration_composition", + passed=not is_final_drift, + failures=(failure,) if is_final_drift else (), + details={ + "checked": True, + "weight_kind": household_weights.kind.value, + "humanitarian_share": humanitarian_share, + }, + ) + monkeypatch.setattr( builder, "us_immigration_composition_gate", - lambda frame: builder.GateResult( - name="immigration_composition", - passed=True, - details={"checked": True}, - ), + fake_immigration_composition_gate, ) monkeypatch.setattr( builder, @@ -5561,6 +5641,12 @@ def fake_other_health_insurance_signal_gate(frame): passed=True, details={"checked": True}, ) + if terminal_mode == "immigration_drift": + return builder.GateResult( + name="other_health_insurance_premiums_signal", + passed=True, + details={"checked": True}, + ) # Export-frame call fails deliberately: the microcosm#547 cofailure # regression proves a failing post-solve signal gate batches # alongside the SSI delivery failure instead of masking it with an @@ -5750,7 +5836,12 @@ def fake_ssi_delivery_gate(diagnostics, *, targets, enforcement_fences=None): # own early failure. Other modes retain the microcosm#547 delivery # cofailure and its written retry basis. captured.setdefault("ssi_event_order", []).append("delivery_gate") - passes = terminal_mode in {"integrity", "retirement", "puf_tail"} + passes = terminal_mode in { + "integrity", + "retirement", + "puf_tail", + "immigration_drift", + } return builder.GateResult( name="ssi_take_up_delivery", passed=passes, @@ -5788,6 +5879,14 @@ def recording_ssi_write(diagnostics, path): def fake_release_gate_failures(*args, **kwargs): if terminal_mode == "crash": raise RuntimeError("release-gate evaluation exploded [crash-sentinel]") + if terminal_mode == "immigration_drift": + final_immigration_gate = args[6] + assert final_immigration_gate.name == "immigration_composition" + assert not final_immigration_gate.passed + return [ + f"Immigration composition failed: {failure}" + for failure in final_immigration_gate.failures + ] if terminal_mode == "retirement": # The degraded pre-solve contract (PR #557 round 3): the failing # degenerate gate is NOT raised early — the gate object itself @@ -5856,16 +5955,23 @@ def fake_release_gate_failures(*args, **kwargs): "Bernoulli-law violation [final-integrity-sentinel]" ) assert "SSI take-up delivery failed:" not in message + elif terminal_mode == "immigration_drift": + assert message == ( + "Release gates failed: Immigration composition failed: weighted " + "humanitarian status share drifted from 0.500000 to 0.255319 " + "after calibration [final-immigration-sentinel]" + ) else: assert message.startswith( "Release gates failed: SSI take-up delivery failed: " "18_64 delivered over envelope [cofailure-sentinel]" ) - assert ( - "Other health insurance signal failed on the export frame: " - "premiums signal flattened [cofailure-sentinel]" in message - ) - if terminal_mode != "crash": + if terminal_mode != "immigration_drift": + assert ( + "Other health insurance signal failed on the export frame: " + "premiums signal flattened [cofailure-sentinel]" in message + ) + if terminal_mode not in {"crash", "immigration_drift"}: assert "ctc failed" in message if terminal_mode == "telemetry": assert ( @@ -5873,13 +5979,33 @@ def fake_release_gate_failures(*args, **kwargs): "attach_artifact('calibration_diagnostics') crashed" in message ) assert "telemetry-crash-sentinel" in message - else: + elif terminal_mode == "crash": assert "health-input exploded [crash-sentinel]" in message assert "release-gate evaluation exploded [crash-sentinel]" in message assert "ctc failed" not in message else: # pragma: no cover - defensive assertion raise AssertionError("Expected post-calibration gate failure.") + if terminal_mode == "immigration_drift": + immigration_calls = captured["immigration_gate_calls"] + assert len(immigration_calls) == 2 + source_call, final_call = immigration_calls + assert source_call["weight_kind"] == "importance" + assert source_call["weights"] == [1.0, 1.0, 1.0, 1.0] + assert source_call["statuses"] == [ + "REFUGEE", + "LEGAL_PERMANENT_RESIDENT", + "ASYLEE", + "LEGAL_PERMANENT_RESIDENT", + ] + assert source_call["humanitarian_share"] == pytest.approx(0.5) + assert source_call["passed"] is True + assert final_call["weight_kind"] == "calibrated" + assert final_call["weights"] == [12.0, 35.0] + assert final_call["statuses"] == ["REFUGEE", "LEGAL_PERMANENT_RESIDENT"] + assert final_call["humanitarian_share"] == pytest.approx(12.0 / 47.0) + assert final_call["passed"] is False + release_dir = out / "releases" / release_id written_diagnostics = json.loads( (release_dir / "calibration_diagnostics.json").read_text() @@ -5902,6 +6028,12 @@ def fake_release_gate_failures(*args, **kwargs): "Bernoulli-law violation [final-integrity-sentinel]" in written_diagnostics["build"]["release_gates"]["failures"] ) + elif terminal_mode == "immigration_drift": + assert written_diagnostics["build"]["release_gates"]["failures"] == [ + "Immigration composition failed: weighted humanitarian status share " + "drifted from 0.500000 to 0.255319 after calibration " + "[final-immigration-sentinel]" + ] # The SSI retry-basis artifact is written even though the run fails # terminally — it IS the remedy input for the next attempt. assert (release_dir / "us_ssi_take_up.json").exists() @@ -5926,6 +6058,15 @@ def fake_release_gate_failures(*args, **kwargs): cache_context = captured["materialize_kwargs"][ "target_materialization_cache_context" ] + active_specs = captured["ssi_band_targets_specs"] + assert any( + spec.metadata.get("target_role") == "humanitarian_immigration_stock" + for spec in active_specs + ) + assert ( + cache_context["target_registry_version"] + == TargetRegistry(active_specs, country="us").version + ) expected_evidence_identity = builder._target_frame_checkpoint_identity( base_dataset_sha256=cache_context["base_dataset_sha256"], policyengine_us_version=cache_context["policyengine_us_version"], @@ -5987,7 +6128,13 @@ def fake_release_gate_failures(*args, **kwargs): "source_sha256": builder.ASEC_2023_WEEKS_UNEMPLOYED_SOURCE_SHA256, "source_rows": 2, } - if terminal_mode in {"integrity", "retirement", "telemetry", "puf_tail"}: + if terminal_mode in { + "integrity", + "retirement", + "telemetry", + "puf_tail", + "immigration_drift", + }: assert captured["terminal_gate_events"] == [ "input_coverage", "input_mass_parity", @@ -6028,6 +6175,12 @@ def fake_release_gate_failures(*args, **kwargs): "premiums signal flattened [cofailure-sentinel]", "ctc failed", ] + elif terminal_mode == "immigration_drift": + expected_gate_failures = [ + "Immigration composition failed: weighted humanitarian status share " + "drifted from 0.500000 to 0.255319 after calibration " + "[final-immigration-sentinel]" + ] else: # The retry line carries the written artifact's sha256 — the # required --ssi-take-up-prior-weight-basis-sha256 pin, handed out @@ -6261,7 +6414,7 @@ def fake_release_gate_failures(*args, **kwargs): "integrity_gate", # persisted-flag recheck on the export frame "delivery_gate", # enforced-band delivery, after the artifact exists ] - if terminal_mode not in {"integrity", "retirement"}: + if terminal_mode not in {"integrity", "retirement", "immigration_drift"}: # A delivery miss rewrites the final measurement as the retry basis. expected_ssi_event_order.append("write:us_ssi_take_up.json") assert captured["ssi_event_order"] == expected_ssi_event_order @@ -8702,6 +8855,236 @@ def _invalidate_all_caches(self): assert compilation["dropped_target_names"] == [] +def test_humanitarian_stock_specs_follow_positive_manifest_draws( + monkeypatch, +) -> None: + builder = _load_builder_module() + draws = ( + SimpleNamespace( + label="paroled_one_year:afghanistan", + category="paroled_one_year", + origin="afghanistan", + status="PAROLED_ONE_YEAR", + target=75_000.0, + source="https://example.test/parole", + ), + SimpleNamespace( + label="refugee", + category="refugee", + origin=None, + status="REFUGEE", + target=160_000.0, + source="https://example.test/refugee", + ), + SimpleNamespace( + label="deportation_withheld", + category="deportation_withheld", + origin=None, + status="DEPORTATION_WITHHELD", + target=0.0, + source="https://example.test/withheld", + ), + SimpleNamespace( + label="tps:venezuela", + category="tps", + origin="venezuela", + status="TPS", + target=344_335.0, + source="https://example.test/tps", + ), + ) + monkeypatch.setattr( + builder, + "us_immigration_controls", + lambda: SimpleNamespace(humanitarian=draws), + ) + + specs = builder._humanitarian_immigration_stock_specs(time_period=2024) + + assert [spec.name for spec in specs] == [ + "humanitarian_immigration_stock.paroled_one_year.afghanistan", + "humanitarian_immigration_stock.refugee", + "humanitarian_immigration_stock.tps.venezuela", + ] + assert [spec.value for spec in specs] == [75_000.0, 160_000.0, 344_335.0] + assert [spec.source for spec in specs] == [ + "https://example.test/parole", + "https://example.test/refugee", + "https://example.test/tps", + ] + assert all(spec.period == 2024 for spec in specs) + assert all( + spec.metadata["materializer"] == "humanitarian_immigration_stock" + for spec in specs + ) + assert specs[0].metadata["humanitarian_origin"] == "afghanistan" + assert "humanitarian_origin" not in specs[1].metadata + # The active-registry content hash therefore binds the manifest controls; + # removing even one draw produces a different checkpoint registry id. + assert ( + TargetRegistry(specs, country="us").version + != TargetRegistry(specs[:-1], country="us").version + ) + + +def test_humanitarian_stock_materializer_collapses_source_aware_person_masks( + monkeypatch, + small_frame, +) -> None: + from microcosm.calibrate import build_constraint_matrix + + builder = _load_builder_module() + draws = ( + SimpleNamespace( + label="paroled_one_year:afghanistan", + category="paroled_one_year", + origin="afghanistan", + status="PAROLED_ONE_YEAR", + target=1.0, + source="fixture", + ), + SimpleNamespace( + label="refugee", + category="refugee", + origin=None, + status="REFUGEE", + target=2.0, + source="fixture", + ), + ) + controls = SimpleNamespace(humanitarian=draws) + monkeypatch.setattr(builder, "us_immigration_controls", lambda: controls) + observed: list[tuple[str, int]] = [] + + def draw_mask(frame, draw, *, time_period): + assert frame is small_frame + observed.append((draw.label, time_period)) + return { + "paroled_one_year:afghanistan": np.asarray([True, False, False, False]), + "refugee": np.asarray([False, True, True, False]), + }[draw.label] + + monkeypatch.setattr(builder, "us_immigration_humanitarian_draw_mask", draw_mask) + specs = builder._humanitarian_immigration_stock_specs(time_period=2024) + household = small_frame.table("household").copy() + + builder._materialize_humanitarian_immigration_stock_targets( + frame=small_frame, + household=household, + target_specs=specs, + time_period=2024, + ) + + np.testing.assert_array_equal( + household[ + "humanitarian_immigration_stock.paroled_one_year.afghanistan" + ].to_numpy(), + np.asarray([1.0, 0.0]), + ) + np.testing.assert_array_equal( + household["humanitarian_immigration_stock.refugee"].to_numpy(), + np.asarray([1.0, 1.0]), + ) + assert observed == [ + ("paroled_one_year:afghanistan", 2024), + ("refugee", 2024), + ] + + target_frame = Frame( + { + "person": small_frame.table("person"), + "household": household, + }, + small_frame.schema, + {"household": small_frame.weights_for("household")}, + small_frame.strata, + ) + registry, compilation = builder._compile_materialized_target_registry( + target_frame, + specs, + ) + target_set = registry.to_target_set() + + assert compilation["dropped_target_names"] == [] + assert tuple(target.row_name for target in target_set) == ( + "humanitarian_immigration_stock.paroled_one_year.afghanistan@2024", + "humanitarian_immigration_stock.refugee@2024", + ) + problem = build_constraint_matrix(target_frame, target_set) + assert problem.names == tuple(target.row_name for target in target_set) + assert problem.skipped == () + np.testing.assert_array_equal( + problem.matrix.toarray(), + np.asarray([[1.0, 0.0], [1.0, 1.0]]), + ) + np.testing.assert_array_equal( + problem.target_vector, + np.asarray([1.0, 2.0]), + ) + + +def test_dense_and_l0_solvers_consume_compiled_registry_target_set() -> None: + import ast + + builder = _load_builder_module() + tree = ast.parse(Path(builder.__file__).read_text()) + main_fn = next( + node + for node in ast.walk(tree) + if isinstance(node, ast.FunctionDef) and node.name == "_main" + ) + + for solver_name in ("calibrate", "calibrate_l0_refit"): + calls = [ + node + for node in ast.walk(main_fn) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == solver_name + ] + assert len(calls) == 1 + assert ast.unparse(calls[0].args[0]) == "target_frame" + assert ast.unparse(calls[0].args[1]) == "registry.to_target_set()" + + +def test_main_rechecks_immigration_composition_on_export_for_terminal_gates() -> None: + import ast + + builder = _load_builder_module() + tree = ast.parse(Path(builder.__file__).read_text()) + main_fn = next( + node + for node in ast.walk(tree) + if isinstance(node, ast.FunctionDef) and node.name == "_main" + ) + final_assignments = [ + node + for node in ast.walk(main_fn) + if isinstance(node, ast.Assign) + and any( + isinstance(target, ast.Name) and target.id == "final_immigration_gate" + for target in node.targets + ) + ] + assert len(final_assignments) == 1 + final_assignment = final_assignments[0] + assert isinstance(final_assignment.value, ast.Call) + assert getattr(final_assignment.value.func, "id", None) == ( + "us_immigration_composition_gate" + ) + assert ast.unparse(final_assignment.value.args[0]) == "export_frame" + + terminal_calls = [ + node + for node in ast.walk(main_fn) + if isinstance(node, ast.Call) + and getattr(node.func, "id", None) == "_release_gate_failures" + and node.lineno > final_assignment.lineno + ] + assert len(terminal_calls) == 1 + assert ast.unparse(terminal_calls[0].args[6]) == "final_immigration_gate" + + def test_unknown_ledger_filter_metadata_fails_closed() -> None: builder = _load_builder_module() target = TargetSpec( @@ -9022,9 +9405,7 @@ def test_exact_k_receipt_stays_strict_even_when_base_h5_opt_in_is_present() -> N builder = _load_builder_module() with pytest.raises(RuntimeError, match="lost its passing agreement gate"): - builder._exact_k_ladder_manifest_payload( - **_gate_failed_exact_k_inputs(builder) - ) + builder._exact_k_ladder_manifest_payload(**_gate_failed_exact_k_inputs(builder)) def _gate_failed_base_pool_receipt() -> dict[str, object]: diff --git a/packages/microcosm-build/tests/test_us_immigration.py b/packages/microcosm-build/tests/test_us_immigration.py index 3040dec198..18618f7d7c 100644 --- a/packages/microcosm-build/tests/test_us_immigration.py +++ b/packages/microcosm-build/tests/test_us_immigration.py @@ -6,6 +6,7 @@ import pandas as pd import pytest +import microcosm.build.us_runtime.immigration as immigration_module from microcosm.build.source_manifest import SourceStageSpec from microcosm.build.source_runtime import ( SourceRuntimeConfig, @@ -30,6 +31,10 @@ us_immigration_stage_spec, with_us_immigration_inputs, ) +from microcosm.build.us_runtime.immigration import ( + us_immigration_controls, + us_immigration_humanitarian_draw_mask, +) from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights _HANDLERS = {"derive_immigration_status": derive_us_immigration_status_from_manifest} @@ -288,6 +293,10 @@ def test_prcitshp_outside_domain_raises(self) -> None: with pytest.raises(SourceRuntimeError, match="PRCITSHP"): _run(_person_table([{"PRCITSHP": 7}])) + def test_2024_asec_rejects_nonexistent_peinusyr_29(self) -> None: + with pytest.raises(SourceRuntimeError, match=r"PEINUSYR.*0\.\.28"): + _run(_person_table([_noncitizen(PEINUSYR=29)])) + def test_missing_required_column_raises(self) -> None: person = _person_table([_noncitizen()]).drop(columns=["PEINUSYR"]) with pytest.raises(SourceRuntimeError, match="PEINUSYR"): @@ -405,6 +414,263 @@ def test_parole_needs_recent_arrival(self) -> None: ) assert output.loc[0, "immigration_status_str"] == "LEGAL_PERMANENT_RESIDENT" + def test_u4u_rejects_2020_2021_ukrainian_arrivals(self) -> None: + person = _person_table( + [ + _noncitizen(PENATVTY=164, PEINUSYR=27, CAID=1), + _noncitizen(PENATVTY=164, PEINUSYR=28, CAID=1), + ] + ) + output = _run( + person, + humanitarian=_humanitarian_block(paroled_one_year={"ukraine": 2.0}), + ) + assert output["immigration_status_str"].tolist() == [ + "LEGAL_PERMANENT_RESIDENT", + "PAROLED_ONE_YEAR", + ] + + compatibility = _us_frame( + [ + _noncitizen( + PENATVTY=164, + PEINUSYR=27, + ssn_card_type="OTHER_NON_CITIZEN", + immigration_status_str="PAROLED_ONE_YEAR", + ), + _noncitizen( + PENATVTY=164, + PEINUSYR=28, + ssn_card_type="OTHER_NON_CITIZEN", + immigration_status_str="PAROLED_ONE_YEAR", + ), + ] + ) + draw = HumanitarianDraw( + category="paroled_one_year", + origin="ukraine", + status="PAROLED_ONE_YEAR", + target=1.0, + source="https://example.com/u4u", + ) + assert us_immigration_humanitarian_draw_mask( + compatibility, + draw, + time_period=TIME_PERIOD, + ).tolist() == [False, True] + + tables = { + entity: compatibility.table(entity).copy() + for entity in compatibility.entities + } + tables["person"] = tables["person"].drop( + columns=["PRCITSHP", "PENATVTY", "PEINUSYR"] + ) + tables["person"]["CIT"] = [5, 5] + tables["person"]["POBP"] = [164, 164] + tables["person"]["YOEP"] = [2021, 2022] + acs_compatibility = Frame( + tables, + compatibility.schema, + { + entity: compatibility.weights_for(entity) + for entity in compatibility.weighted_entities + }, + ) + assert us_immigration_humanitarian_draw_mask( + acs_compatibility, + draw, + time_period=TIME_PERIOD, + ).tolist() == [False, True] + + def test_acs_venezuela_tps_includes_2023_designation_cohort(self) -> None: + compatibility = _us_frame( + [ + _noncitizen( + PENATVTY=373, + PEINUSYR=28, + ssn_card_type="NON_CITIZEN_VALID_EAD", + immigration_status_str=status, + ) + for status in ("TPS", "TPS", "TPS", "PAROLED_ONE_YEAR") + ] + ) + tables = { + entity: compatibility.table(entity).copy() + for entity in compatibility.entities + } + person = tables["person"].drop(columns=["PRCITSHP", "PENATVTY", "PEINUSYR"]) + person["CIT"] = [5, 5, 5, 5] + person["POBP"] = [373, 373, 373, 373] + person["YOEP"] = [2021, 2023, 2024, 2023] + tables["person"] = person + frame = Frame( + tables, + compatibility.schema, + { + entity: compatibility.weights_for(entity) + for entity in compatibility.weighted_entities + }, + ) + tps = HumanitarianDraw( + category="tps", + origin="venezuela", + status="TPS", + target=1.0, + source="https://example.com/venezuela-tps", + ) + parole = HumanitarianDraw( + category="paroled_one_year", + origin="venezuela", + status="PAROLED_ONE_YEAR", + target=1.0, + source="https://example.com/chnv", + ) + + assert us_immigration_humanitarian_draw_mask( + frame, + tps, + time_period=TIME_PERIOD, + ).tolist() == [True, True, False, False] + assert us_immigration_humanitarian_draw_mask( + frame, + parole, + time_period=TIME_PERIOD, + ).tolist() == [False, False, False, True] + + @pytest.mark.parametrize( + ("citizenship", "arrival_year"), + ( + (4, np.nan), + (5, np.nan), + (4, 1899), + (5, TIME_PERIOD + 1), + (5, 2020.5), + ), + ) + def test_acs_foreign_born_arrival_evidence_fails_closed( + self, + citizenship: int, + arrival_year: float, + ) -> None: + compatibility = _us_frame( + [ + _noncitizen( + PENATVTY=373, + PEINUSYR=28, + ssn_card_type="NON_CITIZEN_VALID_EAD", + immigration_status_str="TPS", + ) + ] + ) + tables = { + entity: compatibility.table(entity).copy() + for entity in compatibility.entities + } + person = tables["person"].drop(columns=["PRCITSHP", "PENATVTY", "PEINUSYR"]) + person["CIT"] = [citizenship] + person["POBP"] = [373] + person["YOEP"] = [arrival_year] + tables["person"] = person + frame = Frame( + tables, + compatibility.schema, + { + entity: compatibility.weights_for(entity) + for entity in compatibility.weighted_entities + }, + ) + draw = HumanitarianDraw( + category="tps", + origin="venezuela", + status="TPS", + target=1.0, + source="https://example.com/venezuela-tps", + ) + + with pytest.raises( + SourceRuntimeError, + match="ACS YOEP must be an integral 1900..time_period year", + ): + us_immigration_humanitarian_draw_mask( + frame, + draw, + time_period=TIME_PERIOD, + ) + + @pytest.mark.parametrize( + ("birth_country", "origin", "cutoff"), + ( + (373, "venezuela", 2023), + (312, "el_salvador", 2001), + (314, "honduras", 1998), + (315, "nicaragua", 1998), + (229, "nepal", 2015), + (205, "other_designated", 2024), + (239, "other_designated", 2024), + (248, "other_designated", 2024), + (224, "other_designated", 2024), + (407, "other_designated", 2023), + (416, "other_designated", 2024), + (448, "other_designated", 2024), + (451, "other_designated", 2023), + ), + ) + def test_acs_tps_continuous_residence_cutoff( + self, + birth_country: int, + origin: str, + cutoff: int, + ) -> None: + compatibility = _us_frame( + [ + _noncitizen( + PENATVTY=birth_country, + PEINUSYR=28, + A_AGE=70, + ssn_card_type="NON_CITIZEN_VALID_EAD", + immigration_status_str="TPS", + ), + _noncitizen( + PENATVTY=birth_country, + PEINUSYR=28, + A_AGE=70, + ssn_card_type="NON_CITIZEN_VALID_EAD", + immigration_status_str="TPS", + ), + ] + ) + tables = { + entity: compatibility.table(entity).copy() + for entity in compatibility.entities + } + person = tables["person"].drop(columns=["PRCITSHP", "PENATVTY", "PEINUSYR"]) + person["CIT"] = [5, 5] + person["POBP"] = [birth_country, birth_country] + person["YOEP"] = [cutoff, cutoff + 1] + tables["person"] = person + frame = Frame( + tables, + compatibility.schema, + { + entity: compatibility.weights_for(entity) + for entity in compatibility.weighted_entities + }, + ) + draw = HumanitarianDraw( + category="tps", + origin=origin, + status="TPS", + target=1.0, + source="https://example.com/tps-cutoff", + ) + + assert us_immigration_humanitarian_draw_mask( + frame, + draw, + time_period=max(TIME_PERIOD, cutoff + 1), + ).tolist() == [True, False] + def test_refugee_draw_excludes_residual_pool(self) -> None: # A 2022-arrived Congolese person with no legal-status indicator is # not refugee-consistent (refugees carry immediate benefits access). @@ -633,6 +899,25 @@ class TestDeterminism: def _worker_pool(self) -> pd.DataFrame: return _person_table([_noncitizen(WSAL_VAL=10_000.0) for _ in range(20)]) + def test_partial_household_lineage_uses_complete_legacy_alternative(self) -> None: + person = _person_table([_noncitizen(), _noncitizen()]) + person["source_year"] = [2024, 2024] + person["source_household_id"] = [100, np.nan] + person["source_person_id"] = [1, 2] + + partial_full = immigration_module._stable_person_draws( + person, + seed=17, + salt="lineage-regression", + ) + legacy_only = immigration_module._stable_person_draws( + person.drop(columns="source_household_id"), + seed=17, + salt="lineage-regression", + ) + + np.testing.assert_array_equal(partial_full, legacy_only) + def test_same_seed_is_bit_reproducible(self) -> None: first = _run(self._worker_pool(), workers=10.0, seed=7) second = _run(self._worker_pool(), workers=10.0, seed=7) @@ -706,6 +991,20 @@ def test_manifest_humanitarian_stocks_carry_citations(self) -> None: for label, block in flat_blocks.items(): assert float(block["target"]) >= 0, label assert str(block["source"]).startswith("https://"), label + assert humanitarian["refugee"] == { + "target": 160_000, + "source": ( + "https://ohss.dhs.gov/topics/immigration/refugees/" + "annual-flow-report/fy-24-refugees-flow-report" + ), + } + assert humanitarian["asylee"] == { + "target": 155_000, + "source": ( + "https://ohss.dhs.gov/topics/immigration/asylees/" + "annual-flow-report/fy24-asylees-flow-report" + ), + } # The withheld stock is an explicit cited zero (no published stock); # every other category carries a positive stock. assert float(humanitarian["deportation_withheld"]["target"]) == 0 @@ -932,12 +1231,20 @@ def _composition_frame( for _ in range(count) ] ) + evidence_by_status = { + "PAROLED_ONE_YEAR": {"PENATVTY": 164, "PEINUSYR": 28}, + "REFUGEE": {"PENATVTY": 412, "PEINUSYR": 28}, + "ASYLEE": {"PENATVTY": 207, "PEINUSYR": 26}, + "DEPORTATION_WITHHELD": {"PENATVTY": 303, "PEINUSYR": 24}, + "TPS": {"PENATVTY": 373, "PEINUSYR": 24}, + } for ssn, status in values: rows.append( { "PRCITSHP": 1 if ssn == "CITIZEN" else 5, "ssn_card_type": ssn, "immigration_status_str": status, + **evidence_by_status.get(status, {}), } ) return _us_frame(rows) @@ -1014,6 +1321,9 @@ def test_fails_when_non_citizen_share_implausible(self) -> None: assert any("non-citizen weighted share" in failure for failure in gate.failures) def test_gate_reads_packaged_controls_by_default(self) -> None: + packaged = us_immigration_controls() + assert packaged.undocumented.workers == 8_300_000 + assert packaged.humanitarian_target("refugee") == 160_000 gate = us_immigration_composition_gate(_us_frame([{"PRCITSHP": 1}])) assert not gate.passed controls = gate.details["controls"] @@ -1021,6 +1331,20 @@ def test_gate_reads_packaged_controls_by_default(self) -> None: assert controls["undocumented_population_anchor"] == 11_000_000 stocks = controls["humanitarian_status_stocks"] assert stocks["paroled_one_year:afghanistan"]["target"] == 73_566 + assert stocks["refugee"] == { + "target": 160_000.0, + "source": ( + "https://ohss.dhs.gov/topics/immigration/refugees/" + "annual-flow-report/fy-24-refugees-flow-report" + ), + } + assert stocks["asylee"] == { + "target": 155_000.0, + "source": ( + "https://ohss.dhs.gov/topics/immigration/asylees/" + "annual-flow-report/fy24-asylees-flow-report" + ), + } assert stocks["tps:venezuela"]["target"] == 605_015 assert stocks["deportation_withheld"]["target"] == 0 @@ -1063,6 +1387,39 @@ def test_gate_accepts_saturated_draws_inside_band(self) -> None: ) assert gate.passed, gate.failures + def test_gate_checks_per_origin_draws_not_only_category_total(self) -> None: + source = "https://example.com/parole" + controls = ImmigrationControls( + undocumented=_plausible_controls().undocumented, + humanitarian=( + HumanitarianDraw( + category="paroled_one_year", + origin="afghanistan", + status="PAROLED_ONE_YEAR", + target=1.0, + source=source, + ), + HumanitarianDraw( + category="paroled_one_year", + origin="ukraine", + status="PAROLED_ONE_YEAR", + target=1.0, + source=source, + ), + ), + ) + # Both rows carry Ukraine evidence. The category total is exactly two, + # but the per-origin targets are a zero/two redistribution. + gate = us_immigration_composition_gate( + _composition_frame(humanitarian={"PAROLED_ONE_YEAR": 2}), + controls=controls, + ) + assert not gate.passed + assert any( + "paroled_one_year:afghanistan" in failure for failure in gate.failures + ) + assert any("paroled_one_year:ukraine" in failure for failure in gate.failures) + def test_summary_reports_weighted_composition(self) -> None: summary = us_immigration_composition_summary( _composition_frame(citizens=3, other=1, ead=0, none=1) diff --git a/packages/microcosm-build/tests/test_us_immigration_pool_scaling.py b/packages/microcosm-build/tests/test_us_immigration_pool_scaling.py new file mode 100644 index 0000000000..2d9ffe6e4b --- /dev/null +++ b/packages/microcosm-build/tests/test_us_immigration_pool_scaling.py @@ -0,0 +1,328 @@ +"""Pooled immigration absolute-control mass regressions (microcosm #767).""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.source_manifest import SourceStageSpec +from microcosm.build.us_runtime import immigration as immigration_module +from microcosm.build.us_runtime import multispine_pool as multispine_pool_module +from microcosm.build.us_runtime.immigration import ( + US_IMMIGRATION_OUTPUT_COLUMNS, + US_IMMIGRATION_STAGE_NAME, + with_us_immigration_inputs, +) +from microcosm.build.us_runtime.puf_support import clone_us_frame_for_puf_support +from microcosm.build.us_runtime.spine_assembly import assemble_spines +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights + + +def _humanitarian_controls(*, ukraine: float) -> dict[str, object]: + parole_origins = ("afghanistan", "ukraine", "nicaragua", "venezuela") + tps_origins = ( + "venezuela", + "el_salvador", + "honduras", + "nicaragua", + "nepal", + "other_designated", + ) + return { + "paroled_one_year": { + origin: { + "target": ukraine if origin == "ukraine" else 0.0, + "source": f"https://example.com/parole/{origin}", + } + for origin in parole_origins + }, + "refugee": {"target": 0.0, "source": "https://example.com/refugee"}, + "asylee": {"target": 0.0, "source": "https://example.com/asylee"}, + "deportation_withheld": { + "target": 0.0, + "source": "https://example.com/withheld", + }, + "tps": { + origin: { + "target": 0.0, + "source": f"https://example.com/tps/{origin}", + } + for origin in tps_origins + }, + } + + +def _stage_spec() -> SourceStageSpec: + return SourceStageSpec.from_mapping( + { + "stage": US_IMMIGRATION_STAGE_NAME, + "survey": "test ASEC", + "source": "https://example.com", + "grain": "person", + "operations": [ + {"kind": "read_table", "table": "person"}, + { + "kind": "derive_immigration_status", + "seed_from_build_config": True, + "time_period_from_build_config": True, + "undocumented_workers": { + "target": 4.0, + "source": "https://example.com/workers", + }, + "undocumented_students": { + "target": 2.0, + "source": "https://example.com/students", + }, + "undocumented_population_anchor": { + "value": 10.0, + "source": "https://example.com/population", + }, + "humanitarian_status_stocks": _humanitarian_controls(ukraine=4.0), + }, + ], + "outputs": list(US_IMMIGRATION_OUTPUT_COLUMNS), + } + ) + + +def _immigration_frame() -> Frame: + rows: list[dict[str, object]] = [] + rows.extend({"PRCITSHP": 5, "WSAL_VAL": 10_000.0} for _ in range(4)) + rows.extend({"PRCITSHP": 5, "A_HSCOL": 2} for _ in range(3)) + rows.extend( + { + "PRCITSHP": 5, + "PENATVTY": 164, + "PEINUSYR": 28, + "CAID": 1, + } + for _ in range(8) + ) + baseline: dict[str, object] = { + "PRCITSHP": 1, + "PEINUSYR": 0, + "PENATVTY": 57, + "A_AGE": 40, + "A_MARITL": 7, + "A_SPOUSE": 0, + "A_HSCOL": 0, + "WSAL_VAL": 0.0, + "SEMP_VAL": 0.0, + "MCARE": 2, + "CAID": 2, + "IHSFLG": 2, + "CHAMPVA": 2, + "MIL": 2, + "PEN_SC1": 0, + "PEN_SC2": 0, + "RESNSS1": 0, + "RESNSS2": 0, + "SS_YN": 2, + "SSI_YN": 2, + "PEIO1COW": 0, + "A_MJOCC": 0, + "PEAFEVER": 2, + "SPM_CAPHOUSESUB": 0.0, + } + records = [] + for position, overrides in enumerate(rows, start=1): + record = dict(baseline) + record.update(overrides) + record.update( + { + "person_id": position, + "person_household_id": position, + "person_tax_unit_id": position, + "person_spm_unit_id": position, + "person_family_id": position, + "person_marital_unit_id": position, + } + ) + records.append(record) + person = pd.DataFrame(records) + ids = np.arange(1, len(person) + 1, dtype=np.int64) + return Frame( + { + "person": person, + **{ + entity: pd.DataFrame({f"{entity}_id": ids}) + for entity in US_SCHEMA.group_entities + }, + }, + US_SCHEMA, + { + "household": Weights( + np.ones(len(person), dtype=np.float64), + WeightKind.DESIGN, + ) + }, + ) + + +def _weighted_count(frame: Frame, mask: pd.Series) -> float: + weights = np.asarray(frame.resolve_weights("person").values, dtype=np.float64) + return float(weights[mask.to_numpy(dtype=bool)].sum()) + + +def test_stage_weight_scale_allocates_every_absolute_control_to_source_share( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(immigration_module, "us_immigration_stage_spec", _stage_spec) + + standalone = with_us_immigration_inputs( + _immigration_frame(), seed=0, time_period=2024 + ) + pooled_projection = with_us_immigration_inputs( + _immigration_frame(), + seed=0, + time_period=2024, + person_weight_scale=2.0, + ) + + standalone_person = standalone.table("person") + pooled_person = pooled_projection.table("person") + assert ( + _weighted_count( + standalone, + standalone_person["immigration_status_str"].eq("PAROLED_ONE_YEAR"), + ) + == 4.0 + ) + assert ( + _weighted_count( + pooled_projection, + pooled_person["immigration_status_str"].eq("PAROLED_ONE_YEAR"), + ) + == 2.0 + ) + + standalone_workers = standalone_person["WSAL_VAL"].gt(0) + pooled_workers = pooled_person["WSAL_VAL"].gt(0) + assert ( + _weighted_count( + standalone, + standalone_workers & standalone_person["ssn_card_type"].eq("NONE"), + ) + == 4.0 + ) + assert ( + _weighted_count( + pooled_projection, + pooled_workers & pooled_person["ssn_card_type"].eq("NONE"), + ) + == 2.0 + ) + + standalone_students = standalone_person["A_HSCOL"].eq(2) + pooled_students = pooled_person["A_HSCOL"].eq(2) + assert ( + _weighted_count( + standalone, + standalone_students & standalone_person["ssn_card_type"].eq("NONE"), + ) + == 2.0 + ) + assert ( + _weighted_count( + pooled_projection, + pooled_students & pooled_person["ssn_card_type"].eq("NONE"), + ) + == 1.0 + ) + + +def _spine(*, cps: bool, offset: int = 0) -> Frame: + ids = np.asarray([1, 2], dtype=np.int64) + person = pd.DataFrame( + { + "person_id": ids, + "person_household_id": ids, + "person_tax_unit_id": ids, + "person_spm_unit_id": ids, + "person_family_id": ids, + "person_marital_unit_id": ids, + "age": [30.0, 50.0], + **({"PERIDNUM": [f"p{offset + 1}", f"p{offset + 2}"]} if cps else {}), + } + ) + return Frame( + { + "person": person, + **{ + entity: pd.DataFrame({f"{entity}_id": ids}) + for entity in US_SCHEMA.group_entities + }, + }, + US_SCHEMA, + { + "household": Weights( + np.asarray([2.0, 2.0]), + WeightKind.DESIGN, + ) + }, + ) + + +def _replace_person(frame: Frame, person: pd.DataFrame) -> Frame: + return Frame( + { + "person": person, + **{ + entity: frame.table(entity) + for entity in frame.entities + if entity != "person" + }, + }, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +def test_postclone_chain_derives_and_receipts_inverse_cps_person_mass_share() -> None: + assembled = assemble_spines( + {"asec": _spine(cps=True), "acs": _spine(cps=False, offset=100)}, + household_mass_shares={"asec": 0.5, "acs": 0.5}, + ) + cloned = clone_us_frame_for_puf_support(assembled) + observed_scale: list[float] = [] + + def immigration( + available: Frame, + *, + person_weight_scale: float, + ) -> Frame: + observed_scale.append(person_weight_scale) + person = available.table("person").copy() + person["ssn_card_type"] = "CITIZEN" + person["immigration_status_str"] = "CITIZEN" + return _replace_person(available, person) + + completed = multispine_pool_module._run_source_operator_chain( + cloned, + phase="post_clone", + operator_names=("with_us_immigration_inputs",), + operators={"with_us_immigration_inputs": immigration}, + ) + + person = cloned.table("person") + weights = np.asarray(cloned.resolve_weights("person").values, dtype=np.float64) + cps = person["PERIDNUM"].notna().to_numpy(dtype=bool) + full_mass = float(weights.sum()) + cps_mass = float(weights[cps].sum()) + expected_scale = full_mass / cps_mass + assert cps_mass / full_mass == pytest.approx(0.5) + assert observed_scale == [pytest.approx(expected_scale)] + + scaling = completed.receipt["suboperators"][0]["kernel_receipt"][ + "person_design_weight_scaling" + ] + assert scaling == { + "full_pool_person_design_weight_mass": pytest.approx(full_mass), + "cps_projection_person_design_weight_mass": pytest.approx(cps_mass), + "person_weight_scale": pytest.approx(expected_scale), + "cps_person_mass_share": pytest.approx(cps_mass / full_mass), + } diff --git a/packages/microcosm-build/tests/test_us_late_producer_dag.py b/packages/microcosm-build/tests/test_us_late_producer_dag.py index d1857176f7..1e5b0a05fa 100644 --- a/packages/microcosm-build/tests/test_us_late_producer_dag.py +++ b/packages/microcosm-build/tests/test_us_late_producer_dag.py @@ -784,9 +784,9 @@ def test_canonical_us_late_schedule_is_import_validated_and_byte_stable() -> Non assert reconstructed == CANONICAL_US_LATE_PRODUCER_SCHEDULE receipt = us_late_producer_schedule_receipt() - assert receipt["schema_version"] == 16 + assert receipt["schema_version"] == 17 assert receipt["execution_receipt_contract"] == { - "version": 3, + "version": 4, "row_binding": ( "declared_globally_reconciled_input_and_scope_exact_output_source_" "and_primary_callback_resource_receipt_and_previous_execution_sha256" @@ -794,7 +794,8 @@ def test_canonical_us_late_schedule_is_import_validated_and_byte_stable() -> Non "virtual_resource_binding": ("exact_kind_specific_semantic_payload_and_sha256"), "top_binding": ( "entry_and_output_frame_sha256_execution_chain_source_" - "completion_and_nineteen_transfer_groups" + "completion_nineteen_transfer_groups_and_constrained_" + "immigration_reconciliation" ), "transition_authority": { "authority_id": "us_stacked_late_producer_transition", @@ -910,6 +911,72 @@ def test_every_transfer_declares_predictors_and_optional_absence_receipts() -> N ) +def test_immigration_transfer_declares_source_scoped_raw_evidence() -> None: + producer = transfer_producer_name("person", "source_operator_immigration") + inventory = US_LATE_TRANSFER_INPUT_INVENTORIES[producer] + by_label = { + requirement.label: requirement for requirement in inventory.requirements + } + expected = { + "immigration_asec_citizenship": ("PRCITSHP", "asec_source"), + "immigration_asec_origin": ("PENATVTY", "asec_source"), + "immigration_asec_arrival": ("PEINUSYR", "asec_source"), + "immigration_acs_citizenship": ("CIT", "acs_source"), + "immigration_acs_origin": ("POBP", "acs_source"), + "immigration_acs_arrival": ("YOEP", "acs_source"), + } + + assert set(expected) <= set(by_label) + for label, (column, scope) in expected.items(): + requirement = by_label[label] + assert requirement.required_scope == scope + assert requirement.optional is False + value_kind = "column_present" if column == "YOEP" else "finite_numeric" + assert requirement.alternatives == ( + (ProducerInputColumn("person", column, value_kind),), + ) + + expected_lineage = { + ("person_id",), + ("source_person_id", "source_year"), + ("source_household_id", "source_person_id", "source_year"), + } + transfer_lineage = by_label["immigration_stable_person_lineage"] + assert transfer_lineage.required_scope is None + assert { + tuple(column.column for column in alternative) + for alternative in transfer_lineage.alternatives + } == expected_lineage + + source_inventory = US_LATE_SOURCE_INPUT_INVENTORIES["with_us_immigration_inputs"] + source_lineage = next( + requirement + for requirement in source_inventory.requirements + if requirement.label == "stable_source_identity" + ) + assert { + tuple(column.column for column in alternative) + for alternative in source_lineage.alternatives + } == expected_lineage + + contract = CANONICAL_US_LATE_PRODUCER_REGISTRY[producer] + effective = { + item.column: item + for item in contract.inputs + if item.column.startswith("@effective:immigration_") + } + assert set(effective) == { + *(f"@effective:{label}" for label in expected), + "@effective:immigration_stable_person_lineage", + } + for label, (_column, scope) in expected.items(): + assert effective[f"@effective:{label}"].required_scope == scope + assert ( + effective["@effective:immigration_stable_person_lineage"].required_scope + == "whole_pool" + ) + + def test_every_transfer_declares_complete_cross_grain_validation_surface() -> None: entities = ( "person", @@ -1034,6 +1101,12 @@ def test_every_origin_exclusive_raw_input_has_its_native_scope() -> None: "PERIDNUM", } }, + ("person", "PRCITSHP"): 2, + ("person", "PENATVTY"): 2, + ("person", "PEINUSYR"): 2, + ("person", "CIT"): 1, + ("person", "POBP"): 1, + ("person", "YOEP"): 1, } observed: Counter[tuple[str, str, str]] = Counter() @@ -1055,15 +1128,15 @@ def test_every_origin_exclusive_raw_input_has_its_native_scope() -> None: assert observed == Counter( {(*key, required_scope[key]): count for key, count in expected_counts.items()} ) - assert sum(observed.values()) == 101 - assert sum(count for key, count in observed.items() if key[2] == "acs_source") == 43 + assert sum(observed.values()) == 107 + assert sum(count for key, count in observed.items() if key[2] == "acs_source") == 46 assert ( - sum(count for key, count in observed.items() if key[2] == "asec_source") == 58 + sum(count for key, count in observed.items() if key[2] == "asec_source") == 61 ) assert receipts[("household", "TYPEHUGQ")] == set() execution_identity = acs_transfer_module.acs_transfer_execution_contract_identity() - assert execution_identity["schema_version"] == 2 + assert execution_identity["schema_version"] == 3 assert execution_identity["housing"]["head_source_precedence"] == [ {"source": "is_household_head", "head_codes": [True]}, {"source": "A_EXPRRP", "head_codes": [1, 2]}, diff --git a/packages/microcosm-build/tests/test_us_multispine_pool.py b/packages/microcosm-build/tests/test_us_multispine_pool.py index 8f4f344871..665bdd41e9 100644 --- a/packages/microcosm-build/tests/test_us_multispine_pool.py +++ b/packages/microcosm-build/tests/test_us_multispine_pool.py @@ -1879,6 +1879,9 @@ def _producer_dtype_acs_source_frame() -> Frame: person["acs_social_security_income"] = [0.0, 12_000.0] person["acs_retirement_income"] = [0.0, 8_000.0] person["acs_interest_dividend_rental_income"] = [100.0, 2_000.0] + person["CIT"] = [1, 5] + person["POBP"] = [6, 373] + person["YOEP"] = [np.nan, 2022] household = tables["household"] household["state_fips"] = [6, 36] household["tenure_type"] = ["RENTED", "OWNED_WITH_MORTGAGE"] @@ -2165,7 +2168,7 @@ def test_every_pool_transfer_family_accepts_its_produced_physical_dtype( ) assert len(targets) == 118 - assert len(predictors) == 32 + assert len(predictors) == 35 assert len(primary_predictor_sets) == 65 primary_targets = tuple( ( @@ -3535,8 +3538,13 @@ def apply( name: str = operator_name, column: str = output, value: float = float(index + 1), + person_weight_scale: float = 1.0, ) -> Frame: calls.append(name) + if name == "with_us_immigration_inputs": + assert person_weight_scale > 1.0 + else: + assert person_weight_scale == 1.0 assert "us_spine_assembly_manifest" not in available.metadata assert not available.mass_log person = available.table("person") @@ -3805,9 +3813,10 @@ def test_pool_seed_stage_preserves_inputs_and_receipts_disclosed_defaults() -> N after_person.loc[measured_person, "takes_up_medicare_if_eligible"].tolist() == before_person.loc[measured_person, "takes_up_medicare_if_eligible"].tolist() ) - assert after_person["takes_up_wic_if_eligible"].tolist() == before_person[ - "takes_up_wic_if_eligible" - ].tolist() + assert ( + after_person["takes_up_wic_if_eligible"].tolist() + == before_person["takes_up_wic_if_eligible"].tolist() + ) assert ( after_spm.loc[measured_spm, "takes_up_tanf_if_eligible"].tolist() == before_spm.loc[measured_spm, "takes_up_tanf_if_eligible"].tolist() diff --git a/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py b/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py index 5544e92a5e..7efa5f7e28 100644 --- a/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py +++ b/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py @@ -8,7 +8,9 @@ import pandas as pd import pytest +import microcosm.build.us_runtime.acs_transfer as acs_transfer_runtime import microcosm.build.us_runtime.h5_io as h5_io +import microcosm.build.us_runtime.immigration as immigration_runtime import microcosm.build.us_runtime.post_transfer_calibration as post_transfer_calibration_runtime import microcosm.build.us_runtime.stacked_spine as stacked_spine_module from microcosm.build.frame_checkpoint import ( @@ -44,6 +46,10 @@ write_nullable_us_h5, ) from microcosm.build.us_runtime.support_provenance import ( + BASE_ASEC_SUPPORT_CHANNEL, + spine_assembly_manifest, + spine_provenance_counts, + support_channel_column, support_clone_index_column, support_source_id_column, ) @@ -85,10 +91,7 @@ def _pool_frame() -> Frame: ) -def _pool_frame_with_object_strings_on_every_entity( - *, - stacked: bool = False, -) -> Frame: +def _pool_frame_with_object_strings_on_every_entity() -> Frame: """Match the assembled pool's object-backed source-string shape.""" frame = _pool_frame() @@ -106,21 +109,6 @@ def _pool_frame_with_object_strings_on_every_entity( ), ) tables[entity] = table - if stacked: - household = tables["household"].copy() - household["puma"] = ["0600101", "0600102", "0600102"] - household["congressional_district_geoid"] = np.asarray( - [601, 601, 601], - dtype=np.int64, - ) - household["county_fips"] = ["06001", "06001", "06001"] - household[support_source_id_column("household")] = np.asarray( - [10, 20, 20], dtype=np.int64 - ) - household[support_clone_index_column("household")] = np.asarray( - [0, 0, 1], dtype=np.int64 - ) - tables["household"] = household return Frame( tables, frame.schema, @@ -131,6 +119,181 @@ def _pool_frame_with_object_strings_on_every_entity( ) +def _stacked_pool_frame_with_live_immigration( + *, + include_zero_weight_clone: bool = False, +) -> tuple[Frame, dict[str, object]]: + """Build a persisted-shape final stack and its real immigration receipt.""" + + controls = immigration_runtime.us_immigration_controls() + positive_draws = tuple(draw for draw in controls.humanitarian if draw.target > 0) + acs_evidence = { + "paroled_one_year:afghanistan": (200, 2021, "NONE", "UNDOCUMENTED"), + "paroled_one_year:ukraine": (164, 2022, "NONE", "UNDOCUMENTED"), + "paroled_one_year:nicaragua": (315, 2023, "NONE", "UNDOCUMENTED"), + "paroled_one_year:venezuela": (373, 2022, "NONE", "UNDOCUMENTED"), + "refugee": ( + 412, + 2022, + "OTHER_NON_CITIZEN", + "LEGAL_PERMANENT_RESIDENT", + ), + "asylee": ( + 207, + 2020, + "OTHER_NON_CITIZEN", + "LEGAL_PERMANENT_RESIDENT", + ), + "tps:venezuela": (373, 2015, "NONE", "UNDOCUMENTED"), + "tps:el_salvador": (312, 2001, "NONE", "UNDOCUMENTED"), + "tps:honduras": (314, 1998, "NONE", "UNDOCUMENTED"), + "tps:nicaragua": (315, 1998, "NONE", "UNDOCUMENTED"), + "tps:nepal": (229, 2015, "NONE", "UNDOCUMENTED"), + "tps:other_designated": (248, 2020, "NONE", "UNDOCUMENTED"), + } + expected_labels = {draw.label for draw in positive_draws} + if set(acs_evidence) != expected_labels: + raise AssertionError( + "Stacked H5 immigration evidence must exactly cover the positive " + "canonical humanitarian draws." + ) + + row_count = 1 + len(positive_draws) + int(include_zero_weight_clone) + ids = np.arange(1, row_count + 1, dtype=np.int64) + channels = np.asarray( + [BASE_ASEC_SUPPORT_CHANNEL] + + [stacked_spine_module.ACS_STACKED_SUPPORT_CHANNEL] * len(positive_draws) + + ([BASE_ASEC_SUPPORT_CHANNEL] if include_zero_weight_clone else []), + dtype=object, + ) + clone_indices = np.zeros(row_count, dtype=np.int64) + source_ids = ids.copy() + if include_zero_weight_clone: + clone_indices[-1] = 1 + source_ids[-1] = ids[0] + + person_rows: list[dict[str, object]] = [ + { + "PRCITSHP": 1, + "PENATVTY": 57, + "PEINUSYR": 0, + "CIT": np.nan, + "POBP": np.nan, + "YOEP": np.nan, + "A_AGE": 50, + "age": 50, + "ssn_card_type": "CITIZEN", + "immigration_status_str": "CITIZEN", + } + ] + for draw in positive_draws: + birth_country, arrival_year, ssn_card_type, immigration_status = acs_evidence[ + draw.label + ] + person_rows.append( + { + "PRCITSHP": np.nan, + "PENATVTY": np.nan, + "PEINUSYR": np.nan, + "CIT": 5, + "POBP": birth_country, + "YOEP": arrival_year, + "A_AGE": np.nan, + "age": 50, + "ssn_card_type": ssn_card_type, + "immigration_status_str": immigration_status, + } + ) + if include_zero_weight_clone: + person_rows.append(dict(person_rows[0])) + + person = pd.DataFrame(person_rows) + person.insert(0, "PERIDNUM", pd.Series([f"person-{value}" for value in ids])) + person.insert(1, "person_id", ids) + for entity in US_SCHEMA.group_entities: + person[US_SCHEMA.membership_column(entity)] = ids + person["nullable_input"] = np.resize( + np.asarray([True, None, False], dtype=object), + row_count, + ) + person["is_incapable_of_self_care"] = True + person["tax_unit_role_input"] = "DEPENDENT" + person[support_channel_column("person")] = channels + person[support_source_id_column("person")] = source_ids + person[support_clone_index_column("person")] = clone_indices + + household_weights = np.asarray( + [1.0] + + [float(draw.target) for draw in positive_draws] + + ([0.0] if include_zero_weight_clone else []), + dtype=np.float64, + ) + mutable = channels == stacked_spine_module.ACS_STACKED_SUPPORT_CHANNEL + reconciled_person, reconciliation = ( + immigration_runtime.reconcile_us_immigration_humanitarian_transfer( + person, + weights=household_weights, + mutable_rows=mutable, + seed=0, + time_period=2024, + controls=controls, + ) + ) + + tables = {"person": reconciled_person} + for entity in US_SCHEMA.group_entities: + table = pd.DataFrame({US_SCHEMA.id_column(entity): ids}) + table.insert( + 0, + f"{entity}_source_label", + pd.Series([f"{entity}-{value}" for value in ids], dtype=object), + ) + table[support_channel_column(entity)] = channels + table[support_source_id_column(entity)] = source_ids + table[support_clone_index_column(entity)] = clone_indices + tables[entity] = table + + household = tables["household"] + household["puma"] = "0600101" + household["congressional_district_geoid"] = np.full( + row_count, + 601, + dtype=np.int64, + ) + household["county_fips"] = "06001" + + for index, spec in enumerate( + post_transfer_calibration_runtime.POST_TRANSFER_CALIBRATION_SPECS.values() + ): + tables[spec.entity][spec.target] = ( + 10.0 + index + np.arange(row_count, dtype=np.float64) + ) + + assembly_tables = { + entity: table.loc[table[support_clone_index_column(entity)].eq(0)] + for entity, table in tables.items() + } + metadata = spine_assembly_manifest( + assembly_tables, + channels=( + BASE_ASEC_SUPPORT_CHANNEL, + stacked_spine_module.ACS_STACKED_SUPPORT_CHANNEL, + ), + ) + frame = Frame( + tables, + US_SCHEMA, + { + "household": Weights( + household_weights, + WeightKind.IMPORTANCE, + ) + }, + metadata=metadata, + ) + return frame, reconciliation + + def _semantic_string_columns(table: pd.DataFrame) -> tuple[str, ...]: return tuple( column @@ -580,23 +743,57 @@ def _fixture_geography_assignment( } -def _write_ready_pool(tmp_path: Path, *, stacked: bool = False) -> Path: +def _write_ready_pool( + tmp_path: Path, + *, + stacked: bool = False, + include_zero_weight_clone: bool = False, +) -> Path: run_id = "fixture-publication" pool_path = tmp_path / "pool.h5" diagnostics_path = tmp_path / "pool.agreement.json" manifest_path = tmp_path / "pool.manifest.json" + gate_names = ( + ( + "us_stacked_completeness", + "us_by_origin_battery", + "immigration_composition", + ) + if stacked + else ("us_spine_agreement",) + ) agreement_gate = { "passed": True, "gates": { - "us_spine_agreement": { + name: { "passed": True, "failures": [], "details": {"fixture": True}, } + for name in gate_names }, } + if include_zero_weight_clone and not stacked: + raise ValueError("Only the stacked H5 fixture may include a clone row.") schema_version = US_MULTISPINE_POOL_MANIFEST_SCHEMA_VERSION if stacked else 4 - pool_frame = _pool_frame_with_object_strings_on_every_entity(stacked=stacked) + if stacked: + pool_frame, immigration_reconciliation = ( + _stacked_pool_frame_with_live_immigration( + include_zero_weight_clone=include_zero_weight_clone, + ) + ) + dag = _canonical_stacked_late_dag_receipt( + pool_frame, + immigration_reconciliation=immigration_reconciliation, + ) + provenance_counts = spine_provenance_counts( + pool_frame, + boundary="stacked H5 fixture provenance counts", + ) + else: + pool_frame = _pool_frame_with_object_strings_on_every_entity() + dag = None + provenance_counts = {"household": {"rows": pool_frame.n("household")}} geography_assignment = ( _fixture_geography_assignment(pool_frame.table("household")) if stacked @@ -674,7 +871,7 @@ def _write_ready_pool(tmp_path: Path, *, stacked: bool = False) -> Path: }, }, "agreement_gate": agreement_gate, - "provenance_counts": {"household": {"rows": 3}}, + "provenance_counts": provenance_counts, "pool_h5": { "path": str(pool_path.resolve()), "sha256": _sha256(pool_path), @@ -689,7 +886,7 @@ def _write_ready_pool(tmp_path: Path, *, stacked: bool = False) -> Path: }, } if stacked: - dag = _canonical_stacked_late_dag_receipt() + assert dag is not None assert geography_assignment is not None transition_authority = ( stacked_spine_module._late_producer_transition_authority_receipt(dag) @@ -738,28 +935,44 @@ def _write_ready_pool(tmp_path: Path, *, stacked: bool = False) -> Path: def _canonical_late_calibration_owner_receipt( + frame: Frame, spec: post_transfer_calibration_runtime.PostTransferCalibrationSpec, ) -> dict[str, object]: - values = np.asarray( - [10.0, 20.0, 30.0, 40.0, 50.0, 100.0, 200.0, 300.0, 400.0, 500.0] + table = frame.table(spec.entity) + channel = table[support_channel_column(spec.entity)].astype(str) + clone_index = pd.to_numeric( + table[support_clone_index_column(spec.entity)], + errors="raise", ) - weights = np.asarray([2.0, 3.0, 5.0, 5.0, 5.0, 4.0, 4.0, 4.0, 4.0, 4.0]) - entity_ids = np.arange(1, len(values) + 1) - reference = np.asarray([True] * 5 + [False] * 5) - recipient = ~reference + reference = (channel.eq(BASE_ASEC_SUPPORT_CHANNEL) & clone_index.eq(0)).to_numpy( + dtype=bool + ) + recipient = ( + channel.eq(stacked_spine_module.ACS_STACKED_SUPPORT_CHANNEL) & clone_index.eq(0) + ).to_numpy(dtype=bool) + weights = np.asarray( + frame.resolve_weights(spec.entity).values, + dtype=np.float64, + ) + entity_ids = table[frame.schema.entity_id_column(spec.entity)].to_numpy(copy=False) constrained = spec.special_constraint != "none" - result = post_transfer_calibration_runtime.calibrate_post_transfer_values( - values, - weights, - entity_ids, - spec=spec, + application = post_transfer_calibration_runtime.apply_post_transfer_calibration( + frame, + entity=spec.entity, + family=spec.family, + target=spec.target, reference_rows=reference, recipient_rows=recipient, mutable_rows=recipient, allowed_carrier_rows=recipient if constrained else None, addition_candidate_rows=recipient if constrained else None, ) - calibration = result.receipt + result_values = application.frame.table(spec.entity)[spec.target].to_numpy( + dtype=np.float64, + copy=True, + ) + table[spec.target] = result_values + calibration = application.receipt scope = calibration["scope"] constraint: dict[str, object] = {"constraint": spec.special_constraint} if spec.special_constraint == "adult_care_qualifying_one_per_tax_unit": @@ -800,13 +1013,13 @@ def _canonical_late_calibration_owner_receipt( ), "reference_output_values_sha256": ( stacked_spine_module._post_transfer_float64_sha256( - result.values[reference], + result_values[reference], boundary="synthetic reference calibration output", ) ), "recipient_output_values_sha256": ( stacked_spine_module._post_transfer_float64_sha256( - result.values[recipient], + result_values[recipient], boundary="synthetic recipient calibration output", ) ), @@ -831,7 +1044,11 @@ def _canonical_late_calibration_owner_receipt( return owner -def _canonical_stacked_late_dag_receipt() -> dict[str, object]: +def _canonical_stacked_late_dag_receipt( + frame: Frame, + *, + immigration_reconciliation: dict[str, object], +) -> dict[str, object]: """Build a signed fixture receipt over the live canonical contracts.""" schedule = stacked_spine_module.CANONICAL_US_LATE_PRODUCER_SCHEDULE @@ -881,21 +1098,84 @@ def _canonical_stacked_late_dag_receipt() -> dict[str, object]: "sha256" ] ) + immigration_reconciliation = stacked_spine_module._json_ready( + immigration_reconciliation + ) + immigration_recipient_rows = int(immigration_reconciliation["mutable_rows"]) + immigration_producer_rows = int(immigration_reconciliation["immutable_rows"]) group_receipts: dict[str, object] = {} for group in stacked_spine_module.CANONICAL_US_LATE_TRANSFER_GROUPS: + is_immigration_group = ( + group.entity == "person" and group.family == "source_operator_immigration" + ) group_targets = { f"{group.entity}/{group.family}/{target}": { - "authorized_null_rows": 0, - "imputed_rows": 0, + **( + { + "producer_roles": ["asec_source"], + "producer_rows": immigration_producer_rows, + } + if is_immigration_group + else {} + ), + "authorized_null_rows": ( + immigration_recipient_rows if is_immigration_group else 0 + ), + "imputed_rows": ( + immigration_recipient_rows if is_immigration_group else 0 + ), "unmodeled_rows": 0, "residual_null_rows": 0, } for target in group.targets } + if is_immigration_group: + required_predictors, _optional_predictors = ( + stacked_spine_module._acs_pattern_predictor_authority( + entity=group.entity, + family_targets=group.targets, + ) + ) + pattern = acs_transfer_runtime.AcsTransferPattern( + name=acs_transfer_runtime._pattern_name(0, ()), + observed_optional_predictors=(), + predictors=required_predictors, + seed=0, + weight_kind="design", + donor_rows=immigration_producer_rows, + recipient_rows=immigration_recipient_rows, + target_regimes=tuple( + (model_target, "positive_only") + for model_target in acs_transfer_runtime._model_target_names( + group.targets + ) + ), + ) + for target in group.targets: + key = f"{group.entity}/{group.family}/{target}" + record = acs_transfer_runtime.AcsImputedInput( + column=target, + entity=group.entity, + family=group.family, + donor_spine="synthetic_stacked_h5_fixture", + donor_channel="asec", + predictors=pattern.predictors, + seed=pattern.seed, + weight_kind=pattern.weight_kind, + patterns=(pattern,), + imputed_recipient_rows=immigration_recipient_rows, + reconciliation=immigration_reconciliation, + ) + group_targets[key]["qrf_pattern_evidence"] = ( + stacked_spine_module._acs_imputed_pattern_evidence(record) + ) + group_targets[key]["post_transfer_reconciliation"] = dict( + immigration_reconciliation + ) calibrated_keys = sorted(set(group_targets) & set(late_specs)) for key in calibrated_keys: group_targets[key]["post_transfer_calibration"] = ( - _canonical_late_calibration_owner_receipt(late_specs[key]) + _canonical_late_calibration_owner_receipt(frame, late_specs[key]) ) group_receipts[group.name] = { "producer": group.name, @@ -1339,8 +1619,58 @@ def test_ready_stacked_pool_loader_binds_terminal_gate_aliases( frame, manifest, _ = load_simulation_ready_us_multispine_pool(manifest_path) + positive_draws = tuple( + draw + for draw in immigration_runtime.us_immigration_controls().humanitarian + if draw.target > 0 + ) + expected_rows = 1 + len(positive_draws) + expected_weights = np.asarray( + [1.0, *(float(draw.target) for draw in positive_draws)], + dtype=np.float64, + ) + assert all(frame.n(entity) == expected_rows for entity in US_SCHEMA.entities) + np.testing.assert_array_equal( + frame.weights_for("household").values, + expected_weights, + ) + person = frame.table("person") + assert person.loc[0, "PRCITSHP"] == 1 + assert pd.isna(person.loc[0, "CIT"]) + assert person.loc[0, "immigration_status_str"] == "CITIZEN" + assert person.loc[1:, "CIT"].eq(5).all() + assert person.loc[1:, ["POBP", "YOEP"]].notna().all().all() + for entity in US_SCHEMA.group_entities: + np.testing.assert_array_equal( + person[US_SCHEMA.membership_column(entity)], + frame.table(entity)[US_SCHEMA.id_column(entity)], + ) + for draw in positive_draws: + emitted = immigration_runtime.us_immigration_humanitarian_draw_mask( + frame, + draw, + ) + assert int(emitted.sum()) == 1 + assert float(frame.resolve_weights("person").values[emitted].sum()) == float( + draw.target + ) + for entity in US_SCHEMA.entities: + assert manifest["provenance_counts"][entity] == { + "rows": expected_rows, + "by_source_channel": { + BASE_ASEC_SUPPORT_CHANNEL: 1, + stacked_spine_module.ACS_STACKED_SUPPORT_CHANNEL: len(positive_draws), + }, + "by_clone_index": {"0": expected_rows}, + "by_source_channel_and_clone_index": { + BASE_ASEC_SUPPORT_CHANNEL: {"0": 1}, + stacked_spine_module.ACS_STACKED_SUPPORT_CHANNEL: { + "0": len(positive_draws) + }, + }, + } assert manifest["terminal_gates"] == manifest["agreement_gate"] - assert manifest["schema_version"] == 9 + assert manifest["schema_version"] == 10 assert ( manifest["pool_h5"]["materializer_version"] == US_MULTISPINE_POOL_H5_MATERIALIZER_VERSION @@ -1360,6 +1690,61 @@ def test_ready_stacked_pool_loader_binds_terminal_gate_aliases( ) +def test_ready_stacked_pool_loader_replays_immigration_status_against_h5( + tmp_path: Path, +) -> None: + """An ordinary H5 re-hash cannot forge the sealed live immigration proof.""" + + pytest.importorskip("tables") + manifest_path = _write_ready_pool(tmp_path, stacked=True) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + pool_path = Path(manifest["pool_h5"]["path"]) + with pd.HDFStore(pool_path, mode="a") as store: + person = h5_io.read_frame_table(store, "person") + parole_rows = person["immigration_status_str"].eq("PAROLED_ONE_YEAR") + assert parole_rows.any() + person.loc[parole_rows.idxmax(), "immigration_status_str"] = ( + "LEGAL_PERMANENT_RESIDENT" + ) + h5_io.put_frame_table( + store, + "person", + person, + preferred_format="fixed", + ) + + # Re-sign only the ordinary byte-level artifact envelope. The immutable + # late-transition receipt still describes the untampered live population. + manifest["pool_h5"]["sha256"] = _sha256(pool_path) + manifest["pool_h5"]["size_bytes"] = pool_path.stat().st_size + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + + with pytest.raises( + ValueError, + match="differs from the live ASEC/ACS weighted population", + ): + load_simulation_ready_us_multispine_pool(manifest_path) + + +def test_ready_stacked_pool_loader_requires_immigration_terminal_gate( + tmp_path: Path, +) -> None: + pytest.importorskip("tables") + manifest_path = _write_ready_pool(tmp_path, stacked=True) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + diagnostics_path = Path(manifest["agreement_diagnostics"]["path"]) + diagnostics = json.loads(diagnostics_path.read_text(encoding="utf-8")) + for payload in (manifest, diagnostics): + payload["terminal_gates"]["gates"].pop("immigration_composition") + payload["agreement_gate"]["gates"].pop("immigration_composition") + diagnostics_path.write_text(json.dumps(diagnostics), encoding="utf-8") + manifest["agreement_diagnostics"]["sha256"] = _sha256(diagnostics_path) + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + + with pytest.raises(ValueError, match="canonical terminal gate set"): + load_simulation_ready_us_multispine_pool(manifest_path) + + def test_ready_stacked_pool_loader_binds_h5_cd_vintage_attrs( tmp_path: Path, ) -> None: @@ -1428,7 +1813,11 @@ def test_ready_stacked_pool_loader_rejects_divergent_clone_geography( tmp_path: Path, ) -> None: pytest.importorskip("tables") - manifest_path = _write_ready_pool(tmp_path, stacked=True) + manifest_path = _write_ready_pool( + tmp_path, + stacked=True, + include_zero_weight_clone=True, + ) manifest = json.loads(manifest_path.read_text(encoding="utf-8")) pool_path = Path(manifest["pool_h5"]["path"]) clone_column = support_clone_index_column("household") @@ -1454,15 +1843,12 @@ def test_scoring_pool_loader_authenticates_failed_stacked_terminal_receipt( manifest = json.loads(manifest_path.read_text(encoding="utf-8")) diagnostics_path = Path(manifest["agreement_diagnostics"]["path"]) diagnostics = json.loads(diagnostics_path.read_text(encoding="utf-8")) - failed_gate = { + failed_gate = json.loads(json.dumps(manifest["terminal_gates"])) + failed_gate["passed"] = False + failed_gate["gates"]["us_stacked_completeness"] = { "passed": False, - "gates": { - "us_spine_agreement": { - "passed": False, - "failures": ["fixture terminal failure"], - "details": {"fixture": False}, - } - }, + "failures": ["fixture terminal failure"], + "details": {"fixture": False}, } manifest.update( { @@ -1501,7 +1887,11 @@ def test_scoring_pool_loader_authenticates_failed_stacked_terminal_receipt( ) ) - assert frame.n("household") == 3 + expected_rows = 1 + sum( + draw.target > 0 + for draw in immigration_runtime.us_immigration_controls().humanitarian + ) + assert frame.n("household") == expected_rows assert loaded_manifest["status"] == "gate_failed" assert loaded_manifest["simulation_ready"] is False assert loaded_manifest["terminal_gates"]["passed"] is False @@ -1525,7 +1915,7 @@ def test_scoring_pool_loader_authenticates_failed_stacked_terminal_receipt( "failure_count": 1, "failures": [ { - "gate": "us_spine_agreement", + "gate": "us_stacked_completeness", "message": "fixture terminal failure", } ], diff --git a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py index 462c9927fa..77339de787 100644 --- a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py @@ -22,6 +22,7 @@ import pytest import microcosm.build.us_runtime.acs_transfer as acs_transfer_module +import microcosm.build.us_runtime.immigration as immigration_module import microcosm.build.us_runtime.multispine_pool as multispine_pool_module import microcosm.build.us_runtime.post_transfer_calibration as post_transfer_calibration_runtime import microcosm.build.us_runtime.stacked_spine as stacked_spine_module @@ -185,6 +186,86 @@ def _many_household_source_frame( ) +def _with_household_design_weights(frame: Frame, values: np.ndarray) -> Frame: + """Return a source fixture with replacement household design weights.""" + + weights = np.asarray(values, dtype=np.float64) + assert weights.shape == (len(frame.table("household")),) + return Frame( + {entity: frame.table(entity).copy(deep=True) for entity in frame.entities}, + frame.schema, + {"household": Weights(weights, frame.weights_for("household").kind)}, + frame.strata.copy(deep=True), + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +def _humanitarian_weighted_source_frames( + *, + asec_count: int, + acs_count: int, + sample_fraction: float, + sample_seed: int, + state_fips: str | None = None, + puma: str | None = None, +) -> tuple[Frame, Frame]: + """Shape sampled survey weights to the live humanitarian controls. + + The stacked assembly normalizes each sampled arm back to its source mass, + then splits the ASEC anchor mass equally across ASEC and ACS. Setting the + ASEC source mass to twice the humanitarian target total and the selected + ACS weights to the individual targets therefore makes both assembled arms + exact without changing the frozen assembly manifest later in the fixture + pipeline. + """ + + controls = immigration_module.us_immigration_controls() + positive_draws = tuple(draw for draw in controls.humanitarian if draw.target > 0) + target_weights = np.asarray( + [draw.target for draw in positive_draws], + dtype=np.float64, + ) + target_mass = float(target_weights.sum()) + asec = _many_household_source_frame( + count=asec_count, + state_fips=state_fips, + ) + acs = _many_household_source_frame( + count=acs_count, + measured_offset=1_000.0, + state_fips=state_fips, + puma=puma, + ) + sampled_acs, _receipt = stacked_spine_module.sample_acs_households( + acs, + fraction=sample_fraction, + seed=sample_seed, + ) + selected_ids = sampled_acs.table("household")["household_id"].to_numpy( + dtype=np.int64 + ) + assert len(selected_ids) == len(target_weights) + + asec = _with_household_design_weights( + asec, + np.full(asec_count, 2.0 * target_mass / asec_count, dtype=np.float64), + ) + acs_household_ids = acs.table("household")["household_id"].to_numpy(dtype=np.int64) + acs_weights = np.ones(acs_count, dtype=np.float64) + id_to_position = { + int(household_id): position + for position, household_id in enumerate(acs_household_ids) + } + for household_id, target_weight in zip( + selected_ids, + target_weights, + strict=True, + ): + acs_weights[id_to_position[int(household_id)]] = target_weight + return asec, _with_household_design_weights(acs, acs_weights) + + def _replace_person( frame: Frame, person: pd.DataFrame, @@ -1222,12 +1303,294 @@ def _canonical_late_calibration_owner_receipt( return owner +_HUMANITARIAN_ACS_EVIDENCE = { + "paroled_one_year:afghanistan": (200, 2021), + "paroled_one_year:ukraine": (164, 2022), + "paroled_one_year:nicaragua": (315, 2023), + "paroled_one_year:venezuela": (373, 2022), + "refugee": (412, 2022), + "asylee": (207, 2016), + "deportation_withheld": (501, 2015), + "tps:venezuela": (373, 2015), + "tps:el_salvador": (312, 2001), + "tps:honduras": (314, 1998), + "tps:nicaragua": (315, 1998), + "tps:nepal": (229, 2015), + "tps:other_designated": (205, 2015), +} + + +def _with_reconciled_immigration_fixture( + frame: Frame, +) -> tuple[Frame, dict[str, object]]: + """Build and reconcile one live ACS household per positive manifest draw.""" + + controls = immigration_module.us_immigration_controls() + positive_draws = tuple(draw for draw in controls.humanitarian if draw.target > 0) + expected_labels = {draw.label for draw in controls.humanitarian} + assert set(_HUMANITARIAN_ACS_EVIDENCE) == expected_labels + row_count = 1 + len(positive_draws) + ids = np.arange(1, row_count + 1, dtype=np.int64) + expected_channels = {"asec": 1, "acs": len(positive_draws)} + preserve_live_lineage = all( + len(frame.table(entity)) == row_count + and support_channel_column(entity) in frame.table(entity) + and frame.table(entity)[support_channel_column(entity)] + .astype(str) + .value_counts() + .to_dict() + == expected_channels + for entity in frame.entities + ) + + tables: dict[str, pd.DataFrame] = {} + for entity in frame.entities: + source = frame.table(entity) + assert not source.empty + if preserve_live_lineage: + table = source.copy(deep=True) + else: + table = pd.concat([source.iloc[[0]]] * row_count, ignore_index=True) + table[f"{entity}_id"] = ids + table[support_channel_column(entity)] = np.asarray( + ["asec", *(["acs"] * len(positive_draws))], + dtype=object, + ) + table[support_clone_index_column(entity)] = np.zeros( + row_count, + dtype=np.int64, + ) + tables[entity] = table + + person = tables[frame.schema.person_entity] + if not preserve_live_lineage: + for entity in frame.schema.group_entities: + person[f"person_{entity}_id"] = ids + person["person_id"] = ids + person_channels = person[support_channel_column(frame.schema.person_entity)].astype( + str + ) + asec_rows = person_channels.eq("asec").to_numpy(dtype=bool) + acs_rows = person_channels.eq("acs").to_numpy(dtype=bool) + assert int(asec_rows.sum()) == 1 + assert int(acs_rows.sum()) == len(positive_draws) + # Keep legacy TPS arrivals outside the DACA age-at-entry cohort so each + # row remains eligible for the single humanitarian draw it represents. + person["A_AGE"] = np.full(row_count, 60.0) + if "age" in person: + person["age"] = np.full(row_count, 60.0) + if "source_year" in person and not preserve_live_lineage: + person["source_year"] = np.full(row_count, 2024, dtype=np.int64) + if "source_household_id" in person and not preserve_live_lineage: + person["source_household_id"] = ids + if "source_person_id" in person and not preserve_live_lineage: + person["source_person_id"] = ids + if "PERIDNUM" in person and not preserve_live_lineage: + person["PERIDNUM"] = pd.Series(ids.astype(str), dtype="string") + + acs_evidence = [_HUMANITARIAN_ACS_EVIDENCE[draw.label] for draw in positive_draws] + for column in ("PRCITSHP", "PENATVTY", "PEINUSYR", "CIT", "POBP", "YOEP"): + person[column] = np.full(row_count, np.nan, dtype=np.float64) + person.loc[asec_rows, ["PRCITSHP", "PENATVTY", "PEINUSYR"]] = ( + 1.0, + 57.0, + 0.0, + ) + person.loc[acs_rows, "CIT"] = 5.0 + person.loc[acs_rows, "POBP"] = [item[0] for item in acs_evidence] + person.loc[acs_rows, "YOEP"] = [item[1] for item in acs_evidence] + person["ssn_card_type"] = pd.Series( + np.where(asec_rows, "CITIZEN", "OTHER_NON_CITIZEN"), + index=person.index, + dtype="string", + ) + person["immigration_status_str"] = pd.Series( + np.where(asec_rows, "CITIZEN", "LEGAL_PERMANENT_RESIDENT"), + index=person.index, + dtype="string", + ) + + household = tables["household"] + existing_household_weights = pd.Series( + np.asarray(frame.weights_for("household").values, dtype=np.float64), + index=frame.table("household")["household_id"].to_numpy(dtype=np.int64), + ) + asec_household_id = int(person.loc[asec_rows, "person_household_id"].iloc[0]) + household_id_to_weight = { + asec_household_id: ( + float(existing_household_weights.loc[asec_household_id]) + if preserve_live_lineage + else 1.0 + ), + **{ + int(household_id): float(draw.target) + for household_id, draw in zip( + person.loc[acs_rows, "person_household_id"], + positive_draws, + strict=True, + ) + }, + } + household_weights = household["household_id"].map(household_id_to_weight) + assert household_weights.notna().all() + assert tuple(frame.weighted_entities) == ("household",) + weights = { + "household": Weights( + household_weights.to_numpy(dtype=np.float64), + frame.weights_for("household").kind, + ) + } + source_strata = np.asarray(frame.strata, dtype=object) + strata = ( + frame.strata + if preserve_live_lineage + else pd.Series(np.resize(source_strata, row_count), dtype=object) + ) + metadata = dict(frame.metadata) + assembly_key = stacked_spine_module.SPINE_ASSEMBLY_MANIFEST_KEY + assembly = metadata.get(assembly_key) + if not preserve_live_lineage and isinstance(assembly, Mapping): + assembly = copy.deepcopy(dict(assembly)) + declared_channels = tuple(assembly["channels"]) + assembly["native_row_counts"] = { + entity: { + channel: int(table[support_channel_column(entity)].eq(channel).sum()) + for channel in declared_channels + } + for entity, table in tables.items() + } + metadata[assembly_key] = assembly + stacked_key = stacked_spine_module.STACKED_SPINE_MANIFEST_KEY + stacked = metadata.get(stacked_key) + if not preserve_live_lineage and isinstance(stacked, Mapping): + stacked = copy.deepcopy(dict(stacked)) + household_channels = household[support_channel_column("household")].astype(str) + live_weights = household_weights.to_numpy(dtype=np.float64) + live_masses = { + channel: float( + live_weights[household_channels.eq(channel).to_numpy()].sum() + ) + for channel in ("asec", "acs") + } + total_mass = float(live_weights.sum()) + shares = { + channel: live_masses[channel] / total_mass for channel in ("asec", "acs") + } + sample_receipts = { + channel: dict(sample) + for channel, sample in stacked["survey_samples"].items() + } + mass_anchor = str(stacked["mass_anchor_channel"]) + normalized_masses = { + channel: total_mass if channel == mass_anchor else live_masses[channel] + for channel in ("asec", "acs") + } + for channel, sample in sample_receipts.items(): + sampled_mass = float(sample["sampled_household_mass"]) + normalized_mass = normalized_masses[channel] + sample.update( + { + "incoming_household_mass": normalized_mass, + "normalization_factor": normalized_mass / sampled_mass, + "normalized_household_mass": normalized_mass, + } + ) + stacked["survey_samples"] = sample_receipts + stacked["household_mass_shares"] = shares + harmonization = { + channel: dict(arm) + for channel, arm in stacked["weight_harmonization"].items() + } + for channel, arm in harmonization.items(): + incoming_mass = normalized_masses[channel] + allocated_mass = live_masses[channel] + arm.update( + { + "share": shares[channel], + "incoming_mass": incoming_mass, + "allocated_mass": allocated_mass, + "declared_allocation": shares[channel] * total_mass, + "scale_factor": allocated_mass / incoming_mass, + } + ) + stacked["weight_harmonization"] = harmonization + metadata[stacked_key] = stacked + reconciled_frame = Frame( + tables, + frame.schema, + weights, + strata, + mass_log=frame.mass_log, + metadata=metadata, + ) + reconciled_person, receipt = ( + immigration_module.reconcile_us_immigration_humanitarian_transfer( + reconciled_frame.table(frame.schema.person_entity), + weights=np.asarray( + reconciled_frame.resolve_weights(frame.schema.person_entity).values, + dtype=np.float64, + ), + mutable_rows=acs_rows, + seed=0, + ) + ) + return _replace_person(reconciled_frame, reconciled_person), receipt + + +def _immigration_fixture_qrf_evidence( + *, + target: str, + family_targets: tuple[str, ...], + donor_rows: int, + recipient_rows: int, +) -> dict[str, object]: + """Return canonical joint-codec QRF evidence for one paired output leaf.""" + + required_predictors, _optional_predictors = ( + stacked_spine_module._acs_pattern_predictor_authority( + entity="person", + family_targets=family_targets, + ) + ) + pattern = acs_transfer_module.AcsTransferPattern( + name=acs_transfer_module._pattern_name(0, ()), + observed_optional_predictors=(), + predictors=required_predictors, + seed=0, + weight_kind=WeightKind.DESIGN.value, + donor_rows=donor_rows, + recipient_rows=recipient_rows, + target_regimes=tuple( + (model_target, "positive_only") + for model_target in acs_transfer_module._model_target_names(family_targets) + ), + ) + record = acs_transfer_module.AcsImputedInput( + column=target, + entity="person", + family="source_operator_immigration", + donor_spine="synthetic_pool_tool_fixture", + donor_channel="asec", + predictors=required_predictors, + seed=0, + weight_kind=WeightKind.DESIGN.value, + patterns=(pattern,), + imputed_recipient_rows=recipient_rows, + ) + return stacked_spine_module._acs_imputed_pattern_evidence(record) + + def _canonical_late_transfer_receipt( pool_tool: ModuleType, *, authority: Mapping[str, object] | None = None, frame: Frame | None = None, + immigration_reconciliation: Mapping[str, object] | None = None, ) -> dict[str, object]: + if immigration_reconciliation is None: + _fixture_frame, immigration_reconciliation = ( + _with_reconciled_immigration_fixture(_source_frame()) + ) canonical_family = { (entity, target): family for entity, families in ( @@ -1249,12 +1612,36 @@ def _canonical_late_transfer_receipt( ] ) for group in pool_tool.CANONICAL_US_LATE_TRANSFER_GROUPS: + is_immigration_group = ( + group.entity == "person" and group.family == "source_operator_immigration" + ) + immigration_mutable_rows = int(immigration_reconciliation["mutable_rows"]) + immigration_immutable_rows = int(immigration_reconciliation["immutable_rows"]) group_targets = { f"{group.entity}/{group.family}/{target}": { - "authorized_null_rows": 0, - "imputed_rows": 0, + "authorized_null_rows": immigration_mutable_rows + if is_immigration_group + else 0, + "imputed_rows": immigration_mutable_rows if is_immigration_group else 0, "unmodeled_rows": 0, "residual_null_rows": 0, + **( + { + "qrf_pattern_evidence": ( + _immigration_fixture_qrf_evidence( + target=target, + family_targets=group.targets, + donor_rows=immigration_immutable_rows, + recipient_rows=immigration_mutable_rows, + ) + ), + "post_transfer_reconciliation": copy.deepcopy( + immigration_reconciliation + ), + } + if is_immigration_group + else {} + ), } for target in group.targets } @@ -1312,6 +1699,7 @@ def _canonical_late_dag_receipt( authority: Mapping[str, object] | None = None, output_frame_sha256: str = "f" * 64, frame: Frame | None = None, + immigration_reconciliation: Mapping[str, object] | None = None, ) -> dict[str, object]: schedule = stacked_spine_module.CANONICAL_US_LATE_PRODUCER_SCHEDULE schedule_receipt = pool_tool._json_ready( @@ -1355,6 +1743,7 @@ def _canonical_late_dag_receipt( pool_tool, authority=authority, frame=frame, + immigration_reconciliation=immigration_reconciliation, ) input_frame_sha256 = "e" * 64 previous_sha256 = stacked_spine_module._late_execution_genesis_sha256( @@ -1576,8 +1965,9 @@ def _authorized_late_impute_fixture( *, authority: Mapping[str, object] | None = None, ) -> tuple[Frame, dict[str, object], str]: - """Bind one structurally signed synthetic DAG proof to a live fixture frame.""" + """Bind one structurally signed fixture DAG proof to its live frame.""" + frame, immigration_reconciliation = _with_reconciled_immigration_fixture(frame) tables = {entity: frame.table(entity).copy(deep=True) for entity in frame.entities} for entity, table in tables.items(): if support_channel_column(entity) not in table: @@ -1622,6 +2012,7 @@ def _authorized_late_impute_fixture( authority=authority, output_frame_sha256=stacked_spine_module._late_frame_content_sha256(frame), frame=frame, + immigration_reconciliation=immigration_reconciliation, ) authorized, transition_authority_sha256 = ( stacked_spine_module._bind_late_producer_transition_authority(frame, dag) @@ -1652,15 +2043,17 @@ def _install_stacked_entrypoint_stubs( verified = _verified_inputs_fixture(pool_tool, tmp_path / "pins") source_manifest = pool_tool.load_acs_source_manifest() puf_donor = pd.DataFrame({"fixture": np.arange(7)}) + asec_source, acs_source = _humanitarian_weighted_source_frames( + asec_count=100, + acs_count=1_200, + sample_fraction=0.01, + sample_seed=578, + state_fips="06" if real_geography_assignment else None, + puma="0600100" if real_geography_assignment else None, + ) loaded = pool_tool._LoadedInputs( - asec=_many_household_source_frame( - state_fips="06" if real_geography_assignment else None, - ), - acs=_many_household_source_frame( - measured_offset=1_000.0, - state_fips="06" if real_geography_assignment else None, - puma="0600100" if real_geography_assignment else None, - ), + asec=asec_source, + acs=acs_source, acs_rent_donor=pd.DataFrame({"fixture": [1.0]}), puf_donor=puf_donor, asec_raw_stage_checkpoint={"artifact": "fixture-raw-stage"}, @@ -1801,8 +2194,8 @@ def gap_fill(frame: Frame, **kwargs): ) assert counts == { ("person", "strike_benefits"): { - "authorized_null_rows": 1, - "recipient_rows": 1, + "authorized_null_rows": 12, + "recipient_rows": 12, "donor_rows": 1, } } @@ -1913,9 +2306,12 @@ def late_producer_dag(frame: Frame, **kwargs: object): "family": group.family, "ordered_targets": list(group.targets), } + late_base, immigration_reconciliation = _with_reconciled_immigration_fixture( + primary_puf_result.frame + ) late_tables = { - entity: primary_puf_result.frame.table(entity).copy(deep=True) - for entity in primary_puf_result.frame.entities + entity: late_base.table(entity).copy(deep=True) + for entity in late_base.entities } for ( spec @@ -1943,22 +2339,17 @@ def late_producer_dag(frame: Frame, **kwargs: object): index=late_person.index, dtype="string", ) - late_tables.update( - { - name: primary_puf_result.frame.link(name) - for name in primary_puf_result.frame.links - } - ) + late_tables.update({name: late_base.link(name) for name in late_base.links}) late_frame = Frame( late_tables, - primary_puf_result.frame.schema, + late_base.schema, { - entity: primary_puf_result.frame.weights_for(entity) - for entity in primary_puf_result.frame.weighted_entities + entity: late_base.weights_for(entity) + for entity in late_base.weighted_entities }, - primary_puf_result.frame.strata, - mass_log=primary_puf_result.frame.mass_log, - metadata=primary_puf_result.frame.metadata, + late_base.strata, + mass_log=late_base.mass_log, + metadata=late_base.metadata, ) dag_receipt = _canonical_late_dag_receipt( pool_tool, @@ -1967,6 +2358,7 @@ def late_producer_dag(frame: Frame, **kwargs: object): late_frame ), frame=late_frame, + immigration_reconciliation=immigration_reconciliation, ) authorized_frame, transition_authority_sha256 = ( stacked_spine_module._bind_late_producer_transition_authority( @@ -2058,6 +2450,18 @@ def battery( monkeypatch.setattr(pool_tool, "stacked_completeness_gate", completeness) monkeypatch.setattr(pool_tool, "by_origin_battery", battery) + + def immigration(_frame: Frame) -> GateResult: + order.append("immigration") + if terminal == "immigration_red": + return GateResult( + name="fixture_immigration", + passed=False, + failures=("fixture immigration terminal failure",), + ) + return GateResult(name="fixture_immigration", passed=True) + + monkeypatch.setattr(pool_tool, "us_immigration_composition_gate", immigration) real_publish = pool_tool._write_stacked_outputs def publish(*args, **kwargs): @@ -2118,6 +2522,7 @@ def test_stacked_operator_target_requires_preparation_before_activation_authorit [ ("success", 0, "iterating"), ("red", 1, "failed"), + ("immigration_red", 1, "failed"), ("error", None, "failed"), ], ) @@ -2180,6 +2585,7 @@ def test_stacked_tool_entrypoint_fixture_e2e_emits_one_logbook_row_at_every_term "simulate", "completeness", "battery", + "immigration", "publish", ] # Exported rows must never embed host-absolute paths; pytest tmp @@ -2198,6 +2604,28 @@ def test_stacked_tool_entrypoint_fixture_e2e_emits_one_logbook_row_at_every_term manifest = json.loads( (tmp_path / "stacked-pool.manifest.json").read_text(encoding="utf-8") ) + if terminal == "immigration_red": + assert manifest["status"] == "gate_failed" + assert manifest["simulation_ready"] is False + assert manifest["terminal_gates"]["passed"] is False + assert ( + manifest["terminal_gates"]["gates"]["fixture_battery"]["passed"] is True + ) + assert manifest["terminal_gates"]["gates"]["fixture_immigration"] == { + "passed": False, + "failures": ["fixture immigration terminal failure"], + "details": {}, + } + assert row.gate_verdicts["fixture_battery"]["verdict"] == "passed" + assert row.gate_verdicts["fixture_immigration"]["verdict"] == "failed" + immigration_receipt = json.loads( + _receipt_file_from_reference( + row.gate_verdicts["fixture_immigration"]["receipt"] + ).read_text(encoding="utf-8") + ) + assert immigration_receipt["terminal_gates"]["gates"][ + "fixture_immigration" + ]["failures"] == ["fixture immigration terminal failure"] assert manifest["schema_version"] == pool_tool.POOL_MANIFEST_SCHEMA_VERSION assert manifest["pipeline"] == "us-stacked-pool" assert manifest["pool_h5"]["materializer_version"] == ( @@ -2266,13 +2694,14 @@ def test_stacked_tool_entrypoint_fixture_e2e_emits_one_logbook_row_at_every_term "materialize_multispine_agreement_outputs", "stacked_completeness_gate", "by_origin_battery", + "us_immigration_composition_gate", ] assert manifest["sampling"] == { **manifest["sampling"], "sample_fraction": 0.01, "fraction_token": "f001", "sample_seed": 578, - "realized_households": {"asec": 1, "acs": 1}, + "realized_households": {"asec": 1, "acs": 12}, } @@ -2465,7 +2894,7 @@ def capture_equality(expected: object, actual: object) -> None: "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "4c22013c378c881831b07735963b53c68a6f1bc97b17eb5a2e01d91101ee2cfd", + "spec_sha256": "ea202eebdfe30a68323af5cdce6abcc16b94c9f3f18f44ae5369fe7c1d9a5702", }, } @@ -2668,7 +3097,7 @@ def test_constants_adapter_fixture_checkpoints_are_byte_identical_and_only_recei "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "4c22013c378c881831b07735963b53c68a6f1bc97b17eb5a2e01d91101ee2cfd", + "spec_sha256": "ea202eebdfe30a68323af5cdce6abcc16b94c9f3f18f44ae5369fe7c1d9a5702", } def run_fixture(root: Path, *, config_authority: str) -> dict[str, object]: @@ -3188,6 +3617,7 @@ def fail_publication(*_args, **_kwargs) -> None: assert set(row.gate_verdicts) == { "fixture_completeness", "fixture_battery", + "fixture_immigration", "pipeline_error", } terminal_path = _receipt_file_from_reference( @@ -3460,7 +3890,7 @@ def test_legacy_checkpoint_identity_excludes_stacked_late_producer_schedule( assert changed == current -def test_stacked_checkpoint_identity_binds_v12_semantic_contracts( +def test_stacked_checkpoint_identity_binds_v13_semantic_contracts( pool_tool: ModuleType, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, @@ -3493,7 +3923,7 @@ def identity() -> dict[str, object]: current = identity() pool_code = current["pool_code"] - assert current["materializer_version"] == 12 + assert current["materializer_version"] == 13 assert current["stacked_authority"]["version"] == 11 assert current["geography_assignment"] == ( pool_tool._stacked_geography_assignment_contract() @@ -3516,6 +3946,7 @@ def identity() -> dict[str, object]: "materialize_multispine_agreement_outputs", "stacked_completeness_gate", "by_origin_battery", + "us_immigration_composition_gate", ] assert pool_code["late_producer_schedule"] == pool_tool._json_ready( pool_tool.us_late_producer_schedule_receipt() @@ -3721,7 +4152,7 @@ def changed_source_stage_binding( ) ) - assert current["materializer_version"] == stale_qrf["materializer_version"] == 12 + assert current["materializer_version"] == stale_qrf["materializer_version"] == 13 assert stale_qrf["pool_code"]["primary_qrf_checkpoint_schema_version"] == 5 assert ( pool_tool._discover_stacked_checkpoint_identity( @@ -3769,7 +4200,7 @@ def changed_source_stage_binding( assert "checkpoint base identity is stale" in capsys.readouterr().out -def test_pool_envelope_v7_preserves_stacked_bank_identity_but_rejects_v6( +def test_pool_envelope_v8_preserves_stacked_bank_identity_but_rejects_v7( pool_tool: ModuleType, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, @@ -3802,7 +4233,7 @@ def identity() -> dict[str, object]: legacy.setattr( pool_tool, "POOL_STAGE_CHECKPOINT_MATERIALIZER_VERSION", - 6, + 7, ) assert identity() == current_identity legacy_store = pool_tool._PoolStageCheckpointStore( @@ -3823,7 +4254,7 @@ def identity() -> dict[str, object]: ) capsys.readouterr() - assert pool_tool.POOL_STAGE_CHECKPOINT_MATERIALIZER_VERSION == 7 + assert pool_tool.POOL_STAGE_CHECKPOINT_MATERIALIZER_VERSION == 8 assert identity() == current_identity current_store = pool_tool._PoolStageCheckpointStore( checkpoint_root, @@ -3976,7 +4407,7 @@ def test_qbi_receipt_route_resolution_rejects_wrong_or_ambiguous_paths( ) -@pytest.mark.parametrize("legacy_version", (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11)) +@pytest.mark.parametrize("legacy_version", (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12)) def test_legacy_stacked_materializer_checkpoint_is_not_discovered( pool_tool: ModuleType, monkeypatch: pytest.MonkeyPatch, @@ -4030,7 +4461,7 @@ def test_legacy_stacked_materializer_checkpoint_is_not_discovered( ) ) - assert pool_tool._STACKED_CHECKPOINT_MATERIALIZER_VERSION == 12 + assert pool_tool._STACKED_CHECKPOINT_MATERIALIZER_VERSION == 13 assert ( pool_tool._discover_stacked_checkpoint_identity( checkpoint_root, @@ -4049,9 +4480,15 @@ def test_stacked_resume_rejects_noncanonical_post_puf_transfer_receipt( pool_tool: ModuleType, tmp_path: Path, ) -> None: + asec_source, acs_source = _humanitarian_weighted_source_frames( + asec_count=10, + acs_count=120, + sample_fraction=0.10, + sample_seed=578, + ) stack = pool_tool.assemble_stacked_spine( - _many_household_source_frame(), - _many_household_source_frame(measured_offset=1_000.0), + asec_source, + acs_source, sample_fraction=0.10, sample_seed=578, ) @@ -4172,12 +4609,14 @@ def test_stacked_entrypoint_resumes_each_checkpoint_boundary( "simulate", "completeness", "battery", + "immigration", "publish", ] assert simulated_resume_order == [ "build_stacked_pool", "completeness", "battery", + "immigration", "publish", ] assert transferred_resume_order == [ @@ -4188,6 +4627,7 @@ def test_stacked_entrypoint_resumes_each_checkpoint_boundary( "simulate", "completeness", "battery", + "immigration", "publish", ] assert assembled_resume_order == [ @@ -4202,6 +4642,7 @@ def test_stacked_entrypoint_resumes_each_checkpoint_boundary( "simulate", "completeness", "battery", + "immigration", "publish", ] final_rows = [ @@ -4434,8 +4875,8 @@ def deterministic_fixture_h5( outputs = pool_tool._output_paths(output, checkpoint_root=checkpoint_root) manifest = pool_tool._read_json_object(outputs.manifest) diagnostics = pool_tool._read_json_object(outputs.agreement_diagnostics) - assert pool_tool.POOL_MANIFEST_SCHEMA_VERSION == 9 - assert pool_tool.POOL_STAGE_CHECKPOINT_MATERIALIZER_VERSION == 7 + assert pool_tool.POOL_MANIFEST_SCHEMA_VERSION == 10 + assert pool_tool.POOL_STAGE_CHECKPOINT_MATERIALIZER_VERSION == 8 assert manifest["schema_version"] == 4 assert diagnostics["schema_version"] == 4 assert "materializer_version" not in manifest["pool_h5"] @@ -5744,7 +6185,7 @@ def test_pool_checkpoint_store_round_trips_nullable_boolean_families( manifest = pool_tool._read_json_object( cold_store.checkpoint_manifest_path(stage) ) - assert manifest["materializer_version"] == 7 + assert manifest["materializer_version"] == 8 loaded = pool_tool.load_frame_checkpoint(path).frame if stage == "assembled": assert "fixture_declared_boolean" not in loaded.person @@ -5765,11 +6206,11 @@ def test_pool_checkpoint_store_round_trips_nullable_boolean_families( assert resumed.frame.person["fixture_declared_boolean"].isna().sum() == 1 -def test_simulated_v7_checkpoint_accepts_both_string_encodings_without_rewrite( +def test_simulated_v8_checkpoint_accepts_both_string_encodings_without_rewrite( pool_tool: ModuleType, tmp_path: Path, ) -> None: - """V7 authenticates both physical string encodings as one logical frame.""" + """V8 authenticates both physical string encodings as one logical frame.""" pytest.importorskip("h5py") checkpoint_root = tmp_path / "checkpoints" @@ -5781,7 +6222,7 @@ def test_simulated_v7_checkpoint_accepts_both_string_encodings_without_rewrite( loaded = pool_tool.load_frame_checkpoint(checkpoint_path) canonical_v2_bytes = checkpoint_path.read_bytes() canonical_identity = loaded.metadata["identity"] - assert loaded.metadata["materializer_version"] == 7 + assert loaded.metadata["materializer_version"] == 8 assert any( column["dtype"] == str(CANONICAL_STRING_DTYPE) for columns in loaded.metadata["frame_schema"]["entities"].values() @@ -5812,7 +6253,7 @@ def test_simulated_v7_checkpoint_accepts_both_string_encodings_without_rewrite( banked_v2_bytes = checkpoint_path.read_bytes() assert banked_v2_bytes != canonical_v2_bytes assert legacy_metadata["identity"] == canonical_identity - assert legacy_metadata["materializer_version"] == 7 + assert legacy_metadata["materializer_version"] == 8 assert any( column["dtype"] == "object" for columns in legacy_metadata["frame_schema"]["entities"].values() @@ -6187,7 +6628,7 @@ def test_tail_support_contract_identity_mutation_rebuilds_pool_checkpoints( assert changed_store.load_deepest() is None -@pytest.mark.parametrize("legacy_version", (1, 2, 3, 4, 5, 6)) +@pytest.mark.parametrize("legacy_version", (1, 2, 3, 4, 5, 6, 7)) def test_legacy_pool_materializer_artifacts_fail_closed_with_named_receipts( pool_tool: ModuleType, monkeypatch: pytest.MonkeyPatch, @@ -6220,9 +6661,9 @@ def test_legacy_pool_materializer_artifacts_fail_closed_with_named_receipts( assert manifest["identity"]["materializer_version"] == legacy_version capsys.readouterr() - assert pool_tool.POOL_STAGE_CHECKPOINT_MATERIALIZER_VERSION == 7 + assert pool_tool.POOL_STAGE_CHECKPOINT_MATERIALIZER_VERSION == 8 current_store = _checkpoint_fixture_store(pool_tool, checkpoint_root) - assert current_store.base_identity["materializer_version"] == 7 + assert current_store.base_identity["materializer_version"] == 8 assert current_store.load_deepest() is None output = capsys.readouterr().out diff --git a/packages/microcosm-build/tests/test_us_spec_bundle.py b/packages/microcosm-build/tests/test_us_spec_bundle.py index 7db19defbb..c24c3ed6c2 100644 --- a/packages/microcosm-build/tests/test_us_spec_bundle.py +++ b/packages/microcosm-build/tests/test_us_spec_bundle.py @@ -190,7 +190,7 @@ def _load_generator_module(): LEGACY_COMPATIBILITY_SHA256 = { "source_stages.json": ( - "619a974abd80c34201a0ccf813968ebc09ed52e144ff1d39722a4d9be7bd7821" + "413174820cfb57f85783831dbbf445f007d06128d31562a1234e998e627bd3aa" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" @@ -382,11 +382,11 @@ def test_constant_derived_domain_counts_are_complete( assert len(compiled_schedule["waves"]) == 6 assert ( compiled_schedule["schedule_sha256"] - == "dcf3c6d2eade3449836c49a1dc4d3b8cd395aab9142db700c3c60598fa9c1c79" + == "4965c1485c283dec3685f4ca82fa469d8b88a85f82ccd6b39e2adc84bc0e94d6" ) assert ( compiled_schedule["payload_sha256"] - == "5921cda83725b2801f2713242003e99ba54766851808b94a4f483666bce604c5" + == "9e72ceed9365ddf8993f68a25037a15cf53375093a60e59d5ce01627a7ceb210" ) assert len(take_up["programs"]) == 17 @@ -936,12 +936,8 @@ def test_legacy_seed_vintage_and_publication_grammars_are_pinned( assert geography["assignment"]["anchor"] == "puma" assert geography["assignment"]["order"] == "before_gap_fill" assert geography["assignment"]["assign_tract"] is False - assert geography["assignment"][ - "congressional_district_vintage_crosswalk" - ] == { - "source_ref": ( - "source:us_congressional_district_vintage_crosswalk_117_to_119" - ), + assert geography["assignment"]["congressional_district_vintage_crosswalk"] == { + "source_ref": ("source:us_congressional_district_vintage_crosswalk_117_to_119"), "source_vintage": "vintage:cd_117", "target_vintage": "vintage:cd_119", } @@ -971,9 +967,12 @@ def test_legacy_seed_vintage_and_publication_grammars_are_pinned( assert engine_version_occurrences == { kind: int(kind == "take_up") for kind in TYPED_DOMAIN_KINDS } - assert take_up["legacy_contract_metadata"]["asserted_engine"][ - "inventory_built_against" - ] == engine_version + assert ( + take_up["legacy_contract_metadata"]["asserted_engine"][ + "inventory_built_against" + ] + == engine_version + ) assert _count_scalar(engine_lock, engine_version) == 1 resolved_vintages = thaw_json(resolved_us_spec.vintage_authorities) assert resolved_vintages["records"]["policyengine_us_surface"]["value"] == ( diff --git a/packages/microcosm-build/tests/test_us_stacked_spine.py b/packages/microcosm-build/tests/test_us_stacked_spine.py index 6036bdfc1e..49fe6e8d62 100644 --- a/packages/microcosm-build/tests/test_us_stacked_spine.py +++ b/packages/microcosm-build/tests/test_us_stacked_spine.py @@ -26,6 +26,7 @@ import microcosm.build.us_runtime.acs_income_universe as universe_module import microcosm.build.us_runtime.acs_transfer as acs_transfer_module +import microcosm.build.us_runtime.immigration as immigration_module import microcosm.build.us_runtime.multispine_pool as multispine_pool_module import microcosm.build.us_runtime.post_transfer_calibration as post_transfer_calibration_runtime import microcosm.build.us_runtime.puf_capital_gains_tail as tail_module @@ -2750,6 +2751,24 @@ def _canonical_gap_fill_receipt_with_pattern_evidence() -> tuple[ return receipt, direction_name, key, entity, family, family_targets +def test_immigration_pattern_authority_requires_harmonized_evidence() -> None: + required, optional = stacked_spine_module._acs_pattern_predictor_authority( + entity="person", + family_targets=("ssn_card_type", "immigration_status_str"), + ) + immigration = ( + "__acs_transfer_is_us_citizen", + "__acs_transfer_birth_country_code", + "__acs_transfer_arrival_year", + ) + + assert required == ( + *acs_transfer_module.ACS_PERSON_TRANSFER_PREDICTORS, + *immigration, + ) + assert not set(immigration).intersection(optional) + + def test_gap_fill_validator_accepts_canonical_calibration_evidence() -> None: stacked_spine_module.validate_stacked_gap_fill_receipt( _canonical_gap_fill_calibration_receipt(), @@ -3658,6 +3677,59 @@ def test_late_readiness_rejects_object_typed_nonfinite_numeric_input() -> None: ) +def test_immigration_arrival_readiness_accepts_native_acs_yoep_blanks() -> None: + producer = next( + group.name + for group in stacked_spine_module.CANONICAL_US_LATE_TRANSFER_GROUPS + if group.family == "source_operator_immigration" + ) + full_contract = stacked_spine_module.CANONICAL_US_LATE_PRODUCER_REGISTRY[producer] + requirement = next( + item + for item in full_contract.inputs + if item.column == "@effective:immigration_acs_arrival" + ) + contract = replace(full_contract, inputs=(requirement,), outputs=()) + frame = _post_puf_transfer_fixture() + person = frame.table("person").copy() + acs_rows = person[support_channel_column("person")].astype(str).eq("acs") + assert acs_rows.any() + person["CIT"] = np.where(acs_rows, 1.0, np.nan) + person["YOEP"] = np.nan + tables = {entity: frame.table(entity) for entity in frame.entities} + tables["person"] = person + native_blank = Frame( + tables, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + unfilled, invalid = stacked_spine_module._late_input_readiness_rows( + native_blank, + contract, + ) + + assert requirement.required_scope == "acs_source" + assert requirement.alternatives == ( + (ProducerInputColumn("person", "YOEP", "column_present"),), + ) + assert unfilled == {requirement: 0} + assert invalid == {requirement: 0} + assert ( + stacked_spine_module.run_producer_when_ready( + contract, + lambda: "ran", + unfilled_rows=unfilled, + invalid_rows=invalid, + absence_receipts={}, + ) + == "ran" + ) + + def _typehugq_cross_origin_readiness_fixture() -> tuple[ Frame, ProducerContract, @@ -5204,6 +5276,112 @@ def test_universe_raw_authority_binds_present_column_with_structural_nulls() -> ) +def _humanitarian_late_executor_entry_fixture() -> Frame: + """Build one exact weighted ACS carrier for every positive manifest draw.""" + + controls = immigration_module.us_immigration_controls() + positive_draws = tuple(draw for draw in controls.humanitarian if draw.target > 0) + profiles = { + "paroled_one_year:afghanistan": (200, 2021), + "paroled_one_year:ukraine": (164, 2022), + "paroled_one_year:nicaragua": (315, 2023), + "paroled_one_year:venezuela": (373, 2022), + "refugee": (412, 2022), + "asylee": (207, 2016), + "tps:venezuela": (373, 2021), + "tps:el_salvador": (312, 2001), + "tps:honduras": (314, 1998), + "tps:nicaragua": (315, 1998), + "tps:nepal": (229, 2015), + "tps:other_designated": (224, 2024), + } + assert {draw.label for draw in positive_draws} == set(profiles) + positive_target = float(sum(draw.target for draw in positive_draws)) + + # Stacked assembly assigns half of the ASEC anchor mass to each source. + # Make the ACS incoming total equal that allocation so every live ACS + # household retains its manifest target as its final resolved weight. + asec = _source_frame( + household_ids=[11], + weights=[2.0 * positive_target], + stratum="asec_2024", + ) + asec_person = asec.table("person") + asec_person["is_female"] = False + asec_person["is_household_head"] = True + asec_person["employment_income_before_lsr"] = 10_000.0 + asec_person["self_employment_income_before_lsr"] = 0.0 + asec_person["pre_subsidy_rent"] = 12_000.0 + asec_person["unemployment_compensation"] = 1.0 + asec_person["is_disabled"] = False + for column, value in ( + ("taxable_interest_income", 100.0), + ("tax_exempt_interest_income", 0.0), + ("qualified_dividend_income", 50.0), + ("non_qualified_dividend_income", 25.0), + ("rental_income", 0.0), + ("estate_income", 0.0), + ): + asec_person[column] = value + asec_person["PRCITSHP"] = 1.0 + asec_person["PENATVTY"] = 57.0 + asec_person["PEINUSYR"] = 0.0 + asec_person["source_year"] = 2024 + asec_person["source_household_id"] = asec_person["person_household_id"] + asec_person["source_person_id"] = asec_person["person_id"] + asec.table("household")["tenure_type"] = "RENTED" + + acs = _source_frame( + household_ids=list(range(101, 101 + len(positive_draws))), + weights=[float(draw.target) for draw in positive_draws], + stratum="acs_2024_1yr", + ) + acs_person = acs.table("person") + acs_person["is_female"] = False + acs_person["is_household_head"] = True + acs_person["employment_income_before_lsr"] = 10_000.0 + acs_person["WAGP"] = 10_000.0 + acs_person["self_employment_income_before_lsr"] = 0.0 + acs_person["SEMP"] = 0.0 + acs_person["acs_interest_dividend_rental_income"] = 0.0 + acs_person["CIT"] = 5.0 + acs_person["POBP"] = [profiles[draw.label][0] for draw in positive_draws] + acs_person["YOEP"] = [profiles[draw.label][1] for draw in positive_draws] + acs_person["age"] = [12.0, *([60.0] * (len(positive_draws) - 1))] + acs_person["source_year"] = 2024 + acs_person["source_household_id"] = acs_person["person_household_id"] + acs_person["source_person_id"] = acs_person["person_id"] + acs.table("household")["TYPEHUGQ"] = 1 + acs.table("household")["tenure_type"] = "RENTED" + + registry = stacked_spine_module.CANONICAL_US_LATE_PRODUCER_REGISTRY + primary_contract = registry[stacked_spine_module.US_LATE_PRIMARY_PUF_STAGE] + initial = _fill_late_contract_surface( + assemble_stacked_spine( + asec, + acs, + acs_sample_fraction=1.0, + acs_sample_seed=578, + ).frame, + contracts=(primary_contract,), + include_outputs=False, + ) + initial_person = initial.table("person") + structural_row = initial_person.index[ + initial_person[support_channel_column("person")].eq("acs") + ][0] + initial_person.loc[ + structural_row, + [ + "WAGP", + "SEMP", + "employment_income_before_lsr", + "self_employment_income_before_lsr", + ], + ] = np.nan + return initial + + def _run_real_late_executor_fixture( monkeypatch: pytest.MonkeyPatch, *, @@ -5212,7 +5390,7 @@ def _run_real_late_executor_fixture( asec_earnings_delta: float = 0.0, ) -> tuple[stacked_spine_module.StackedLateProducerResult, tuple[str, ...], int]: registry = stacked_spine_module.CANONICAL_US_LATE_PRODUCER_REGISTRY - initial = _late_universe_entry_fixture() + initial = _humanitarian_late_executor_entry_fixture() initial_person = initial.table("person") if asec_earnings_delta: asec_row = initial_person.index[ @@ -5278,6 +5456,33 @@ def primary(frame: Frame): include_outputs=True, ) completed_person = completed.table("person") + asec_rows = completed_person[support_channel_column("person")].eq("asec") + completed_person.loc[asec_rows, ["CIT", "POBP", "YOEP"]] = np.nan + completed_person.loc[ + ~asec_rows, + ["PRCITSHP", "PENATVTY", "PEINUSYR"], + ] = np.nan + completed_person["A_AGE"] = completed_person["age"] + completed_person["ssn_card_type"] = np.where( + asec_rows, + "CITIZEN", + "NONE", + ) + completed_person["immigration_status_str"] = np.where( + asec_rows, + "CITIZEN", + "UNDOCUMENTED", + ) + indicator_documented = ~asec_rows & pd.to_numeric( + completed_person["POBP"], + errors="raise", + ).isin((207, 412)) + completed_person.loc[indicator_documented, "ssn_card_type"] = ( + "OTHER_NON_CITIZEN" + ) + completed_person.loc[indicator_documented, "immigration_status_str"] = ( + "LEGAL_PERMANENT_RESIDENT" + ) completed_person["unemployment_compensation"] = np.ones( len(completed_person), dtype=np.float64, @@ -5387,10 +5592,55 @@ def transfer( for spec in post_transfer_calibration_runtime.POST_TRANSFER_CALIBRATION_SPECS.values() if spec.stage == "late_transfer" } + is_immigration_group = ( + group.entity == "person" and group.family == "source_operator_immigration" + ) + immigration_reconciliation: Mapping[str, object] | None = None + imputed_recipient_rows = 1 + if is_immigration_group: + mutable_rows = ( + frame.table("person")[support_channel_column("person")] + .astype(str) + .eq("acs") + .to_numpy(dtype=bool) + ) + reconciled_person, immigration_reconciliation = ( + immigration_module.reconcile_us_immigration_humanitarian_transfer( + frame.table("person"), + weights=np.asarray( + frame.resolve_weights("person").values, + dtype=np.float64, + ), + mutable_rows=mutable_rows, + seed=0, + time_period=2024, + ) + ) + tables = {entity: frame.table(entity) for entity in frame.entities} + tables["person"] = reconciled_person + frame = Frame( + tables, + frame.schema, + { + entity: frame.weights_for(entity) + for entity in frame.weighted_entities + }, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + imputed_recipient_rows = int(mutable_rows.sum()) evidence_targets = tuple( - target - for target in group.targets - if f"{group.entity}/{group.family}/{target}" in late_specs + dict.fromkeys( + ( + *( + target + for target in group.targets + if f"{group.entity}/{group.family}/{target}" in late_specs + ), + *(group.targets if is_immigration_group else ()), + ) + ) ) model_targets = acs_transfer_module._model_target_names(evidence_targets) pattern = AcsTransferPattern( @@ -5400,7 +5650,7 @@ def transfer( seed=0, weight_kind="design", donor_rows=1, - recipient_rows=1, + recipient_rows=imputed_recipient_rows, target_regimes=tuple((target, "positive_only") for target in model_targets), ) plain_pattern = replace(pattern, target_regimes=()) @@ -5415,7 +5665,8 @@ def transfer( seed=pattern.seed, weight_kind=pattern.weight_kind, patterns=(pattern if target in evidence_targets else plain_pattern,), - imputed_recipient_rows=1, + imputed_recipient_rows=imputed_recipient_rows, + reconciliation=immigration_reconciliation, ) for target in group.targets ) @@ -5437,15 +5688,19 @@ def transfer( ): key = f"{group.entity}/{group.family}/{target}" target_receipt: dict[str, object] = { - "authorized_null_rows": 1, - "imputed_rows": 1, + "authorized_null_rows": imputed_recipient_rows, + "imputed_rows": imputed_recipient_rows, "unmodeled_rows": 0, "residual_null_rows": 0, } - if key in late_specs: + if key in late_specs or is_immigration_group: target_receipt["qrf_pattern_evidence"] = ( stacked_spine_module._acs_imputed_pattern_evidence(record) ) + if is_immigration_group: + target_receipt["post_transfer_reconciliation"] = dict( + record.reconciliation or {} + ) target_receipts[key] = target_receipt calibrated_keys = sorted(set(target_receipts) & set(late_specs)) for key in calibrated_keys: @@ -5655,6 +5910,32 @@ def test_real_late_executor_follows_canonical_order_and_finalizes_sources_once( stacked_spine_module.US_LATE_PRODUCER_TRANSITION_AUTHORITY_KEY ]["sha256"] ) + immigration_group = result.receipt["post_puf_transfer"]["groups"][ + "transfer:person/source_operator_immigration" + ] + immigration_targets = immigration_group["targets"] + pair_receipts = [ + immigration_targets[f"person/source_operator_immigration/{target}"] + for target in ("ssn_card_type", "immigration_status_str") + ] + assert ( + pair_receipts[0]["post_transfer_reconciliation"] + == pair_receipts[1]["post_transfer_reconciliation"] + ) + assert pair_receipts[0]["post_transfer_reconciliation"]["kind"] == ( + "deterministic_humanitarian_residual_target" + ) + expected_required, _optional = ( + stacked_spine_module._acs_pattern_predictor_authority( + entity="person", + family_targets=("ssn_card_type", "immigration_status_str"), + ) + ) + assert all( + tuple(receipt["qrf_pattern_evidence"]["patterns"][0]["predictors"]) + == expected_required + for receipt in pair_receipts + ) stacked_spine_module.validate_stacked_late_producer_receipt( result.receipt, boundary="executor regression", @@ -5663,6 +5944,306 @@ def test_real_late_executor_follows_canonical_order_and_finalizes_sources_once( ) +def test_immigration_group_receipt_rejects_reordered_harmonized_predictors( + monkeypatch: pytest.MonkeyPatch, +) -> None: + result, _events, _finalizer_calls = _run_real_late_executor_fixture(monkeypatch) + transfer = deepcopy(dict(result.receipt["post_puf_transfer"])) + target = transfer["groups"]["transfer:person/source_operator_immigration"][ + "targets" + ]["person/source_operator_immigration/immigration_status_str"] + evidence = target["qrf_pattern_evidence"] + predictors = evidence["patterns"][0]["predictors"] + predictors[-1], predictors[-2] = predictors[-2], predictors[-1] + unsigned = dict(evidence) + unsigned.pop("sha256") + evidence["sha256"] = stacked_spine_module._canonical_sha256(unsigned) + + with pytest.raises(ValueError, match="predictor order is outside canonical"): + stacked_spine_module.validate_stacked_post_puf_transfer_receipt( + transfer, + boundary="reordered immigration predictor regression", + ) + + +@pytest.mark.parametrize( + ("mutation", "message"), + ( + ("remove", "immigration post-transfer reconciliation is absent"), + ("diverge", "paired immigration reconciliation evidence is incomplete"), + ("inflated_tolerance", "reconciliation tolerance is non-canonical"), + ("zero_target_immutable", "mass against an explicit-zero target"), + ), +) +def test_immigration_group_receipt_rejects_missing_or_divergent_reconciliation( + monkeypatch: pytest.MonkeyPatch, + mutation: str, + message: str, +) -> None: + result, _events, _finalizer_calls = _run_real_late_executor_fixture(monkeypatch) + transfer = deepcopy(dict(result.receipt["post_puf_transfer"])) + targets = transfer["groups"]["transfer:person/source_operator_immigration"][ + "targets" + ] + status = targets["person/source_operator_immigration/immigration_status_str"] + reconciliation = status["post_transfer_reconciliation"] + if mutation == "remove": + status.pop("post_transfer_reconciliation") + elif mutation == "diverge": + reconciliation["seed"] += 1 + elif mutation == "inflated_tolerance": + reconciliation["floating_tolerance"] *= 1_000_000 + else: + zero_draw = next( + draw for draw in reconciliation["draws"].values() if draw["target"] == 0 + ) + zero_draw.update( + { + "immutable_population": 1.0, + "achieved_population": 1.0, + "absolute_error": 1.0, + "immutable_overshoot": 1.0, + } + ) + + with pytest.raises(ValueError, match=message): + stacked_spine_module.validate_stacked_post_puf_transfer_receipt( + transfer, + boundary=f"{mutation} immigration reconciliation regression", + ) + + +def test_immigration_reconciliation_replays_receipt_and_downstream_live_frame( + monkeypatch: pytest.MonkeyPatch, +) -> None: + result, _events, _finalizer_calls = _run_real_late_executor_fixture(monkeypatch) + transfer = deepcopy(dict(result.receipt["post_puf_transfer"])) + immigration_group = transfer["groups"][ + "transfer:person/source_operator_immigration" + ] + immigration_targets = immigration_group["targets"] + source_receipt = immigration_targets[ + "person/source_operator_immigration/ssn_card_type" + ]["post_transfer_reconciliation"] + fabricated_reconciliation = deepcopy(source_receipt) + draw = next( + item + for item in fabricated_reconciliation["draws"].values() + if item["target"] > 0 + ) + draw["eligible_recipient_population"] += 1.0 + draw["selected_recipient_population"] += 1.0 + draw["achieved_population"] += 1.0 + draw["residual_selection_error"] = 1.0 + draw["absolute_error"] = 1.0 + for target in ("ssn_card_type", "immigration_status_str"): + key = f"person/source_operator_immigration/{target}" + immigration_targets[key]["post_transfer_reconciliation"] = deepcopy( + fabricated_reconciliation + ) + transfer["targets"][key] = deepcopy(immigration_targets[key]) + + # The forged values remain internally self-consistent; only live replay can + # distinguish them from the actually emitted weighted population. + stacked_spine_module.validate_stacked_post_puf_transfer_receipt( + transfer, + boundary="fabricated immigration mass receipt-only control", + ) + with pytest.raises( + ValueError, + match="differs from the live ASEC/ACS weighted population", + ): + stacked_spine_module.validate_stacked_post_puf_transfer_receipt( + transfer, + boundary="fabricated immigration mass live replay", + frame=result.frame, + ) + + tables = {entity: result.frame.table(entity) for entity in result.frame.entities} + person = tables["person"].copy() + first_positive_draw = next( + draw + for draw in immigration_module.us_immigration_controls().humanitarian + if draw.target > 0 + ) + emitted = immigration_module.us_immigration_humanitarian_draw_mask( + result.frame, + first_positive_draw, + time_period=2024, + ) + person.loc[person.index[np.flatnonzero(emitted)[0]], "immigration_status_str"] = ( + "LEGAL_PERMANENT_RESIDENT" + ) + tables["person"] = person + drifted = Frame( + tables, + result.frame.schema, + { + entity: result.frame.weights_for(entity) + for entity in result.frame.weighted_entities + }, + result.frame.strata, + mass_log=result.frame.mass_log, + metadata=result.frame.metadata, + ) + with pytest.raises( + ValueError, + match="differs from the live ASEC/ACS weighted population", + ): + stacked_spine_module.validate_stacked_late_producer_transition_authority( + drifted, + result.receipt, + boundary="downstream immigration label live replay", + expected_transition_authority_sha256=(result.transition_authority_sha256), + ) + + +def test_immigration_live_replay_rejects_equal_weight_selection_identity_swap( + monkeypatch: pytest.MonkeyPatch, +) -> None: + asec = _source_frame( + household_ids=[11], + weights=[2.0], + stratum="asec_2024", + ) + acs = _source_frame( + household_ids=[101, 102], + weights=[1.0, 1.0], + extra_household_columns={"TYPEHUGQ": 1}, + stratum="acs_2024_1yr", + ) + base = assemble_stacked_spine( + asec, + acs, + acs_sample_fraction=1.0, + acs_sample_seed=578, + ).frame + person = base.table("person").copy() + channel = person[support_channel_column("person")].astype(str) + immutable = channel.eq("asec").to_numpy(dtype=bool) + mutable = channel.eq("acs").to_numpy(dtype=bool) + person["PRCITSHP"] = np.where(immutable, 1.0, np.nan) + person["PENATVTY"] = np.where(immutable, 57.0, np.nan) + person["PEINUSYR"] = np.where(immutable, 0.0, np.nan) + person["CIT"] = np.where(mutable, 5.0, np.nan) + person["POBP"] = np.where(mutable, 164.0, np.nan) + person["YOEP"] = np.where(mutable, 2022.0, np.nan) + person["source_year"] = 2024 + person["source_household_id"] = person["person_household_id"] + person["source_person_id"] = person["person_id"] + person["ssn_card_type"] = np.where( + immutable, + "CITIZEN", + "OTHER_NON_CITIZEN", + ) + person["immigration_status_str"] = np.where( + immutable, + "CITIZEN", + "LEGAL_PERMANENT_RESIDENT", + ) + weights = np.asarray(base.resolve_weights("person").values, dtype=np.float64) + mutable_weight = float(weights[mutable][0]) + assert weights[mutable].tolist() == [mutable_weight, mutable_weight] + draw = immigration_module.HumanitarianDraw( + category="paroled_one_year", + origin="ukraine", + status="PAROLED_ONE_YEAR", + target=mutable_weight, + source="https://example.com/u4u", + ) + controls = immigration_module.ImmigrationControls( + undocumented=immigration_module.UndocumentedControls( + workers=1.0, + students=1.0, + population_anchor=1.0, + sources={ + "undocumented_workers": "https://example.com/workers", + "undocumented_students": "https://example.com/students", + "undocumented_population_anchor": "https://example.com/population", + }, + ), + humanitarian=(draw,), + ) + reconciled_person, reconciliation = ( + immigration_module.reconcile_us_immigration_humanitarian_transfer( + person, + weights=weights, + mutable_rows=mutable, + seed=0, + time_period=2024, + controls=controls, + ) + ) + + def with_person(frame: Frame, replacement: pd.DataFrame) -> Frame: + tables = {entity: frame.table(entity) for entity in frame.entities} + tables["person"] = replacement + return Frame( + tables, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + reconciled = with_person(base, reconciled_person) + target_receipt = { + "producer_rows": int(immutable.sum()), + "authorized_null_rows": int(mutable.sum()), + "imputed_rows": int(mutable.sum()), + "unmodeled_rows": 0, + "residual_null_rows": 0, + "post_transfer_reconciliation": reconciliation, + } + monkeypatch.setattr( + stacked_spine_module, + "us_immigration_controls", + lambda: controls, + ) + stacked_spine_module._validate_immigration_post_transfer_reconciliation( + target_receipt, + boundary="canonical equal-weight selection", + frame=reconciled, + ) + + swapped_person = reconciled_person.copy() + selected_index = swapped_person.index[ + mutable + & swapped_person["immigration_status_str"].eq("PAROLED_ONE_YEAR").to_numpy() + ][0] + unselected_index = swapped_person.index[ + mutable + & swapped_person["immigration_status_str"] + .eq("LEGAL_PERMANENT_RESIDENT") + .to_numpy() + ][0] + swapped_person.loc[selected_index, "immigration_status_str"] = ( + "LEGAL_PERMANENT_RESIDENT" + ) + swapped_person.loc[unselected_index, "immigration_status_str"] = "PAROLED_ONE_YEAR" + swapped = with_person(reconciled, swapped_person) + assert float( + swapped.resolve_weights("person") + .values[ + swapped_person["immigration_status_str"] + .eq("PAROLED_ONE_YEAR") + .to_numpy(dtype=bool) + ] + .sum() + ) == pytest.approx(mutable_weight) + + with pytest.raises( + ValueError, + match="selected mutable-row identities differ from the canonical seeded selection", + ): + stacked_spine_module._validate_immigration_post_transfer_reconciliation( + target_receipt, + boundary="equal-weight selection identity swap", + frame=swapped, + ) + + def test_late_executor_signature_rejects_qrf_regime_evidence_tampering( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tools/build_us_fiscal_refresh_release.py b/tools/build_us_fiscal_refresh_release.py index df0634b20e..d57af36786 100644 --- a/tools/build_us_fiscal_refresh_release.py +++ b/tools/build_us_fiscal_refresh_release.py @@ -234,6 +234,10 @@ require_authenticated_us_multispine_pool_h5, us_multispine_pool_release_receipt, ) +from microcosm.build.us_runtime.immigration import ( + us_immigration_controls, + us_immigration_humanitarian_draw_mask, +) from microcosm.build.us_runtime.input_mass import us_input_mass_totals from microcosm.build.us_runtime.l0_refit_export import ( attach_l0_refit_entity_weights, @@ -436,7 +440,10 @@ class PoolReleaseIdentityMismatchError(ValueError): # 11: target-frame checkpoint columns now preserve nullable booleans as # canonical bool values plus an explicit uint8 null mask. Older checkpoints # cannot attest this lossless physical representation. -TARGET_FRAME_CHECKPOINT_MATERIALIZER_VERSION = 11 +# 12: source-aware humanitarian-stock targets now materialize from the paired +# immigration labels and native ASEC/ACS origin/arrival evidence. Version-11 +# checkpoints cannot attest those per-draw calibration columns or mask semantics. +TARGET_FRAME_CHECKPOINT_MATERIALIZER_VERSION = 12 DEFAULT_MAXIMUM_MICROSIM_BATCH_SIZE = 5_000 DEFAULT_L0_REFIT_LAMBDA_SHARE = 0.8 DEFAULT_US_FISCAL_CALIBRATION_EPOCHS = 1_500 @@ -2123,6 +2130,92 @@ def _selection_mass_protection_specs( return tuple(specs) +def _humanitarian_immigration_stock_specs( + *, + time_period: int = PERIOD, +) -> tuple[TargetSpec, ...]: + """Return one positive calibration target per manifest humanitarian draw. + + These are builder-owned preservation targets, not Ledger fiscal facts, so + the caller adds them only after the fiscal target-parity gate has checked + the feed registry. Values and citations come directly from the same + immigration source-stage manifest that assigns the person labels. + """ + + specs: list[TargetSpec] = [] + for draw in us_immigration_controls().humanitarian: + if draw.target <= 0: + continue + measure = f"humanitarian_immigration_stock.{draw.label.replace(':', '.')}" + metadata = { + "materializer": "humanitarian_immigration_stock", + "measure_mode": "indicator_sum", + "target_role": "humanitarian_immigration_stock", + "humanitarian_draw": draw.label, + "humanitarian_category": draw.category, + "humanitarian_status": draw.status, + "issue": "PolicyEngine/microcosm#767", + } + if draw.origin is not None: + metadata["humanitarian_origin"] = draw.origin + specs.append( + TargetSpec( + name=measure, + entity="household", + value=draw.target, + measure=measure, + period=time_period, + source=draw.source, + family="humanitarian_immigration_stock", + metadata=metadata, + ) + ) + return tuple(specs) + + +def _materialize_humanitarian_immigration_stock_targets( + *, + frame: Frame, + household: pd.DataFrame, + target_specs: Sequence[TargetSpec], + time_period: int = PERIOD, +) -> None: + """Collapse source-aware humanitarian person indicators to households.""" + + humanitarian_specs = [ + spec + for spec in target_specs + if spec.metadata.get("materializer") == "humanitarian_immigration_stock" + ] + if not humanitarian_specs: + return + controls = us_immigration_controls() + draws = {draw.label: draw for draw in controls.humanitarian} + if len(draws) != len(controls.humanitarian): + raise RuntimeError("Humanitarian immigration controls have duplicate draws.") + for spec in humanitarian_specs: + draw_label = spec.metadata.get("humanitarian_draw") + if draw_label not in draws: + raise RuntimeError( + f"Humanitarian target {spec.name!r} refers to unknown manifest " + f"draw {draw_label!r}." + ) + mask = np.asarray( + us_immigration_humanitarian_draw_mask( + frame, + draws[draw_label], + time_period=time_period, + ), + dtype=bool, + ) + if mask.shape != (frame.n("person"),): + raise RuntimeError( + f"Humanitarian draw {draw_label!r} returned mask shape " + f"{mask.shape}; expected {(frame.n('person'),)}." + ) + household[spec.measure] = _collapse_person(frame, mask.astype(np.float64)) + + def _target_frame_checkpoint_identity( *, base_dataset_sha256: str, @@ -4615,6 +4708,12 @@ def _materialize_target_frame( variable="state_income_tax", tax_unit_positions=tax_unit_positions, ) + _materialize_humanitarian_immigration_stock_targets( + frame=base_frame, + household=hh, + target_specs=target_specs, + time_period=PERIOD, + ) population_age_target_specs = [ spec for spec in target_specs @@ -8776,7 +8875,10 @@ def _main(argv: Sequence[str] | None = None) -> None: for failure in target_parity_gate.failures ) ) - target_specs = target_registry.specs + target_specs = ( + *target_registry.specs, + *_humanitarian_immigration_stock_specs(time_period=PERIOD), + ) active_target_registry = TargetRegistry(target_specs, country="us") # SSI take-up wiring resolves as soon as the registry exists (fail-fast, # microcosm#507/#508): the band targets come from the same ledger-fed @@ -10787,6 +10889,11 @@ def _main(argv: Sequence[str] | None = None) -> None: ) else: export_frame = _with_l0_refit_weights(base_frame, result) + # Calibration can redistribute person stocks even though the persisted + # labels remain structurally valid. Re-evaluate the composition contract + # at delivered weights and use this final verdict in every terminal gate + # and release artifact below. + final_immigration_gate = us_immigration_composition_gate(export_frame) compilation = dict(compilation) final_uncapped_ssi = _ssi_person_uncapped_amount( export_frame, @@ -11074,7 +11181,7 @@ def _main(argv: Sequence[str] | None = None) -> None: health_input_gate, base_population_gate, incumbent_diagnostics, - immigration_gate, + final_immigration_gate, enforced_input_mass_reference_gate, degenerate_input_gate, ecps_parity_gate=enforced_ecps_parity_gate, @@ -11123,7 +11230,7 @@ def _main(argv: Sequence[str] | None = None) -> None: target_profile_gate=target_profile_gate, health_input_gate=health_input_gate, base_population_gate=base_population_gate, - immigration_gate=immigration_gate, + immigration_gate=final_immigration_gate, input_mass_reference_gate=input_mass_reference_gate, hours_worked_gate=hours_worked_gate, snap_take_up_gate=snap_take_up_gate, @@ -11769,7 +11876,7 @@ def _main(argv: Sequence[str] | None = None) -> None: health_input_gate=health_input_gate, base_population_gate=base_population_gate, incumbent_diagnostics=incumbent_diagnostics, - immigration_gate=immigration_gate, + immigration_gate=final_immigration_gate, input_mass_reference_gate=enforced_input_mass_reference_gate, degenerate_input_gate=degenerate_input_gate, ecps_parity_gate=enforced_ecps_parity_gate, diff --git a/tools/build_us_multispine_pool.py b/tools/build_us_multispine_pool.py index 23e0de84d2..306ea41bf2 100644 --- a/tools/build_us_multispine_pool.py +++ b/tools/build_us_multispine_pool.py @@ -6,9 +6,10 @@ ``stack -> geography -> gap-fill -> PUF pass + tail -> late DAG -> derive -> seed -> simulate -> gates``. Both survey arms use one composition-preserving ``--sample-fraction``; PUF -donors always remain full. The terminal completeness gate plus by-origin -battery replace two-spine agreement. ``--legacy-two-spine`` retains the -retiring pipeline byte-for-byte for reproducibility. +donors always remain full. The terminal completeness, by-origin, and +humanitarian-immigration gates replace two-spine agreement. +``--legacy-two-spine`` retains the retiring pipeline byte-for-byte for +reproducibility. Every input is local and explicitly SHA-pinned; this tool never downloads data. It writes a nullable input-only H5 plus a manifest and terminal gate @@ -130,6 +131,7 @@ ACS_2022_RENT_ARTIFACT_SHA256, load_acs_2022_rent_donor, ) +from microcosm.build.us_runtime.immigration import us_immigration_composition_gate from microcosm.build.us_runtime.multispine_pool import ( POOL_CHECKPOINT_STAGE_ORDER, POOL_DERIVE_OPERATOR_ORDER, @@ -284,6 +286,9 @@ # 7: Stacked primary-PUF output universes are explicit. Earlier envelopes can # contain nulls outside the PUF clone for an output declared over the whole # pool and therefore cannot resume safely even when their bank is reusable. +# 8: ACS native citizenship/origin/arrival inputs, pooled immigration control +# scaling, and hard post-transfer humanitarian reconciliation change the +# assembled and transferred outputs. Earlier checkpoints cannot resume. # # Bump this version whenever any producer above changes a stage output without # changing one of the explicit identity fields below. In particular, adding, @@ -298,7 +303,7 @@ # normalizes that logical view in memory. Moving between those encodings does # not change a producer's scalar output and therefore does not advance this # ledger; changing string values or the canonical logical dtype policy does. -POOL_STAGE_CHECKPOINT_MATERIALIZER_VERSION = 7 +POOL_STAGE_CHECKPOINT_MATERIALIZER_VERSION = 8 _PRIMARY_QRF_N_ESTIMATORS = 100 _ACS_TRANSFER_N_ESTIMATORS = 100 @@ -329,6 +334,9 @@ _STACKED_CHECKPOINT_IDENTITY_ARTIFACT_KIND = ( "populace_us_stacked_pool_checkpoint_identity" ) +# Version 13 binds native ACS humanitarian eligibility, pooled immigration +# control scaling, reconciliation, and the final composition gate. Earlier +# stacked checkpoints predate the source-aware status surface. # Version 12 binds the post-assembly household geography assignment authority, # target vintage, algorithm, operator order, and seed. Earlier checkpoints # predate the congressional-district support required by release preflight. @@ -336,7 +344,7 @@ # Earlier checkpoints must rebuild rather than resume with a nullable # s_corp_income leaf. Version 10 bound the complete late-resource semantics and # corrected outer order (the primary PUF callback is nested inside the DAG). -_STACKED_CHECKPOINT_MATERIALIZER_VERSION = 12 +_STACKED_CHECKPOINT_MATERIALIZER_VERSION = 13 _STACKED_RELEASE_ID_PATTERN = re.compile( r"^populace-us-2024-stacked-f(?:001|004|010|025|100)-s[0-9]+-" r"asec[0-9]+-acs[0-9]+-[0-9]{8}T[0-9]{6}Z-[0-9a-f]{8}$" @@ -391,14 +399,14 @@ class PoolBuildOutputs: @dataclass(frozen=True) class StackedPoolBuildResult: - """Input-only stacked pool and its two fresh terminal gate verdicts.""" + """Input-only stacked pool and its fresh terminal gate verdicts.""" frame: Frame stack_receipt: Mapping[str, object] assembly_receipt: Mapping[str, object] provenance_counts: Mapping[str, Mapping[str, object]] stage_receipts: Mapping[str, Mapping[str, object]] - terminal_gates: tuple[GateResult, GateResult] + terminal_gates: tuple[GateResult, ...] release_id: str qbi_transition_authority_sha256: str | None = None late_producer_transition_authority_sha256: str | None = None @@ -1648,9 +1656,7 @@ def _ordered_household_geography_receipt( dtype=np.float64, na_value=np.nan, ) - valid_ids = np.isfinite(household_ids) & ( - household_ids == np.floor(household_ids) - ) + valid_ids = np.isfinite(household_ids) & (household_ids == np.floor(household_ids)) if not valid_ids.all(): raise ValueError("Stacked geography output household IDs must be integral.") @@ -1718,9 +1724,11 @@ def _validate_stacked_geography_assignment_receipt( ) -> None: """Bind a live assembled-or-later Frame to its assignment receipt.""" - if receipt.get("artifact_kind") != ( - "populace_us_stacked_household_geography_assignment" - ) or receipt.get("schema_version") != 1: + if ( + receipt.get("artifact_kind") + != ("populace_us_stacked_household_geography_assignment") + or receipt.get("schema_version") != 1 + ): raise ValueError(f"{boundary}: geography assignment receipt is unsupported.") expected_contract = _stacked_geography_assignment_contract() if _json_ready(receipt.get("contract")) != _json_ready(expected_contract): @@ -1729,9 +1737,7 @@ def _validate_stacked_geography_assignment_receipt( target_districts ) if receipt.get("target_universe") != expected_universe: - raise ValueError( - f"{boundary}: congressional-district target universe changed." - ) + raise ValueError(f"{boundary}: congressional-district target universe changed.") output = receipt.get("output") if not isinstance(output, Mapping): raise ValueError(f"{boundary}: geography assignment output is missing.") @@ -1759,9 +1765,10 @@ def _validate_stacked_geography_assignment_receipt( clone_index = pd.to_numeric(household[clone_column], errors="coerce") native_household = household.loc[clone_index.eq(0)] expected_order = receipt.get("pre_assignment_household_order") - if ( - len(native_household) != assigned_rows - or expected_order != _ordered_household_id_receipt(native_household) + if len( + native_household + ) != assigned_rows or expected_order != _ordered_household_id_receipt( + native_household ): raise ValueError( f"{boundary}: ordered native household IDs differ from the seeded " @@ -1901,9 +1908,7 @@ def _assign_stacked_household_geography( "schema_version": 1, "contract": _stacked_geography_assignment_contract(), "pre_assignment_household_order": pre_assignment_order, - "assigned_household_geography": _ordered_household_geography_receipt( - household - ), + "assigned_household_geography": _ordered_household_geography_receipt(household), "target_universe": _target_congressional_district_universe_receipt( target_districts ), @@ -3994,9 +3999,10 @@ def primary_puf_producer(primary_input: Frame): simulation_frame, tail_manifest=tail_manifest, ) + immigration = us_immigration_composition_gate(current) # Manifest conversion is itself the final canonical-authority check and # deliberately happens before publication or readiness is asserted. - GateReport((completeness, battery)).to_manifest() + GateReport((completeness, battery, immigration)).to_manifest() mark_phase("terminal_gates") counts = spine_provenance_counts( current, @@ -4018,7 +4024,7 @@ def primary_puf_producer(primary_input: Frame): assembly_receipt=dict(assembly_receipt), provenance_counts=counts, stage_receipts=receipts, - terminal_gates=(completeness, battery), + terminal_gates=(completeness, battery, immigration), release_id=release_id, qbi_transition_authority_sha256=qbi_transition_authority_sha256, late_producer_transition_authority_sha256=( @@ -4489,9 +4495,7 @@ def _write_stacked_outputs( materializer_version=US_MULTISPINE_POOL_H5_MATERIALIZER_VERSION, root_attributes={ CONGRESSIONAL_DISTRICT_VINTAGE_CROSSWALK_SHA256_ATTR: ( - verified_inputs[ - _STACKED_CD_CROSSWALK_INPUT_ROLE - ].actual_sha256 + verified_inputs[_STACKED_CD_CROSSWALK_INPUT_ROLE].actual_sha256 ), CONGRESSIONAL_DISTRICT_VINTAGE_TARGET_ATTR: ( CURRENT_CONGRESSIONAL_DISTRICT_VINTAGE diff --git a/tools/build_us_release_input_coverage_manifest.py b/tools/build_us_release_input_coverage_manifest.py index 10043a98ff..33d5044456 100644 --- a/tools/build_us_release_input_coverage_manifest.py +++ b/tools/build_us_release_input_coverage_manifest.py @@ -1436,7 +1436,7 @@ "only through the REFUGEE/ASYLEE/DEPORTATION_WITHHELD/" "PAROLED_ONE_YEAR values of immigration_status_str. The source " "stage draws roughly 160k refugees (74% reporting Medicaid on the " - "2024 ASEC), 160k asylees, and 405k parolees to cited DHS/OHSS " + "2024 ASEC), 155k asylees, and 405k parolees to cited DHS/OHSS " "stocks, so restoring their eligibility must re-enroll anchored " "takers and move person-level medicaid dollars by far more than " "the floor. A ~$0 score means the humanitarian statuses regressed " diff --git a/tools/generate_us_bundle_from_constants.py b/tools/generate_us_bundle_from_constants.py index 9909da4782..542b75c2de 100644 --- a/tools/generate_us_bundle_from_constants.py +++ b/tools/generate_us_bundle_from_constants.py @@ -122,7 +122,7 @@ def _domain_builders() -> dict[str, Callable[[], object]]: # frozen files, is the forward YAML -> legacy-payload path. FROZEN_LEGACY_RESOURCE_SHA256 = { "source_stages.json": ( - "619a974abd80c34201a0ccf813968ebc09ed52e144ff1d39722a4d9be7bd7821" + "413174820cfb57f85783831dbbf445f007d06128d31562a1234e998e627bd3aa" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" diff --git a/tools/spec_engine_coverage.py b/tools/spec_engine_coverage.py index 3fa3373d59..ee002b6b37 100644 --- a/tools/spec_engine_coverage.py +++ b/tools/spec_engine_coverage.py @@ -41,7 +41,7 @@ REPORT_SCHEMA_VERSION = 3 EXPECTED_POINTER_INVENTORY_SHA256 = ( - "879bf3c70ad0d10c7a123a3b3194d3822f3015bed4bfba2b4a6703885a6bc95a" + "db02c16cd852fb80d7316d69990cfeb9e2b2f9dec87ddb18a44178ea57422397" ) DEFAULT_REPORT_PATH = ( Path(__file__).resolve().parents[1] @@ -202,9 +202,10 @@ def valid_sha256(value: object) -> bool: if not isinstance(compiler_abi, Mapping): failures.append("compiler_ir_abi is missing") else: - if set(compiler_abi) != {"version", "sha256"} or compiler_abi.get( - "version" - ) != COMPILER_IR_ABI_VERSION: + if ( + set(compiler_abi) != {"version", "sha256"} + or compiler_abi.get("version") != COMPILER_IR_ABI_VERSION + ): failures.append("compiler_ir_abi contract differs") if not valid_sha256(compiler_abi.get("sha256")): failures.append("compiler_ir_abi/sha256 is invalid") @@ -249,9 +250,9 @@ def valid_sha256(value: object) -> bool: if not isinstance(claims, list): failures.append("field_usage claim receipts are incomplete") claims = [] - if field_usage.get("claim_count") != len(definitions) or len( - claims - ) != len(definitions): + if field_usage.get("claim_count") != len(definitions) or len(claims) != len( + definitions + ): failures.append( "field_usage claim count differs from the closed claim registry" ) diff --git a/tools/us_bundle_generation/imputation.py b/tools/us_bundle_generation/imputation.py index d80696bf45..bb80f89f2a 100644 --- a/tools/us_bundle_generation/imputation.py +++ b/tools/us_bundle_generation/imputation.py @@ -871,6 +871,15 @@ def _normalise_transfer_execution( ) ) adult_care.pop("enabled") + humanitarian_immigration = deepcopy( + dict( + _mapping_like( + post_transfer["humanitarian_immigration"], + "humanitarian-immigration post-transfer contract", + ) + ) + ) + humanitarian_immigration.pop("enabled") result["predictor_bindings"] = { "person_required": "acs_person_required", "person_optional": [ @@ -890,6 +899,17 @@ def _normalise_transfer_execution( }, "contract": adult_care, }, + "humanitarian_immigration": { + "activation": { + "all_targets": list( + _array_like( + result["immigration_status_targets"], + "immigration status targets", + ) + ) + }, + "contract": humanitarian_immigration, + }, "schedule_d_capital_gain_distributions": { "activation": { "derive_schedule_d": True, @@ -2151,7 +2171,7 @@ def _assert_invariants( len(primary_node["outputs"]), late_authored_output_count, len(tolerated_receipts), - ) != (2742, 92, 227, 35, 0, 212): + ) != (2749, 92, 227, 35, 0, 212): raise RuntimeError("US producer graph input/output/absence counts changed.") if len(set(tolerated_receipts)) != len(tolerated_receipts): raise RuntimeError("US producer graph absence receipt IDs are not unique.") From 405f9c2d660f7c02b51220d138ef4dd2bc713f03 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 29 Aug 2026 01:35:55 +0200 Subject: [PATCH 11/49] Align immigration controls with Pew methodology --- docs/evidence/spec-engine/us-f0-coverage.json | 70 ++--- .../build/spec_engine/field_usage.py | 8 +- .../build/spec_engine/inventory_coverage.py | 22 +- .../src/microcosm/build/us/source_stages.json | 14 +- .../microcosm/build/us/spec/imputation.yaml | 27 +- .../src/microcosm/build/us/spec/sources.yaml | 39 ++- .../microcosm/build/us_runtime/__init__.py | 20 +- .../build/us_runtime/asec_checkpoint.py | 25 +- .../us_runtime/education_assistance_source.py | 143 ++++++++- .../microcosm/build/us_runtime/immigration.py | 293 ++++++++++++++---- .../us_runtime/us_late_producer_registry.py | 3 +- .../tests/test_spec_engine_compiler_ir.py | 2 +- .../tests/test_spec_engine_coverage_tool.py | 12 +- .../tests/test_spec_engine_field_usage.py | 10 +- .../test_spec_engine_imputation_semantics.py | 2 +- .../test_spec_engine_inventory_coverage.py | 2 +- .../tests/test_spec_engine_legacy_adapter.py | 4 +- ...spec_engine_stacked_authority_semantics.py | 6 +- .../tests/test_us_asec_checkpoint.py | 6 +- .../tests/test_us_bundle_core_contracts.py | 2 +- .../test_us_education_assistance_source.py | 54 +++- .../tests/test_us_immigration.py | 229 ++++++++++++-- .../tests/test_us_immigration_pool_scaling.py | 7 +- .../tests/test_us_late_producer_dag.py | 8 +- .../tests/test_us_multispine_pool_tool.py | 2 +- .../microcosm-build/tests/test_us_plan.py | 4 +- .../tests/test_us_puf_support_base_builder.py | 68 +++- .../tests/test_us_spec_bundle.py | 6 +- tools/build_us_puf_support_base.py | 104 ++++++- tools/generate_us_bundle_from_constants.py | 2 +- tools/spec_engine_coverage.py | 2 +- tools/us_bundle_generation/imputation.py | 2 +- 32 files changed, 948 insertions(+), 250 deletions(-) diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index b4cc33db4b..618147178a 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -4,9 +4,9 @@ "version": 1 }, "country": "us", - "documentation_sha256": "bc7bf008ee59fcc44890b06cb2a8a085e388a9764d2a5a7aa5e870caeb0112e4", + "documentation_sha256": "02ac100a7aaa1770930dd6005b2d7f26550560e769c4ffb8d642b76f1774dd84", "field_usage": { - "authored_normative_field_count": 32528, + "authored_normative_field_count": 32521, "claim_count": 49, "claims": [ { @@ -274,8 +274,8 @@ ], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 24561, - "pointer_sha256": "70c1a41f9c826fdf47dc413fe76876b67e2b17abb00daba0e935d75f9dbaaa1e", + "pointer_count": 24554, + "pointer_sha256": "18140dbc554c2a17a673fec5ae5839f504d4b9980c5d8f1c1ccb545342977cb4", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/spec~1imputation.yaml/producer_graph", @@ -772,20 +772,20 @@ "verifier": "vintages" } ], - "configuration_field_count": 42545, - "consumed_field_count": 42545, + "configuration_field_count": 42538, + "consumed_field_count": 42538, "generation0_effect_counts": { - "legacy_behavior": 38581, + "legacy_behavior": 38574, "no_generation0_effect": 3964 }, "mode_counts": { - "compiler_semantic": 28074, + "compiler_semantic": 28067, "front_end_validation": 348, "identity_only": 103, "legacy_behavior": 14020 }, "multiple_primary_use_field_count": 0, - "pointer_inventory_sha256": "29c8dbe6c5335217b3d5ac7fb239a75c48b33362637611dcf297597eca20352f", + "pointer_inventory_sha256": "faef18fd543699ba65a91f3ffaf6fdf38a807ffdc88bc3debfeb970881050d62", "resolved_binding_field_count": 10017, "unused_field_count": 0 }, @@ -810,7 +810,7 @@ "primary_targets": 65, "producer_authored_outputs": 92, "producer_compiled_outputs": 227, - "producer_inputs": 2751, + "producer_inputs": 2750, "producer_nodes": 38, "producer_virtual_resources": 75, "release_rungs": 5, @@ -1074,11 +1074,11 @@ "legacy_adapter.stacked_checkpoint_static_components" ], "expected": { - "schedule_sha256": "b7e40a89f5100c34b748ce960c209807b98d0f5e2c5834bb4c250aec733a37dc" + "schedule_sha256": "88bc9243a3518982ae951c3de21bd55877e296ce4fcb183b9bee420d3a684b10" }, "failures": [], "observed": { - "schedule_sha256": "b7e40a89f5100c34b748ce960c209807b98d0f5e2c5834bb4c250aec733a37dc" + "schedule_sha256": "88bc9243a3518982ae951c3de21bd55877e296ce4fcb183b9bee420d3a684b10" }, "status": "covered" }, @@ -1250,14 +1250,14 @@ "expected": { "edges": 71, "nodes": 38, - "schedule_sha256": "b7e40a89f5100c34b748ce960c209807b98d0f5e2c5834bb4c250aec733a37dc", + "schedule_sha256": "88bc9243a3518982ae951c3de21bd55877e296ce4fcb183b9bee420d3a684b10", "waves": 6 }, "failures": [], "observed": { "edges": 71, "nodes": 38, - "schedule_sha256": "b7e40a89f5100c34b748ce960c209807b98d0f5e2c5834bb4c250aec733a37dc", + "schedule_sha256": "88bc9243a3518982ae951c3de21bd55877e296ce4fcb183b9bee420d3a684b10", "waves": 6 }, "status": "covered" @@ -1275,11 +1275,11 @@ ], "expected": { "relation": "source rows preserved exactly", - "rows": 2751 + "rows": 2750 }, "failures": [], "observed": { - "rows": 2751 + "rows": 2750 }, "status": "covered" }, @@ -1338,12 +1338,12 @@ ], "expected": { "nodes": 38, - "sha256": "90714af60b75624e0347258b511fa09b35e47d904d44b96fbd7276306a334381" + "sha256": "014f90315324f72f45ece928bccc44f34b4d1e95c0582f1d7d73620f482da15f" }, "failures": [], "observed": { "nodes": 38, - "sha256": "90714af60b75624e0347258b511fa09b35e47d904d44b96fbd7276306a334381" + "sha256": "014f90315324f72f45ece928bccc44f34b4d1e95c0582f1d7d73620f482da15f" }, "status": "covered" }, @@ -1361,12 +1361,12 @@ ], "expected": { "producer_count": 38, - "sha256": "14005db84ddedc9a668c87411100a6e20a1107f2171c7ae2d9060999b6340e0e" + "sha256": "db175a952340b3ff2774215f320c4d19592c06b9959442639a6c561055f2442d" }, "failures": [], "observed": { "producer_count": 38, - "sha256": "14005db84ddedc9a668c87411100a6e20a1107f2171c7ae2d9060999b6340e0e" + "sha256": "db175a952340b3ff2774215f320c4d19592c06b9959442639a6c561055f2442d" }, "status": "covered" }, @@ -1669,13 +1669,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "6a91f7621cfd0a29ec80ce95cde714684f8787ea70793989fb4eb1ccb9a1d627", - "protocol_sha256": "8094bb67bebc0bbc99a80965de184e8d0d0f5d9c821d0c2ddf3e94fd3f9d73a3" + "map_sha256": "f5301b13c672bd21cb33e357f0c25c762acab3ef44ca15ed88895ff2ffebc0af", + "protocol_sha256": "61db02328d1844c9cb9c490599e33a25aea86eafadc7b940147585340f948d92" }, "failures": [], "observed": { - "map_sha256": "6a91f7621cfd0a29ec80ce95cde714684f8787ea70793989fb4eb1ccb9a1d627", - "protocol_sha256": "8094bb67bebc0bbc99a80965de184e8d0d0f5d9c821d0c2ddf3e94fd3f9d73a3" + "map_sha256": "f5301b13c672bd21cb33e357f0c25c762acab3ef44ca15ed88895ff2ffebc0af", + "protocol_sha256": "61db02328d1844c9cb9c490599e33a25aea86eafadc7b940147585340f948d92" }, "status": "covered" }, @@ -1690,7 +1690,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "8094bb67bebc0bbc99a80965de184e8d0d0f5d9c821d0c2ddf3e94fd3f9d73a3", + "implementation_sha256": "61db02328d1844c9cb9c490599e33a25aea86eafadc7b940147585340f948d92", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1711,7 +1711,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "8094bb67bebc0bbc99a80965de184e8d0d0f5d9c821d0c2ddf3e94fd3f9d73a3", + "implementation_sha256": "61db02328d1844c9cb9c490599e33a25aea86eafadc7b940147585340f948d92", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1788,12 +1788,12 @@ "legacy_adapter.imputation.late_producer_resource_semantics" ], "expected": { - "sha256": "6bf67f40b1afd2c63a7d7fa2c1f448570f4b7ab186475521966af65013de9e71", + "sha256": "b82d911a0263ebc1b6d5d8f82bb1076fd8d779bea49072ec42f02a8fe027cbe3", "stages": 37 }, "failures": [], "observed": { - "sha256": "6bf67f40b1afd2c63a7d7fa2c1f448570f4b7ab186475521966af65013de9e71", + "sha256": "b82d911a0263ebc1b6d5d8f82bb1076fd8d779bea49072ec42f02a8fe027cbe3", "stages": 37 }, "status": "covered" @@ -1853,11 +1853,11 @@ "legacy_adapter.stacked_checkpoint_static_components" ], "expected": { - "sha256": "4c8a2d5478a4fef9c77f298f692f1c5f8667045c16c940855845d797023bc409" + "sha256": "406b2cf93a7fb94dc63f1a24a52cfb20362ab83195eb8ebf61d4ef43072280ec" }, "failures": [], "observed": { - "sha256": "4c8a2d5478a4fef9c77f298f692f1c5f8667045c16c940855845d797023bc409" + "sha256": "406b2cf93a7fb94dc63f1a24a52cfb20362ab83195eb8ebf61d4ef43072280ec" }, "status": "covered" }, @@ -1900,7 +1900,7 @@ "alpha", "zeta" ], - "sha256": "a45c2d6856195861174a493b3e994e94583c13744f4a146cda400fd86b688deb" + "sha256": "a53da3c26dfaf8f400fb74ef92f79555dd0c152319aa50317b52c5603d0af7b4" }, "failures": [], "observed": { @@ -1924,7 +1924,7 @@ "alpha", "zeta" ], - "sha256": "a45c2d6856195861174a493b3e994e94583c13744f4a146cda400fd86b688deb" + "sha256": "a53da3c26dfaf8f400fb74ef92f79555dd0c152319aa50317b52c5603d0af7b4" }, "status": "covered" }, @@ -1988,7 +1988,7 @@ "take_up_contract", "us_qbi_reconciliation_contract" ], - "sha256": "e72a567aba78d3fa0d20abd9a390cd3ee414232177b6603c4ae5c625575466bb" + "sha256": "fbc3631977c43ba558f89b3f36c1b8d0d3958a593f93198c0c5b5833cd18999b" }, "status": "covered" }, @@ -2612,7 +2612,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "97922fe1d5c5737c7d51e8af1c77fcad97d564935e5229fbe10b7ef203a123e0" + "spec_sha256": "ae83c7a32a4a2970070f71b95aa7de771a56ab435f71004c11ee2e6b181fe07b" } }, "report_schema_version": 3, @@ -2622,7 +2622,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "97922fe1d5c5737c7d51e8af1c77fcad97d564935e5229fbe10b7ef203a123e0" + "spec_sha256": "ae83c7a32a4a2970070f71b95aa7de771a56ab435f71004c11ee2e6b181fe07b" }, "status": "pass" } diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py index f1b8ccaaa9..638a4dd504 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py @@ -26,9 +26,9 @@ ) from .schemas import load_schema_registry -EXPECTED_AUTHORED_FIELD_COUNT = 32_528 +EXPECTED_AUTHORED_FIELD_COUNT = 32_521 EXPECTED_RESOLVED_BINDING_FIELD_COUNT = 10_017 -EXPECTED_CONFIGURATION_FIELD_COUNT = 42_545 +EXPECTED_CONFIGURATION_FIELD_COUNT = 42_538 class FieldUsageError(AssertionError): @@ -421,8 +421,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "e1dd7dc5123ab0f39d08ea4939d98dd09a6fdb8e7449a7ca3125fb1ddbd5b4e9", ), "imputation_producer_graph": ( - 24_561, - "70c1a41f9c826fdf47dc413fe76876b67e2b17abb00daba0e935d75f9dbaaa1e", + 24_554, + "18140dbc554c2a17a673fec5ae5839f504d4b9980c5d8f1c1ccb545342977cb4", ), "imputation_transfer_execution": ( 115, diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 7ba2c99dc6..6ecd560f37 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -361,20 +361,20 @@ EXPECTED_HASHES = { "acs_group_predictors": "a927bb7ecf3e84f54c93583ab79318654514ac546aefafba67da5285615fbd60", "acs_person_predictors": "878c788a6f037d7aca12b3586ea034eff04f3034ffa11935a736493042551f25", - "authority": "4c8a2d5478a4fef9c77f298f692f1c5f8667045c16c940855845d797023bc409", + "authority": "406b2cf93a7fb94dc63f1a24a52cfb20362ab83195eb8ebf61d4ef43072280ec", "early_families": "4aa9f736fd76e83955477ad1667e58f48f264783f05bdc7f0102cd32d61323bd", - "full_checkpoint": "a45c2d6856195861174a493b3e994e94583c13744f4a146cda400fd86b688deb", + "full_checkpoint": "a53da3c26dfaf8f400fb74ef92f79555dd0c152319aa50317b52c5603d0af7b4", "gap_fill_schedule": "1c31f9868f7884347cc19cf1ff65da43f950b9114941a715bab168246db414a7", - "graph_nodes": "90714af60b75624e0347258b511fa09b35e47d904d44b96fbd7276306a334381", + "graph_nodes": "014f90315324f72f45ece928bccc44f34b4d1e95c0582f1d7d73620f482da15f", "geography_assignment": "f49425ca8734ac559c73cf44f6458d86d3162a48956b98a27e6e758959361585", "late_families": "d91f9ff0eb52f43e7b6eed3d5c58c37abe1620c3a11021da15dae9c10e16d382", - "late_resource_semantics": "14005db84ddedc9a668c87411100a6e20a1107f2171c7ae2d9060999b6340e0e", - "late_schedule": "b7e40a89f5100c34b748ce960c209807b98d0f5e2c5834bb4c250aec733a37dc", + "late_resource_semantics": "db175a952340b3ff2774215f320c4d19592c06b9959442639a6c561055f2442d", + "late_schedule": "88bc9243a3518982ae951c3de21bd55877e296ce4fcb183b9bee420d3a684b10", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "987b501c695e31f45521c4a178528f75ab3df22c09bc407b182213b2de99ee57", - "seed_map": "6a91f7621cfd0a29ec80ce95cde714684f8787ea70793989fb4eb1ccb9a1d627", - "seed_protocol": "8094bb67bebc0bbc99a80965de184e8d0d0f5d9c821d0c2ddf3e94fd3f9d73a3", - "source_manifest": "6bf67f40b1afd2c63a7d7fa2c1f448570f4b7ab186475521966af65013de9e71", + "seed_map": "f5301b13c672bd21cb33e357f0c25c762acab3ef44ca15ed88895ff2ffebc0af", + "seed_protocol": "61db02328d1844c9cb9c490599e33a25aea86eafadc7b940147585340f948d92", + "source_manifest": "b82d911a0263ebc1b6d5d8f82bb1076fd8d779bea49072ec42f02a8fe027cbe3", "take_up": "fa186daea0f8dd641cc470e41d1a2953f887d45282ec990201298f47bedf8d4d", "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", } @@ -442,7 +442,7 @@ "primary_targets": 65, "producer_authored_outputs": 92, "producer_compiled_outputs": 227, - "producer_inputs": 2_751, + "producer_inputs": 2_750, "producer_nodes": 38, "producer_virtual_resources": 75, "release_rungs": 5, @@ -1018,7 +1018,7 @@ def add( "producer_inputs_exact", clauses={ "producer input rows differ": inputs_exact, - "input row count differs": input_count == 2751, + "input row count differs": input_count == 2750, }, homes=("/imputation/producer_graph/nodes/*/inputs",), consumers=( @@ -1026,7 +1026,7 @@ def add( "compiler_ir.node_slices", ), observed={"rows": input_count}, - expected={"rows": 2751, "relation": "source rows preserved exactly"}, + expected={"rows": 2750, "relation": "source rows preserved exactly"}, ) outputs_exact = set(expected_outputs) == set(compiled_by_id) and all( _json_equal( diff --git a/packages/microcosm-build/src/microcosm/build/us/source_stages.json b/packages/microcosm-build/src/microcosm/build/us/source_stages.json index d471f38808..401468cb01 100644 --- a/packages/microcosm-build/src/microcosm/build/us/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/us/source_stages.json @@ -1866,14 +1866,14 @@ { "stage": "immigration_status", "survey": "CPS ASEC + published unauthorized-population estimates", - "source": "https://www.pewresearch.org/short-reads/2024/07/22/what-we-know-about-unauthorized-immigrants-living-in-the-us/", + "source": "https://www.pewresearch.org/race-and-ethnicity/2025/08/21/u-s-unauthorized-immigrant-population-reached-a-record-14-million-in-2023/", "grain": "person", "artifacts": [ { "kind": "public_microdata", "format": "census_asec", "vintage": "build_year", - "locator": "Census CPS ASEC person file citizenship (PRCITSHP), entry year (PEINUSYR), nativity (PENATVTY), and program-participation fields" + "locator": "Census CPS ASEC person file citizenship (PRCITSHP), entry year (PEINUSYR), nativity (PENATVTY), labor-force status (A_LFSR), and program-participation fields" }, { "kind": "administrative_table", @@ -1893,16 +1893,16 @@ "seed_from_build_config": true, "time_period_from_build_config": true, "undocumented_workers": { - "target": 8300000, - "source": "https://www.pewresearch.org/short-reads/2024/07/22/what-we-know-about-unauthorized-immigrants-living-in-the-us/" + "target": 9700000, + "source": "https://www.pewresearch.org/race-and-ethnicity/2025/08/21/u-s-unauthorized-immigrant-population-reached-a-record-14-million-in-2023/" }, "undocumented_students": { "target": 408000, "source": "https://www.higheredimmigrationportal.org/research/undocumented-students-in-higher-education-updated-march-2021/" }, "undocumented_population_anchor": { - "value": 11000000, - "source": "https://www.pewresearch.org/short-reads/2024/07/22/what-we-know-about-unauthorized-immigrants-living-in-the-us/" + "value": 14000000, + "source": "https://www.pewresearch.org/race-and-ethnicity/2025/08/21/u-s-unauthorized-immigrant-population-reached-a-record-14-million-in-2023/" }, "humanitarian_status_stocks": { "paroled_one_year": { @@ -1968,7 +1968,7 @@ "ssn_card_type", "immigration_status_str" ], - "notes": "Citizenship is measured (PRCITSHP), not imputed. Non-citizens with any ASEC-UA legal-status indicator (Van Hook et al., SSRN 4662801) become OTHER_NON_CITIZEN. Humanitarian/temporary-protection statuses (microcosm #767) are drawn between the indicators and the EAD spill, sequentially and mutually exclusively in decreasing order of target precision \u2014 PAROLED_ONE_YEAR per program origin (OAW Afghans 73,566 as of 2022-03-31; U4U Ukrainians over 158,000 as of 2023-09-30; CHNV Nicaraguans 93,070 and Venezuelans 117,330 paroled through 2024-12), REFUGEE (FY2023+FY2024 admissions, 60,050+100,060 OHSS \u2014 refugees must apply for LPR adjustment after one year, so the not-yet-adjusted stock is about the trailing two years), ASYLEE (FY2022-FY2024 grants 35,080+51,410+68,060=154,550 OHSS, rounded to 155,000; 88% of 2015-22 affirmative adult grantees had adjusted by end-2024, so about three trailing years), then TPS per country (CRS RS20844 Table 1 as of 2025-03-31, with required-arrival cutoffs binding at PEINUSYR granularity: El Salvador <=2001, Honduras/Nicaragua <=1999, Nepal <=2015). Candidates are restricted by program origin countries (Census Appendix I PENATVTY codes) and arrival windows; Cuba/Haiti-born are excluded everywhere (the CUBAN_HAITIAN_ENTRANT class is the better statutory label and keeps H.R.1 eligibility \u2014 Haiti TPS 330,735 rides that label), the DACA statutory cohort is excluded (dual TPS/DACA holders stay DACA), and Ukraine/Afghanistan TPS registrants (101,150/8,105) are carried by the parole draw since they are overwhelmingly the same U4U/OAW parolees. Parole targets are gross admissions: cohort members who later won asylum keep the parole label, which is channel-equivalent under H.R.1 (both classes lose Medicaid/SNAP/PTC on identical dates). DEPORTATION_WITHHELD carries an explicit zero target: EOIR grants about 2.2k withholdings a year (CRS IN12501: under 1% of 666,177 FY2024 decisions) and publishes no stock, so the accumulated population is below the CPS's resolution for a defensible draw. CONDITIONAL_ENTRANT is deliberately not emitted (the INA 203(a)(7) class closed in 1980). REFUGEE/ASYLEE draw from the indicator-documented pool only; parole/TPS may draw residual-pool people, who then receive NON_CITIZEN_VALID_EAD SSN cards (both programs confer employment authorization), consistent with Pew's own accounting of TPS/parolees inside its unauthorized totals. The EAD worker/student spill then binds the remaining residual pool to Pew's 8.3M workers and the Higher Ed Immigration Portal's ~408k students \u2014 still the only forced margins. The total undocumented population is emergent and gated against Pew's 11.0M 2022 anchor with a coarse band; per-category humanitarian masses are gated against their cited targets with a coarse band (the ASEC undercovers 2022-24 arrivals \u2014 the Ukraine parole pool saturates below its admin count); reconciling levels is the calibration lane's job. DACA (arrival cohort among EAD holders) and CUBAN_HAITIAN_ENTRANT (nativity plus post-1980 arrival) keep their statutory tests; other documented non-citizens stay LEGAL_PERMANENT_RESIDENT, the modal true status." + "notes": "Citizenship is measured (PRCITSHP), not imputed. Non-citizens with any ASEC-UA legal-status indicator (Van Hook et al., SSRN 4662801) become OTHER_NON_CITIZEN. Humanitarian/temporary-protection statuses (microcosm #767) are drawn between the indicators and the EAD spill, sequentially and mutually exclusively in decreasing order of target precision \u2014 PAROLED_ONE_YEAR per program origin (OAW Afghans 73,566 as of 2022-03-31; U4U Ukrainians over 158,000 as of 2023-09-30; CHNV Nicaraguans 93,070 and Venezuelans 117,330 paroled through 2024-12), REFUGEE (FY2023+FY2024 admissions, 60,050+100,060 OHSS \u2014 refugees must apply for LPR adjustment after one year, so the not-yet-adjusted stock is about the trailing two years), ASYLEE (FY2022-FY2024 grants 35,080+51,410+68,060=154,550 OHSS, rounded to 155,000; 88% of 2015-22 affirmative adult grantees had adjusted by end-2024, so about three trailing years), then TPS per country (CRS RS20844 Table 1 as of 2025-03-31, with required-arrival cutoffs binding at PEINUSYR granularity: El Salvador <=2001, Honduras/Nicaragua <=1999, Nepal <=2015). Candidates are restricted by program origin countries (Census Appendix I PENATVTY codes) and arrival windows; Cuba/Haiti-born are excluded everywhere (the CUBAN_HAITIAN_ENTRANT class is the better statutory label and keeps H.R.1 eligibility \u2014 Haiti TPS 330,735 rides that label), the DACA statutory cohort is excluded (dual TPS/DACA holders stay DACA), and Ukraine/Afghanistan TPS registrants (101,150/8,105) are carried by the parole draw since they are overwhelmingly the same U4U/OAW parolees. Parole targets are gross admissions: cohort members who later won asylum keep the parole label, which is channel-equivalent under H.R.1 (both classes lose Medicaid/SNAP/PTC on identical dates). DEPORTATION_WITHHELD carries an explicit zero target: EOIR grants about 2.2k withholdings a year (CRS IN12501: under 1% of 666,177 FY2024 decisions) and publishes no stock, so the accumulated population is below the CPS's resolution for a defensible draw. CONDITIONAL_ENTRANT is deliberately not emitted (the INA 203(a)(7) class closed in 1980). REFUGEE/ASYLEE draw from the indicator-documented pool only; parole/TPS may draw residual-pool people, who then receive NON_CITIZEN_VALID_EAD SSN cards (both programs confer employment authorization). Pew's 2025 report supersedes its earlier series and estimates a coherent 2023 pair of 14.0M unauthorized immigrants and 9.7M in the labor force. Its modeled universe is broader than the engine's UNDOCUMENTED enum: it retains DACA recipients, parolees, TPS holders, similar temporary protections, and the residual-EAD subset of the engine's broad CUBAN_HAITIAN_ENTRANT class, while excluding refugees, people already granted asylum, and assumed-documented Cuban/Haitian entrants. The worker EAD spill binds that modeled universe to 9.7M using actual ASEC labor-force status (A_LFSR codes 1-4 at ages 16+) restored from the same pinned official ASEC archives by exact PERIDNUM join, never prior-year WSAL_VAL/SEMP_VAL earnings. The student spill binds the corresponding broad residual universe to the Higher Ed Immigration Portal's ~408k students. The total modeled Pew-defined unauthorized population is emergent and gated against the 14.0M 2023 anchor with a coarse band; per-category humanitarian masses are gated against their cited targets with a coarse band (the ASEC undercovers 2022-24 arrivals \u2014 the Ukraine parole pool saturates below its admin count); reconciling levels is the calibration lane's job. DACA (arrival cohort among EAD holders) and CUBAN_HAITIAN_ENTRANT (nativity plus post-1980 arrival) keep their statutory tests; other documented non-citizens stay LEGAL_PERMANENT_RESIDENT, the modal true status." }, { "stage": "hours_worked", diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/imputation.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/imputation.yaml index 8e63ba6e6d..1548aca4de 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/imputation.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/imputation.yaml @@ -9541,6 +9541,15 @@ producer_graph: producing_stage: post_clone_input_surface required_scope: asec_source tolerated_absence_receipts: [] + - alternatives: + - - column: A_LFSR + entity: person + value_kind: finite_numeric + column: '@effective:raw_person:A_LFSR' + entity: person + producing_stage: post_clone_input_surface + required_scope: asec_source + tolerated_absence_receipts: [] - alternatives: - - column: A_MARITL entity: person @@ -9694,15 +9703,6 @@ producer_graph: producing_stage: post_clone_input_surface required_scope: asec_source tolerated_absence_receipts: [] - - alternatives: - - - column: SEMP_VAL - entity: person - value_kind: finite_numeric - column: '@effective:raw_person:SEMP_VAL' - entity: person - producing_stage: post_clone_input_surface - required_scope: asec_source - tolerated_absence_receipts: [] - alternatives: - - column: SPM_CAPHOUSESUB entity: person @@ -9730,15 +9730,6 @@ producer_graph: producing_stage: post_clone_input_surface required_scope: asec_source tolerated_absence_receipts: [] - - alternatives: - - - column: WSAL_VAL - entity: person - value_kind: finite_numeric - column: '@effective:raw_person:WSAL_VAL' - entity: person - producing_stage: post_clone_input_surface - required_scope: asec_source - tolerated_absence_receipts: [] - alternatives: - - column: '@resolved_weight' entity: person diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml index b20d6856e7..7234703744 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml @@ -99,7 +99,7 @@ sources: stage_asset: id: source_stages path: microcosm.build.us/source_stages.json - sha256: 413174820cfb57f85783831dbbf445f007d06128d31562a1234e998e627bd3aa + sha256: 788b815f748abb2061c41efb0cec4cc4952435dbcb4448ecf21ccac85a5ccec2 stage_manifest: version: 1 country: us @@ -1884,14 +1884,14 @@ stages: formula-owned employment_income_last_year; the signed self-employment leaf and availability flag persist. - stage: immigration_status survey: CPS ASEC + published unauthorized-population estimates - source: https://www.pewresearch.org/short-reads/2024/07/22/what-we-know-about-unauthorized-immigrants-living-in-the-us/ + source: https://www.pewresearch.org/race-and-ethnicity/2025/08/21/u-s-unauthorized-immigrant-population-reached-a-record-14-million-in-2023/ grain: person artifacts: - kind: public_microdata format: census_asec vintage: build_year locator: Census CPS ASEC person file citizenship (PRCITSHP), entry year (PEINUSYR), nativity (PENATVTY), - and program-participation fields + labor-force status (A_LFSR), and program-participation fields - kind: administrative_table format: published_estimate vintage: latest_available @@ -1907,14 +1907,14 @@ stages: seed_from_build_config: true time_period_from_build_config: true undocumented_workers: - target: 8300000 - source: https://www.pewresearch.org/short-reads/2024/07/22/what-we-know-about-unauthorized-immigrants-living-in-the-us/ + target: 9700000 + source: https://www.pewresearch.org/race-and-ethnicity/2025/08/21/u-s-unauthorized-immigrant-population-reached-a-record-14-million-in-2023/ undocumented_students: target: 408000 source: https://www.higheredimmigrationportal.org/research/undocumented-students-in-higher-education-updated-march-2021/ undocumented_population_anchor: - value: 11000000 - source: https://www.pewresearch.org/short-reads/2024/07/22/what-we-know-about-unauthorized-immigrants-living-in-the-us/ + value: 14000000 + source: https://www.pewresearch.org/race-and-ethnicity/2025/08/21/u-s-unauthorized-immigrant-population-reached-a-record-14-million-in-2023/ humanitarian_status_stocks: paroled_one_year: afghanistan: @@ -1982,16 +1982,21 @@ stages: FY2024 decisions) and publishes no stock, so the accumulated population is below the CPS''s resolution for a defensible draw. CONDITIONAL_ENTRANT is deliberately not emitted (the INA 203(a)(7) class closed in 1980). REFUGEE/ASYLEE draw from the indicator-documented pool only; parole/TPS may draw residual-pool - people, who then receive NON_CITIZEN_VALID_EAD SSN cards (both programs confer employment authorization), - consistent with Pew''s own accounting of TPS/parolees inside its unauthorized totals. The EAD worker/student - spill then binds the remaining residual pool to Pew''s 8.3M workers and the Higher Ed Immigration - Portal''s ~408k students — still the only forced margins. The total undocumented population is emergent - and gated against Pew''s 11.0M 2022 anchor with a coarse band; per-category humanitarian masses are - gated against their cited targets with a coarse band (the ASEC undercovers 2022-24 arrivals — the - Ukraine parole pool saturates below its admin count); reconciling levels is the calibration lane''s - job. DACA (arrival cohort among EAD holders) and CUBAN_HAITIAN_ENTRANT (nativity plus post-1980 arrival) - keep their statutory tests; other documented non-citizens stay LEGAL_PERMANENT_RESIDENT, the modal - true status.' + people, who then receive NON_CITIZEN_VALID_EAD SSN cards (both programs confer employment authorization). + Pew''s 2025 report supersedes its earlier series and estimates a coherent 2023 pair of 14.0M unauthorized + immigrants and 9.7M in the labor force. Its modeled universe is broader than the engine''s UNDOCUMENTED + enum: it retains DACA recipients, parolees, TPS holders, similar temporary protections, and the residual-EAD + subset of the engine''s broad CUBAN_HAITIAN_ENTRANT class, while excluding refugees, people already + granted asylum, and assumed-documented Cuban/Haitian entrants. The worker EAD spill binds that modeled + universe to 9.7M using actual ASEC labor-force status (A_LFSR codes 1-4 at ages 16+) restored from + the same pinned official ASEC archives by exact PERIDNUM join, never prior-year WSAL_VAL/SEMP_VAL + earnings. The student spill binds the corresponding broad residual universe to the Higher Ed Immigration + Portal''s ~408k students. The total modeled Pew-defined unauthorized population is emergent and gated + against the 14.0M 2023 anchor with a coarse band; per-category humanitarian masses are gated against + their cited targets with a coarse band (the ASEC undercovers 2022-24 arrivals — the Ukraine parole + pool saturates below its admin count); reconciling levels is the calibration lane''s job. DACA (arrival + cohort among EAD holders) and CUBAN_HAITIAN_ENTRANT (nativity plus post-1980 arrival) keep their statutory + tests; other documented non-citizens stay LEGAL_PERMANENT_RESIDENT, the modal true status.' - stage: hours_worked survey: Census CPS ASEC source: https://www.census.gov/programs-surveys/cps.html diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py index 1b05bf8b20..df16b8c596 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py @@ -210,8 +210,12 @@ from microcosm.build.us_runtime.education_assistance_source import ( ASEC_EDUCATION_ASSISTANCE_ARCHIVES, ASEC_EDUCATION_ASSISTANCE_INCOME_YEARS, + ASEC_EDUCATION_ASSISTANCE_SOURCE_COLUMNS, + ASEC_LABOR_FORCE_STATUS_COLUMN, + ASEC_LABOR_FORCE_STATUS_VALID_CODES, fetch_asec_education_assistance_source, fill_asec_education_assistance_source, + fill_asec_labor_force_status_source, load_asec_education_assistance_sources, ) from microcosm.build.us_runtime.education_inputs import ( @@ -1474,8 +1478,12 @@ "US_EDUCATION_INPUTS_OWNED_OUTPUT_COLUMNS", "ASEC_EDUCATION_ASSISTANCE_ARCHIVES", "ASEC_EDUCATION_ASSISTANCE_INCOME_YEARS", + "ASEC_EDUCATION_ASSISTANCE_SOURCE_COLUMNS", + "ASEC_LABOR_FORCE_STATUS_COLUMN", + "ASEC_LABOR_FORCE_STATUS_VALID_CODES", "fetch_asec_education_assistance_source", "fill_asec_education_assistance_source", + "fill_asec_labor_force_status_source", "load_asec_education_assistance_sources", "ASEC_PUBLIC_ASSISTANCE_TYPE_AUDIT_PINS", "ASEC_PUBLIC_ASSISTANCE_TYPE_INCOME_YEARS", @@ -2180,14 +2188,16 @@ def to_manifest(self) -> dict[str, object]: US_IMMIGRATION_STAGE_NAME: DonorSpec( survey="CPS ASEC + published unauthorized-population estimates", source=( - "https://www.pewresearch.org/short-reads/2024/07/22/" - "what-we-know-about-unauthorized-immigrants-living-in-the-us/" + "https://www.pewresearch.org/race-and-ethnicity/2025/08/21/" + "u-s-unauthorized-immigrant-population-reached-a-record-" + "14-million-in-2023/" ), notes=( "SSN card type and immigration status from ASEC citizenship, " - "entry-year, nativity, and program-participation fields via the " - "ASEC-UA residual method (SSRN 4662801), targeted to published " - "undocumented population/worker/student control totals." + "entry-year, nativity, labor-force, and program-participation " + "fields via the ASEC-UA residual method (SSRN 4662801), with " + "Pew's broad 2023 unauthorized population and labor-force " + "definitions applied consistently." ), ), US_HOURS_WORKED_STAGE_NAME: DonorSpec( diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_checkpoint.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_checkpoint.py index 0c6ebe0fa2..63bc85593c 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_checkpoint.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_checkpoint.py @@ -57,10 +57,10 @@ ASEC_RAW_STAGE_ARTIFACT_KIND = "populace_us_asec_raw_stage" ASEC_RAW_STAGE_CHECKPOINT_FILENAME = "asec_raw_stage.checkpoint.h5" ASEC_RAW_STAGE_OPERATOR_STATUS = "operator_untouched" -# Version 3 added the PAW_TYP restoration that gates TANF enrollment; older -# artifacts lack the gate column and must fail loudly rather than let -# PAW_VAL-only conflation back in (microcosm#591). -ASEC_RAW_STAGE_SCHEMA_VERSION = 3 +# Version 4 added measured A_LFSR for the Pew civilian-labor-force control; +# older artifacts lack the current labor-force status and must fail loudly +# rather than substitute prior-year earnings (microcosm#767). +ASEC_RAW_STAGE_SCHEMA_VERSION = 4 ASEC_RAW_STAGE_STAGE = "raw_source_mapping" _RAW_STAGE_BINDING_KEYS = frozenset( { @@ -75,9 +75,9 @@ "stage", } ) -_RAW_SOURCE_MAPPING_COLUMNS = frozenset({"ED_VAL", "LKWEEKS", "PAW_TYP"}) +_RAW_SOURCE_MAPPING_COLUMNS = frozenset({"A_LFSR", "ED_VAL", "LKWEEKS", "PAW_TYP"}) _RAW_STAGE_REQUIRED_PERSON_COLUMNS = frozenset( - {"ED_VAL", "LKWEEKS", "PAW_TYP", "PERIDNUM", "source_year"} + {"A_LFSR", "ED_VAL", "LKWEEKS", "PAW_TYP", "PERIDNUM", "source_year"} ) _RAW_SOURCE_MAPPING_KEYS = frozenset( { @@ -458,6 +458,19 @@ def _validate_raw_stage_source_columns(frame: Frame, *, path: Path) -> None: "in {0, 1, 2, 3}." ) + labor_force_status = pd.to_numeric(person["A_LFSR"], errors="coerce").to_numpy( + dtype=np.float64 + ) + valid_labor_force_status = np.isfinite(labor_force_status) & np.isin( + labor_force_status, + (0.0, 1.0, 2.0, 3.0, 4.0, 7.0), + ) + if not valid_labor_force_status.all(): + raise ValueError( + f"ASEC raw-stage checkpoint {path} A_LFSR must be complete integers " + "in {0, 1, 2, 3, 4, 7}." + ) + def _validate_asec_frame( frame: Frame, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/education_assistance_source.py b/packages/microcosm-build/src/microcosm/build/us_runtime/education_assistance_source.py index c24a819c76..5eccefa41e 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/education_assistance_source.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/education_assistance_source.py @@ -1,4 +1,4 @@ -"""Measured CPS ASEC educational assistance for the education-inputs stage. +"""Pinned CPS ASEC person sidecar for measured ED_VAL and A_LFSR fields. The retired eCPS build carried person-level educational assistance directly from ASEC ``ED_VAL`` (archived ``datasets/cps/cps.py`` line 1493: @@ -6,8 +6,11 @@ recode, filter, or universe restriction), and the archived ``datasets/cps/census_cps.py`` listed ``ED_VAL`` among the raw ASEC columns it kept. The frozen ``census_cps_*.h5`` inputs microcosm builds from never -carried that column, so this module restores it from the official, immutable -Census ASEC public-use archives — the same repair class as +carried that column. Those frozen inputs also omitted current civilian labor- +force status ``A_LFSR``, which the immigration stage needs to reproduce Pew's +age-16+ working-or-looking-for-work definition without a prior-year earnings +proxy. This module restores both fields from the official, immutable Census +ASEC public-use archives — the same repair class as :mod:`.weeks_unemployed`'s ``LKWEEKS`` sidecar, extended to every pooled income year. @@ -36,11 +39,14 @@ "ASEC_EDUCATION_ASSISTANCE_ARCHIVES", "ASEC_EDUCATION_ASSISTANCE_INCOME_YEARS", "ASEC_EDUCATION_ASSISTANCE_SOURCE_COLUMNS", + "ASEC_LABOR_FORCE_STATUS_COLUMN", + "ASEC_LABOR_FORCE_STATUS_VALID_CODES", "EDUCATION_ASSISTANCE_ARCHIVED_DERIVATION_URL", "EDUCATION_ASSISTANCE_ARCHIVED_SOURCE_URL", "AsecEducationArchive", "fetch_asec_education_assistance_source", "fill_asec_education_assistance_source", + "fill_asec_labor_force_status_source", "load_asec_education_assistance_sources", ] @@ -59,12 +65,15 @@ ) _SOURCE = "ED_VAL" +ASEC_LABOR_FORCE_STATUS_COLUMN = "A_LFSR" +ASEC_LABOR_FORCE_STATUS_VALID_CODES = frozenset({0, 1, 2, 3, 4, 7}) ASEC_EDUCATION_ASSISTANCE_SOURCE_COLUMNS: tuple[str, ...] = ( "PH_SEQ", "P_SEQ", "A_LINENO", "PERIDNUM", _SOURCE, + ASEC_LABOR_FORCE_STATUS_COLUMN, ) _AUDIT_WEIGHT_COLUMN = "A_FNLWGT" @@ -458,6 +467,19 @@ def load_asec_education_assistance_sources( f"ASEC {pins.survey_year} ED_VAL must be finite and nonnegative " f"at row(s): {rows}." ) + labor_force_status = pd.to_numeric( + raw[ASEC_LABOR_FORCE_STATUS_COLUMN], errors="coerce" + ).to_numpy(dtype=np.float64) + valid_labor_force_status = np.isfinite(labor_force_status) & np.isin( + labor_force_status, + sorted(ASEC_LABOR_FORCE_STATUS_VALID_CODES), + ) + if not valid_labor_force_status.all(): + rows = np.flatnonzero(~valid_labor_force_status)[:5].tolist() + raise ValueError( + f"ASEC {pins.survey_year} A_LFSR must be a complete integer in " + f"{sorted(ASEC_LABOR_FORCE_STATUS_VALID_CODES)} at row(s): {rows}." + ) weights = pd.to_numeric(raw[_AUDIT_WEIGHT_COLUMN], errors="coerce").to_numpy( dtype=np.float64 ) @@ -509,6 +531,7 @@ def load_asec_education_assistance_sources( ) audits[income_year] = audit part = raw.loc[:, list(ASEC_EDUCATION_ASSISTANCE_SOURCE_COLUMNS)].copy() + part[ASEC_LABOR_FORCE_STATUS_COLUMN] = labor_force_status.astype("int64") part.insert(0, "source_year", np.int64(income_year)) parts.append(part) result = pd.concat(parts, ignore_index=True) @@ -620,3 +643,117 @@ def fill_asec_education_assistance_source( "source_audit", {} ) return result + + +def fill_asec_labor_force_status_source( + person: pd.DataFrame, + source: pd.DataFrame, +) -> pd.DataFrame: + """Fill measured ``A_LFSR`` via the pinned exact Census identity join.""" + + required_person = ("source_year", "PERIDNUM") + missing_person = [column for column in required_person if column not in person] + if missing_person: + raise ValueError( + "ASEC labor-force-status repair requires person column(s): " + f"{missing_person}." + ) + required_source = ("source_year", *ASEC_EDUCATION_ASSISTANCE_SOURCE_COLUMNS) + missing_source = [column for column in required_source if column not in source] + if missing_source: + raise ValueError( + f"ASEC labor-force-status sidecar missing column(s): {missing_source}." + ) + result = person.copy(deep=True) + person_years = pd.to_numeric(result["source_year"], errors="coerce") + if person_years.isna().any() or (person_years != np.floor(person_years)).any(): + raise ValueError("ASEC labor-force-status person source_year is invalid.") + needed_years = sorted(int(year) for year in person_years.unique()) + covered = set( + pd.to_numeric(source["source_year"], errors="coerce").astype(int).unique() + ) + uncovered = [year for year in needed_years if year not in covered] + if uncovered: + raise ValueError( + "ASEC labor-force-status sidecar does not cover pooled income " + f"year(s): {uncovered}." + ) + + donor = source.copy(deep=True) + donor["PERIDNUM"] = _fixed_width_peridnum( + donor["PERIDNUM"], label="ASEC labor-force-status sidecar" + ) + if ASEC_LABOR_FORCE_STATUS_COLUMN in result.columns: + existing = pd.to_numeric( + result[ASEC_LABOR_FORCE_STATUS_COLUMN], errors="coerce" + ) + if existing.notna().any(): + raise ValueError( + "ASEC labor-force-status fill found a preexisting A_LFSR " + "column with values; refusing to overwrite measured data." + ) + filled = np.zeros(len(result), dtype=np.int64) + + for year in needed_years: + year_mask = person_years.eq(year).to_numpy() + year_donor = donor.loc[ + pd.to_numeric(donor["source_year"], errors="coerce").eq(year) + ].set_index("PERIDNUM") + keys = _fixed_width_peridnum( + result.loc[year_mask, "PERIDNUM"], + label="ASEC labor-force-status frame", + ) + missing_keys = keys[~keys.isin(year_donor.index)].drop_duplicates() + if not missing_keys.empty: + raise ValueError( + "ASEC labor-force-status sidecar does not cover frame " + f"PERIDNUM key(s) for income year {year}: " + f"{missing_keys.tolist()[:5]}." + ) + aligned = year_donor.reindex(keys.to_numpy()) + aligned.index = result.index[year_mask] + identity_pairs = [ + ( + "PH_SEQ", + "source_household_id" if "source_household_id" in result else "PH_SEQ", + ), + ("P_SEQ", "P_SEQ"), + ("A_LINENO", "A_LINENO"), + ] + for donor_column, frame_column in identity_pairs: + if frame_column not in result: + continue + observed = pd.to_numeric( + result.loc[year_mask, frame_column], errors="coerce" + ).to_numpy(dtype=np.float64) + expected = pd.to_numeric(aligned[donor_column], errors="coerce").to_numpy( + dtype=np.float64 + ) + mismatch = ( + ~np.isfinite(observed) | ~np.isfinite(expected) | (observed != expected) + ) + if mismatch.any(): + rows = result.index[year_mask].to_numpy()[mismatch][:5].tolist() + raise ValueError( + "ASEC labor-force-status redundant identity mismatch for " + f"{frame_column} against sidecar {donor_column} in income " + f"year {year} at row(s): {rows}." + ) + values = pd.to_numeric( + aligned[ASEC_LABOR_FORCE_STATUS_COLUMN], errors="coerce" + ).to_numpy(dtype=np.float64) + valid = np.isfinite(values) & np.isin( + values, + sorted(ASEC_LABOR_FORCE_STATUS_VALID_CODES), + ) + if not valid.all(): + raise ValueError( + f"ASEC labor-force-status sidecar A_LFSR is invalid for " + f"income year {year}." + ) + filled[year_mask] = values.astype(np.int64) + result[ASEC_LABOR_FORCE_STATUS_COLUMN] = filled + result.attrs["labor_force_status_source_audit"] = source.attrs.get( + "source_audit", {} + ) + return result diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py b/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py index 3ad55226d2..0bceea92c0 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py @@ -12,7 +12,8 @@ ``OTHER_NON_CITIZEN`` / ``NONE`` (likely undocumented, i.e. ITIN-filer territory for tax purposes). - ``immigration_status_str``: ``CITIZEN`` / ``LEGAL_PERMANENT_RESIDENT`` / - ``CUBAN_HAITIAN_ENTRANT`` / ``DACA`` / ``UNDOCUMENTED``. + ``CUBAN_HAITIAN_ENTRANT`` / ``DACA`` / ``UNDOCUMENTED`` plus the supported + humanitarian and temporary-protection statuses listed below. Method — survey measurement first, published residual method second, and one narrowly-scoped control where the survey carries no signal at all: @@ -28,19 +29,20 @@ Residual Methods Robust?", SSRN 4662801). 3. **Work/study authorization split.** The CPS carries no work-authorization variable, so among residual non-citizens the split between EAD holders and - the unauthorized is unidentified from the survey alone. Workers and - students spill to ``NON_CITIZEN_VALID_EAD`` — in deterministic seeded - order — until the *remaining* undocumented worker/student counts match - their published control totals (Pew Research Center; Higher Ed - Immigration Portal). This is the only forced margin. -4. **The total undocumented population is emergent, not forced.** + the unauthorized is unidentified from the survey alone. Workers (measured + with ASEC ``A_LFSR`` at age 16+, not prior-year earnings) and students + spill to ``NON_CITIZEN_VALID_EAD`` in deterministic seeded order until the + broad reported universes match their controls. Pew retains DACA, parole, + TPS, and similar temporary protections in its unauthorized estimates, so + those statuses continue to count after receiving EADs. Residual + Cuban/Haitian rows that receive an EAD also stay in that universe because + the engine's broader ``CUBAN_HAITIAN_ENTRANT`` label carries the modeled + CHNV-parole/Haiti-TPS subset. +4. **The total Pew-defined unauthorized population is emergent, not forced.** Representation is calibration's job, not the label stage's: the release - gate checks the emergent total against a cited published anchor with a - coarse plausibility band, and a follow-up calibration target (Ledger - facts lane) can reconcile the level in the weights. On the 2024 ASEC the - emergent total is ≈13M — inside the range of published 2023–24 estimates - — with 26.6M non-citizens, 18.0M residual after the indicators, and 4.8M - spilled to EAD. + gate checks the emergent broad-universe total against Pew's cited 2023 + anchor with a coarse plausibility band. The engine's narrower + ``UNDOCUMENTED`` enum remains exactly paired with ``ssn_card_type=NONE``. 5. **Humanitarian/temporary-protection statuses are drawn to published stocks, never blanketed (microcosm #767).** Blanket ``REFUGEE``/``TPS`` labels for recent arrivals were originally refused because LPR treatment @@ -177,8 +179,7 @@ class is the better statutory label and keeps H.R.1 eligibility), as is "A_MARITL", "A_SPOUSE", "A_HSCOL", - "WSAL_VAL", - "SEMP_VAL", + "A_LFSR", "MCARE", "CAID", "IHSFLG", @@ -277,6 +278,33 @@ class is the better statutory label and keeps H.R.1 eligibility), as is "tps": "TPS", } +# Pew's residual estimates use a deliberately broader universe than the +# engine's ``UNDOCUMENTED`` enum. In particular, Pew retains DACA recipients +# and people with temporary protection (including parole and TPS) in its +# unauthorized-immigrant totals. Refugees and people already granted asylum +# are lawfully admitted statuses and therefore stay outside this universe. +# ``CUBAN_HAITIAN_ENTRANT`` is conditional: only EAD rows that came from the +# residual pool count, not assumed-documented entrants sharing that broad +# engine label. +_PEW_UNAUTHORIZED_STATUS_VALUES: tuple[str, ...] = ( + "UNDOCUMENTED", + "DACA", + "PAROLED_ONE_YEAR", + "DEPORTATION_WITHHELD", + "TPS", +) +_PEW_INCLUDED_HUMANITARIAN_STATUS_VALUES: tuple[str, ...] = ( + "PAROLED_ONE_YEAR", + "DEPORTATION_WITHHELD", + "TPS", +) + +# CPS ASEC A_LFSR: 1 working, 2 with a job/not at work, 3 unemployed/looking, +# 4 unemployed/on layoff. Code 0 covers children and Armed Forces; 7 is not +# in the labor force. Pew's labor-force total is for people age 16 or older. +_CPS_LABOR_FORCE_STATUS_CODES = (1, 2, 3, 4) +_CPS_LABOR_FORCE_STATUS_DOMAIN = (0, 1, 2, 3, 4, 7) + #: PEINUSYR codes for 2020+ arrivals. The 2024 ASEC distinguishes 2020-2021 #: (27) from 2022-2024 (28). Program-specific rules below do not treat those #: windows as interchangeable: U4U and CHNV cannot draw from code 27, while @@ -407,10 +435,11 @@ class is the better statutory label and keeps H.R.1 eligibility), as is #: in this band (Census counts ~22–27M non-citizens of ~336M residents; a #: share outside it means the imputation collapsed or exploded). _NON_CITIZEN_SHARE_BAND = (0.03, 0.12) -#: Emergent weighted ``NONE`` (undocumented) population relative to the cited -#: published anchor. Coarse by design — a release-blocking backstop against -#: collapse or explosion, not a calibration objective; the level belongs to -#: the calibration lane. +#: Emergent Pew-defined unauthorized population relative to the cited +#: published anchor. This includes DACA and specified temporary protections, +#: not only the engine's ``NONE``/``UNDOCUMENTED`` pair. Coarse by design — a +#: release-blocking backstop against collapse or explosion, not a calibration +#: objective; the level belongs to the calibration lane. _UNDOCUMENTED_ANCHOR_RELATIVE_BAND = (0.5, 1.6) #: Emitted humanitarian mass per category relative to its cited target. #: The draw forces the target when candidates suffice, so the band only @@ -425,11 +454,12 @@ class UndocumentedControls: """Published totals the stage forces or gates against. Attributes: - workers: Undocumented-worker control the EAD spill enforces. - students: Undocumented-student control the EAD spill enforces. - population_anchor: Published total undocumented population the - composition gate checks the *emergent* total against (never - forced in the labels). + workers: Pew unauthorized-immigrant labor-force control the EAD spill + enforces, including DACA and temporary protections Pew retains. + students: Broad undocumented-student control the EAD spill enforces. + population_anchor: Published Pew unauthorized population the + composition gate checks the *emergent broad-universe* total + against (never forced in the labels). sources: Citation URL per key, straight from the manifest. """ @@ -1104,6 +1134,100 @@ def _special_status_masks( return cuban_haitian, daca +def _pew_unauthorized_projection_mask( + *, + ssn_codes: np.ndarray, + humanitarian_marks: np.ndarray, + retained_ead_cohort: np.ndarray, +) -> np.ndarray: + """Project final membership in Pew's broad unauthorized universe. + + This helper runs while EAD assignment is still in progress. Residual + ``NONE`` rows count directly; DACA and residual Cuban/Haitian cohort rows + continue to count if an EAD spill gives them a protected engine label; + and the temporary-protection statuses Pew explicitly retains count + regardless of their non-``NONE`` SSN code. + """ + + return ( + (ssn_codes == 0) + | ((ssn_codes == 2) & retained_ead_cohort) + | np.isin( + humanitarian_marks, + _PEW_INCLUDED_HUMANITARIAN_STATUS_VALUES, + ) + ) + + +def _spill_pew_unauthorized_excess( + person: pd.DataFrame, + ssn_codes: np.ndarray, + humanitarian_marks: np.ndarray, + weights: np.ndarray, + *, + noncitizens: np.ndarray, + scope: np.ndarray, + preserve_scope: np.ndarray, + retained_ead_cohort: np.ndarray, + target: float, + seed: int, + salt: str, +) -> None: + """Spill a broad Pew-universe margin to EAD in deterministic order. + + The first draw preserves the prior EAD/DACA allocation behavior by + selecting from the whole residual scope. A selected DACA or residual + Cuban/Haitian cohort row still belongs to Pew's universe, so a second pass + spills additional rows outside those retained cohorts when needed. Within + each pass, rows outside the other controlled margin are selected first; + this keeps the student and worker controls from disturbing one another + unless their overlap makes that unavoidable. + """ + + draws = _stable_person_draws(person, seed=seed, salt=salt) + + def current_excess() -> float: + included = _pew_unauthorized_projection_mask( + ssn_codes=ssn_codes, + humanitarian_marks=humanitarian_marks, + retained_ead_cohort=retained_ead_cohort, + ) + return float(weights[scope & included].sum()) - target + + # Preserve the existing seeded EAD allocation surface. DACA selections + # do not reduce the Pew count and are compensated by the corrective pass. + initial_candidates = (ssn_codes == 0) & noncitizens & scope + initial_excess = current_excess() + if initial_excess > 0: + for priority in (~preserve_scope, preserve_scope): + selected = _select_weight_to_target( + initial_candidates & priority, + weights, + draws, + current_excess(), + ) + ssn_codes[selected] = 2 + if current_excess() <= 0: + return + + # DACA and residual Cuban/Haitian cohort members retain protected engine + # labels after EAD assignment and therefore remain inside Pew's estimate. + # Only rows outside those cohorts can close any remaining gap. + for priority in (~preserve_scope, preserve_scope): + corrective_candidates = ( + (ssn_codes == 0) & noncitizens & scope & ~retained_ead_cohort & priority + ) + selected = _select_weight_to_target( + corrective_candidates, + weights, + draws, + current_excess(), + ) + ssn_codes[selected] = 2 + if current_excess() <= 0: + return + + def _humanitarian_draw_candidates( draw: HumanitarianDraw, *, @@ -1685,8 +1809,8 @@ def _assign_humanitarian_statuses( """Mark humanitarian statuses and upgrade residual draws' SSN codes. Runs after the legal-status indicators and before the EAD worker/student - spill, so the Pew worker/student controls still bind the remaining - residual pool exactly. Draws are sequential over + spill. Pew-included temporary protections remain inside the broad control + universe even after receiving EADs. Draws are sequential over ``controls.humanitarian``; a person marked by an earlier draw is out of every later candidate pool. Residual-pool selections (parole/TPS only) move to ``NON_CITIZEN_VALID_EAD`` — both programs confer employment @@ -1777,9 +1901,9 @@ def _assign_ssn_card_codes( ) ssn_codes[(ssn_codes == 0) & assumed_documented] = 3 - # Humanitarian draws run between the indicators and the EAD spill: the - # spill controls then bind the *remaining* residual pool, so the final - # undocumented worker/student counts still land on the Pew anchors. + # Humanitarian draws run between the indicators and the EAD spill. Pew + # retains temporary protection in its unauthorized estimate, so those + # rows contribute to the broad controls even though they receive EADs. humanitarian_marks = _assign_humanitarian_statuses( person, ssn_codes, @@ -1789,43 +1913,65 @@ def _assign_ssn_card_codes( time_period=time_period, ) - is_worker = (_float_column(person, "WSAL_VAL") > 0) | ( - _float_column(person, "SEMP_VAL") > 0 + labor_force_status = _integer_column(person, "A_LFSR") + invalid_labor_force_status = ~np.isin( + labor_force_status, + _CPS_LABOR_FORCE_STATUS_DOMAIN, + ) + if invalid_labor_force_status.any(): + bad = sorted(set(labor_force_status[invalid_labor_force_status].tolist()))[:5] + raise SourceRuntimeError( + "A_LFSR carries value(s) outside the CPS ASEC domain " + f"{list(_CPS_LABOR_FORCE_STATUS_DOMAIN)}: {bad}." + ) + is_worker = (age >= 16) & np.isin( + labor_force_status, + _CPS_LABOR_FORCE_STATUS_CODES, ) is_student = _integer_column(person, "A_HSCOL") == 2 - # The CPS has no work-authorization variable, so the EAD-vs-unauthorized - # split inside the residual pool is unidentified from the survey. Spill - # excess undocumented workers/students to EAD so the *remaining* counts - # match their published controls; counts already at or below a control - # spill nothing. The total undocumented population is emergent — the - # composition gate checks it against a cited anchor, and reconciling the - # level is the calibration lane's job, not this label stage's. - worker_candidates = (ssn_codes == 0) & noncitizens & is_worker - worker_excess = ( - float(weights[worker_candidates].sum()) - controls.undocumented.workers - ) - worker_draws = _stable_person_draws( - person, seed=seed, salt="immigration:ead_workers" + _, arrival_year, age_at_entry = _arrival_profile(person, time_period=time_period) + daca_cohort = _daca_statutory_cohort(arrival_year, age_at_entry, age) + cuban_haitian_cohort = ( + np.isin(_integer_column(person, "PENATVTY"), _CUBAN_HAITIAN_BIRTH_CODES) + & (arrival_year >= _CUBAN_HAITIAN_ARRIVAL_CUTOFF) + & ~daca_cohort ) - ssn_codes[ - _select_weight_to_target( - worker_candidates, weights, worker_draws, worker_excess - ) - ] = 2 - - student_candidates = (ssn_codes == 0) & noncitizens & is_student - student_excess = ( - float(weights[student_candidates].sum()) - controls.undocumented.students + retained_ead_cohort = daca_cohort | cuban_haitian_cohort + + # The CPS has no work-authorization variable, so the EAD split inside the + # residual pool is unidentified. The two controls bind the same broad + # unauthorized universe their publishers report: residual undocumented + # people plus DACA and Pew-included temporary protections. Student spill + # runs first; the worker spill runs last and is therefore authoritative + # for Pew's published labor-force total. Each pass prefers rows outside + # the other margin so their overlap is disturbed only when unavoidable. + _spill_pew_unauthorized_excess( + person, + ssn_codes, + humanitarian_marks, + weights, + noncitizens=noncitizens, + scope=is_student, + preserve_scope=is_worker, + retained_ead_cohort=retained_ead_cohort, + target=controls.undocumented.students, + seed=seed, + salt="immigration:ead_students", ) - student_draws = _stable_person_draws( - person, seed=seed, salt="immigration:ead_students" + _spill_pew_unauthorized_excess( + person, + ssn_codes, + humanitarian_marks, + weights, + noncitizens=noncitizens, + scope=is_worker, + preserve_scope=is_student, + retained_ead_cohort=retained_ead_cohort, + target=controls.undocumented.workers, + seed=seed, + salt="immigration:ead_workers", ) - ssn_codes[ - _select_weight_to_target( - student_candidates, weights, student_draws, student_excess - ) - ] = 2 return ssn_codes, humanitarian_marks @@ -1994,8 +2140,8 @@ def us_immigration_composition_gate( mode: everyone a citizen with a valid SSN), when a value falls outside the engine enum domain, when the two columns disagree about citizenship or undocumented status, when the weighted non-citizen share leaves its - plausibility band, when the emergent undocumented population strays - outside a coarse band around its cited published anchor, or when a + plausibility band, when the emergent Pew-defined unauthorized population + strays outside a coarse band around its cited published anchor, or when a humanitarian category's emitted mass leaves the coarse band around its cited stock target (microcosm #767 — the H.R.1 §71109/§71301/§71302 and SNAP §10108 eligibility channels all bind through these categories; an @@ -2119,19 +2265,34 @@ def us_immigration_composition_gate( f"[{low}, {high}]." ) - undocumented = float(weights[ssn_none].sum()) - relative = undocumented / controls.undocumented.population_anchor + status_values = status.to_numpy() + ssn_values = ssn.to_numpy() + pew_unauthorized_rows = np.isin( + status_values, + _PEW_UNAUTHORIZED_STATUS_VALUES, + ) | ( + (status_values == "CUBAN_HAITIAN_ENTRANT") + & (ssn_values == "NON_CITIZEN_VALID_EAD") + ) + pew_unauthorized = float(weights[pew_unauthorized_rows].sum()) + details["pew_unauthorized_population"] = pew_unauthorized + details["pew_unauthorized_statuses"] = list(_PEW_UNAUTHORIZED_STATUS_VALUES) + details["pew_unauthorized_paired_status"] = { + "immigration_status_str": "CUBAN_HAITIAN_ENTRANT", + "ssn_card_type": "NON_CITIZEN_VALID_EAD", + } + relative = pew_unauthorized / controls.undocumented.population_anchor rel_low, rel_high = _UNDOCUMENTED_ANCHOR_RELATIVE_BAND if not (rel_low <= relative <= rel_high): failures.append( - f"emergent undocumented population {undocumented:,.0f} is " + f"emergent Pew-defined unauthorized population " + f"{pew_unauthorized:,.0f} is " f"{relative:.2f}x the published anchor " f"{controls.undocumented.population_anchor:,.0f} " f"(band [{rel_low}, {rel_high}], " f"{controls.undocumented.sources[_ANCHOR_KEY]})." ) - status_values = status.to_numpy() hum_low, hum_high = _HUMANITARIAN_TARGET_RELATIVE_BAND humanitarian_statuses = set(_HUMANITARIAN_STATUS_BY_CATEGORY.values()) humanitarian_rows = np.isin(status_values, list(humanitarian_statuses)) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_producer_registry.py b/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_producer_registry.py index 640bec8d46..6cfb5f751e 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_producer_registry.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_producer_registry.py @@ -958,8 +958,7 @@ def _inventory( "A_MARITL", "A_SPOUSE", "A_HSCOL", - "WSAL_VAL", - "SEMP_VAL", + "A_LFSR", "MCARE", "CAID", "IHSFLG", diff --git a/packages/microcosm-build/tests/test_spec_engine_compiler_ir.py b/packages/microcosm-build/tests/test_spec_engine_compiler_ir.py index ad8888ffc8..acfd3be250 100644 --- a/packages/microcosm-build/tests/test_spec_engine_compiler_ir.py +++ b/packages/microcosm-build/tests/test_spec_engine_compiler_ir.py @@ -22,7 +22,7 @@ ) from microcosm.build.spec_engine.resolver import F0_CONTRACT_ONLY_KERNEL_IDS -US_SCHEDULE_SHA256 = "b7e40a89f5100c34b748ce960c209807b98d0f5e2c5834bb4c250aec733a37dc" +US_SCHEDULE_SHA256 = "88bc9243a3518982ae951c3de21bd55877e296ce4fcb183b9bee420d3a684b10" @pytest.fixture(scope="module") diff --git a/packages/microcosm-build/tests/test_spec_engine_coverage_tool.py b/packages/microcosm-build/tests/test_spec_engine_coverage_tool.py index b9fb753c1c..dec9a4c6c8 100644 --- a/packages/microcosm-build/tests/test_spec_engine_coverage_tool.py +++ b/packages/microcosm-build/tests/test_spec_engine_coverage_tool.py @@ -52,21 +52,21 @@ def test_us_coverage_is_exact_complete_and_honest( assert_coverage_complete(coverage_report) assert coverage_report["status"] == "pass" fields = coverage_report["field_usage"] - assert fields["configuration_field_count"] == 42_545 - assert fields["authored_normative_field_count"] == 32_528 + assert fields["configuration_field_count"] == 42_538 + assert fields["authored_normative_field_count"] == 32_521 assert fields["resolved_binding_field_count"] == 10_017 - assert fields["consumed_field_count"] == 42_545 + assert fields["consumed_field_count"] == 42_538 assert fields["unused_field_count"] == 0 assert fields["multiple_primary_use_field_count"] == 0 assert fields["claim_count"] == 49 assert fields["mode_counts"] == { "legacy_behavior": 14_020, - "compiler_semantic": 28_074, + "compiler_semantic": 28_067, "front_end_validation": 348, "identity_only": 103, } assert fields["generation0_effect_counts"] == { - "legacy_behavior": 38_581, + "legacy_behavior": 38_574, "no_generation0_effect": 3_964, } @@ -75,7 +75,7 @@ def test_us_coverage_is_exact_complete_and_honest( assert inventory["covered_item_count"] == 41 assert inventory["missing_item_count"] == 0 assert inventory["missing_items"] == [] - assert inventory["counts"]["producer_inputs"] == 2_751 + assert inventory["counts"]["producer_inputs"] == 2_750 assert inventory["counts"]["ownership_rows"] == 18 assert inventory["counts"]["tail_control_fields"] == 934 assert inventory["counts"]["seed_owner_bindings"] == 125 diff --git a/packages/microcosm-build/tests/test_spec_engine_field_usage.py b/packages/microcosm-build/tests/test_spec_engine_field_usage.py index 3be7fe4aa7..90eb846fbd 100644 --- a/packages/microcosm-build/tests/test_spec_engine_field_usage.py +++ b/packages/microcosm-build/tests/test_spec_engine_field_usage.py @@ -92,22 +92,22 @@ def _mutated_bundle( def test_exact_complete_ledger_has_one_primary_mode_per_pointer(field_ledger) -> None: - assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_545 + assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_538 assert field_ledger.source_counts == { - "authored": 32_528, + "authored": 32_521, "resolved_bindings": 10_017, } assert field_ledger.mode_counts == { "legacy_behavior": 14_020, - "compiler_semantic": 28_074, + "compiler_semantic": 28_067, "front_end_validation": 348, "identity_only": 103, } assert field_ledger.generation0_effect_counts == { - "legacy_behavior": 38_581, + "legacy_behavior": 38_574, "no_generation0_effect": 3_964, } - assert len({field.pointer for field in field_ledger.fields}) == 42_545 + assert len({field.pointer for field in field_ledger.fields}) == 42_538 def test_eligibility_concepts_are_validation_not_generation0_behavior( diff --git a/packages/microcosm-build/tests/test_spec_engine_imputation_semantics.py b/packages/microcosm-build/tests/test_spec_engine_imputation_semantics.py index abb93425ae..28a03e4323 100644 --- a/packages/microcosm-build/tests/test_spec_engine_imputation_semantics.py +++ b/packages/microcosm-build/tests/test_spec_engine_imputation_semantics.py @@ -98,7 +98,7 @@ def test_imputation_projector_matches_live_plans_and_graph_receipts( for key, expected in live.items(): assert canonical_json_bytes(projected[key]) == canonical_json_bytes(expected) assert projected["late_producer_schedule_receipt"]["schedule_sha256"] == ( - "b7e40a89f5100c34b748ce960c209807b98d0f5e2c5834bb4c250aec733a37dc" + "88bc9243a3518982ae951c3de21bd55877e296ce4fcb183b9bee420d3a684b10" ) assert projected["overlap_ownership"]["sha256"] == ( "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356" diff --git a/packages/microcosm-build/tests/test_spec_engine_inventory_coverage.py b/packages/microcosm-build/tests/test_spec_engine_inventory_coverage.py index 5bdeca00d9..3b8a8cc3bf 100644 --- a/packages/microcosm-build/tests/test_spec_engine_inventory_coverage.py +++ b/packages/microcosm-build/tests/test_spec_engine_inventory_coverage.py @@ -79,7 +79,7 @@ "primary_targets": 65, "producer_authored_outputs": 92, "producer_compiled_outputs": 227, - "producer_inputs": 2_751, + "producer_inputs": 2_750, "producer_nodes": 38, "producer_virtual_resources": 75, "release_rungs": 5, diff --git a/packages/microcosm-build/tests/test_spec_engine_legacy_adapter.py b/packages/microcosm-build/tests/test_spec_engine_legacy_adapter.py index 39acdf7fc3..5c72c776e7 100644 --- a/packages/microcosm-build/tests/test_spec_engine_legacy_adapter.py +++ b/packages/microcosm-build/tests/test_spec_engine_legacy_adapter.py @@ -195,10 +195,10 @@ def test_adapter_preserves_generation_zero_identity_components( imputation = legacy_payload["imputation"] assert isinstance(imputation, dict) assert legacy_payload["stacked_authority_receipt"]["sha256"] == ( - "4c8a2d5478a4fef9c77f298f692f1c5f8667045c16c940855845d797023bc409" + "406b2cf93a7fb94dc63f1a24a52cfb20362ab83195eb8ebf61d4ef43072280ec" ) assert imputation["late_producer_schedule_receipt"]["schedule_sha256"] == ( - "b7e40a89f5100c34b748ce960c209807b98d0f5e2c5834bb4c250aec733a37dc" + "88bc9243a3518982ae951c3de21bd55877e296ce4fcb183b9bee420d3a684b10" ) assert imputation["overlap_ownership"]["sha256"] == ( "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356" diff --git a/packages/microcosm-build/tests/test_spec_engine_stacked_authority_semantics.py b/packages/microcosm-build/tests/test_spec_engine_stacked_authority_semantics.py index 78b0a14903..96a870f317 100644 --- a/packages/microcosm-build/tests/test_spec_engine_stacked_authority_semantics.py +++ b/packages/microcosm-build/tests/test_spec_engine_stacked_authority_semantics.py @@ -73,7 +73,7 @@ def test_authority_projection_is_field_and_byte_identical_to_live_generation_zer assert projected == live assert stacked_identity_bytes(projected) == _canonical_bytes(live) assert projected["sha256"] == ( - "4c8a2d5478a4fef9c77f298f692f1c5f8667045c16c940855845d797023bc409" + "406b2cf93a7fb94dc63f1a24a52cfb20362ab83195eb8ebf61d4ef43072280ec" ) assert { name: component["sha256"] for name, component in projected["components"].items() @@ -88,7 +88,7 @@ def test_authority_projection_is_field_and_byte_identical_to_live_generation_zer "cacc6c11e114dbae3aaa2761cc6b3fcb1191cd9b689b1c2bd096614c51ebff8b" ), "late_producer_schedule": ( - "bd0f7d71971e7530f2979550ee919f7e78ab023869db8b3fcc7615d1d01747c7" + "8f49aad23470d7b09b401cc0411f3199657cf9d77b26edfad2a435f271a8643d" ), "metric_registry": ( "d75cb9b29f8b0a9a085471a11f4c19c32ba04cbe5419053df94ea81cbe6125a9" @@ -191,7 +191,7 @@ def test_checkpoint_projection_is_field_and_byte_identical_to_live_oracle( ] assert ( projected["pool_code"]["late_producer_schedule"]["schedule_sha256"] - == "b7e40a89f5100c34b748ce960c209807b98d0f5e2c5834bb4c250aec733a37dc" + == "88bc9243a3518982ae951c3de21bd55877e296ce4fcb183b9bee420d3a684b10" ) diff --git a/packages/microcosm-build/tests/test_us_asec_checkpoint.py b/packages/microcosm-build/tests/test_us_asec_checkpoint.py index aa2a1338a6..e15ff5ec71 100644 --- a/packages/microcosm-build/tests/test_us_asec_checkpoint.py +++ b/packages/microcosm-build/tests/test_us_asec_checkpoint.py @@ -134,7 +134,7 @@ def _raw_binding(frame: Frame) -> dict[str, object]: "operation": "exact_source_join", "source_pins": [pin], } - for column in ("ED_VAL", "LKWEEKS", "PAW_TYP") + for column in ("A_LFSR", "ED_VAL", "LKWEEKS", "PAW_TYP") }, "schema_version": ASEC_RAW_STAGE_SCHEMA_VERSION, "source_construction_identity": frame_identity(frame).to_payload(), @@ -166,6 +166,7 @@ def _raw_us_frame(*, id_offset: int = 0) -> Frame: tables["person"]["ED_VAL"] = [0.0, 500.0] tables["person"]["LKWEEKS"] = [-1, 12] tables["person"]["PAW_TYP"] = np.asarray([0, 1], dtype=np.int64) + tables["person"]["A_LFSR"] = np.asarray([1, 7], dtype=np.int64) return Frame( tables, source.schema, @@ -248,7 +249,7 @@ def test_loads_operator_untouched_raw_stage_checkpoint(tmp_path: Path) -> None: @pytest.mark.parametrize( "column", - ("ED_VAL", "LKWEEKS", "PAW_TYP", "PERIDNUM", "source_year"), + ("A_LFSR", "ED_VAL", "LKWEEKS", "PAW_TYP", "PERIDNUM", "source_year"), ) def test_raw_loader_rejects_missing_input_complete_source_column( tmp_path: Path, @@ -278,6 +279,7 @@ def test_raw_loader_rejects_missing_input_complete_source_column( ("ED_VAL", [0.0, np.nan], "ED_VAL must be complete"), ("LKWEEKS", [-1, 53], "LKWEEKS must be complete"), ("PAW_TYP", [0, 4], "PAW_TYP must be complete integers"), + ("A_LFSR", [1, 6], "A_LFSR must be complete integers"), ), ) def test_raw_loader_rejects_invalid_input_complete_source_values( diff --git a/packages/microcosm-build/tests/test_us_bundle_core_contracts.py b/packages/microcosm-build/tests/test_us_bundle_core_contracts.py index a4a83b7646..d8b677c600 100644 --- a/packages/microcosm-build/tests/test_us_bundle_core_contracts.py +++ b/packages/microcosm-build/tests/test_us_bundle_core_contracts.py @@ -129,7 +129,7 @@ def test_source_surface_classification_is_complete() -> None: } assert normative["stage_asset"] == { "id": "source_stages", - "sha256": "413174820cfb57f85783831dbbf445f007d06128d31562a1234e998e627bd3aa", + "sha256": "788b815f748abb2061c41efb0cec4cc4952435dbcb4448ecf21ccac85a5ccec2", } assert operational["stage_asset"] == { "path": "microcosm.build.us/source_stages.json" diff --git a/packages/microcosm-build/tests/test_us_education_assistance_source.py b/packages/microcosm-build/tests/test_us_education_assistance_source.py index f821ea2ccb..9394a79738 100644 --- a/packages/microcosm-build/tests/test_us_education_assistance_source.py +++ b/packages/microcosm-build/tests/test_us_education_assistance_source.py @@ -1,4 +1,4 @@ -"""The pinned ASEC education-assistance sidecar: load, verify, and fill. +"""The pinned ASEC ED_VAL/A_LFSR sidecar: load, verify, and fill. The frozen census_cps inputs never carried raw ASEC ``ED_VAL`` (microcosm#417's sibling gap), so the education-inputs stage restores it from @@ -21,8 +21,11 @@ from microcosm.build.us_runtime.education_assistance_source import ( ASEC_EDUCATION_ASSISTANCE_ARCHIVES, ASEC_EDUCATION_ASSISTANCE_INCOME_YEARS, + ASEC_LABOR_FORCE_STATUS_COLUMN, + ASEC_LABOR_FORCE_STATUS_VALID_CODES, AsecEducationArchive, fill_asec_education_assistance_source, + fill_asec_labor_force_status_source, load_asec_education_assistance_sources, ) @@ -41,6 +44,7 @@ def _source_frame(rows: int = 6) -> pd.DataFrame: "A_LINENO": np.ones(rows, dtype=np.int64), "PERIDNUM": [_peridnum(index) for index in range(rows)], "ED_VAL": [0.0, 0.0, 2_500.0, 0.0, 12_000.0, 0.0][:rows], + "A_LFSR": [0, 1, 2, 3, 4, 7][:rows], "A_FNLWGT": np.full(rows, 100.0), } ) @@ -112,9 +116,29 @@ def test_loader_reads_pinned_zip_and_audits(pinned_archive) -> None: ) assert list(source["source_year"].unique()) == [_YEAR] assert len(source) == len(frame) + assert source[ASEC_LABOR_FORCE_STATUS_COLUMN].tolist() == [0, 1, 2, 3, 4, 7] + assert source[ASEC_LABOR_FORCE_STATUS_COLUMN].dtype == np.dtype("int64") assert source.attrs["source_audit"][_YEAR]["positive_rows"] == 2 +def test_loader_rejects_invalid_labor_force_status( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + frame = _source_frame() + frame.loc[0, ASEC_LABOR_FORCE_STATUS_COLUMN] = 6 + member = f"pppub{str(_YEAR + 1)[2:]}.csv" + path = tmp_path / "invalid-a-lfsr.zip" + _write_archive(path, frame, member) + monkeypatch.setitem( + ASEC_EDUCATION_ASSISTANCE_ARCHIVES, + _YEAR, + _pins_for(path, frame, member), + ) + + with pytest.raises(ValueError, match="A_LFSR must be a complete integer"): + load_asec_education_assistance_sources({_YEAR: path}, income_years=(_YEAR,)) + + def test_loader_rejects_wrong_zip_bytes(pinned_archive, tmp_path) -> None: path, frame = pinned_archive tampered = tmp_path / "tampered.zip" @@ -155,6 +179,21 @@ def test_fill_joins_ed_val_by_identity(pinned_archive) -> None: assert row["ED_VAL"] == expected[row["PERIDNUM"]] +def test_fill_joins_labor_force_status_by_identity(pinned_archive) -> None: + path, frame = pinned_archive + source = load_asec_education_assistance_sources( + {_YEAR: path}, income_years=(_YEAR,) + ) + filled = fill_asec_labor_force_status_source(_person_frame(), source) + expected = frame.set_index("PERIDNUM")[ASEC_LABOR_FORCE_STATUS_COLUMN] + for _, row in filled.iterrows(): + assert row[ASEC_LABOR_FORCE_STATUS_COLUMN] == expected[row["PERIDNUM"]] + assert filled[ASEC_LABOR_FORCE_STATUS_COLUMN].dtype == np.dtype("int64") + assert set(filled[ASEC_LABOR_FORCE_STATUS_COLUMN]) <= ( + ASEC_LABOR_FORCE_STATUS_VALID_CODES + ) + + def test_fill_fails_closed_on_uncovered_key(pinned_archive) -> None: path, _ = pinned_archive source = load_asec_education_assistance_sources( @@ -199,6 +238,19 @@ def test_fill_refuses_to_overwrite_measured_values(pinned_archive) -> None: fill_asec_education_assistance_source(person, source) +def test_labor_force_fill_refuses_to_overwrite_measured_values( + pinned_archive, +) -> None: + path, _ = pinned_archive + source = load_asec_education_assistance_sources( + {_YEAR: path}, income_years=(_YEAR,) + ) + person = _person_frame() + person[ASEC_LABOR_FORCE_STATUS_COLUMN] = 7 + with pytest.raises(ValueError, match="refusing to overwrite"): + fill_asec_labor_force_status_source(person, source) + + def test_pins_reject_audit_drift(pinned_archive, monkeypatch) -> None: path, frame = pinned_archive pins = ASEC_EDUCATION_ASSISTANCE_ARCHIVES[_YEAR] diff --git a/packages/microcosm-build/tests/test_us_immigration.py b/packages/microcosm-build/tests/test_us_immigration.py index 18618f7d7c..259f1ddaf5 100644 --- a/packages/microcosm-build/tests/test_us_immigration.py +++ b/packages/microcosm-build/tests/test_us_immigration.py @@ -139,6 +139,7 @@ def _person_table(rows: list[dict]) -> pd.DataFrame: "A_MARITL": 7, "A_SPOUSE": 0, "A_HSCOL": 0, + "A_LFSR": 7, "WSAL_VAL": 0.0, "SEMP_VAL": 0.0, "MCARE": 2, @@ -243,13 +244,15 @@ def test_noncitizen_without_indicators_is_undocumented(self) -> None: assert output.loc[0, "ssn_card_type"] == "NONE" assert output.loc[0, "immigration_status_str"] == "UNDOCUMENTED" - def test_worker_spill_leaves_undocumented_workers_at_control(self) -> None: + def test_worker_spill_leaves_pew_unauthorized_labor_force_at_control( + self, + ) -> None: person = _person_table( - [_noncitizen(WSAL_VAL=10_000.0) for _ in range(10)] + [_noncitizen(A_LFSR=1, WSAL_VAL=10_000.0) for _ in range(10)] + [_noncitizen(), _noncitizen()] ) output = _run(person, workers=4.0) - workers = output["WSAL_VAL"] > 0 + workers = output["A_LFSR"].isin([1, 2, 3, 4]) undocumented_workers = ((output["ssn_card_type"] == "NONE") & workers).sum() ead_workers = ( (output["ssn_card_type"] == "NON_CITIZEN_VALID_EAD") & workers @@ -259,8 +262,89 @@ def test_worker_spill_leaves_undocumented_workers_at_control(self) -> None: # Non-workers are untouched by the worker spill. assert (output.loc[~workers, "ssn_card_type"] == "NONE").all() + def test_worker_spill_uses_labor_force_status_not_prior_year_earnings( + self, + ) -> None: + person = _person_table( + [ + # Looking for work now, despite having no prior-year earnings. + _noncitizen(A_LFSR=3), + # Earned wages last year, but is now outside the labor force. + _noncitizen(A_LFSR=7, WSAL_VAL=10_000.0), + ] + ) + output = _run(person, workers=0.001) + assert output.loc[0, "ssn_card_type"] == "NON_CITIZEN_VALID_EAD" + assert output.loc[1, "ssn_card_type"] == "NONE" + + def test_worker_control_retains_residual_cuban_haitian_ead_rows(self) -> None: + person = _person_table( + [ + _noncitizen(A_LFSR=1, PENATVTY=327, PEINUSYR=24), + _noncitizen(A_LFSR=1, PENATVTY=332, PEINUSYR=24), + *[_noncitizen(A_LFSR=1) for _ in range(8)], + ] + ) + output = _run(person, workers=4.0) + labor_force = output["A_LFSR"].isin([1, 2, 3, 4]) + pew_unauthorized = output["immigration_status_str"].isin( + [ + "UNDOCUMENTED", + "DACA", + "PAROLED_ONE_YEAR", + "DEPORTATION_WITHHELD", + "TPS", + ] + ) | ( + output["immigration_status_str"].eq("CUBAN_HAITIAN_ENTRANT") + & output["ssn_card_type"].eq("NON_CITIZEN_VALID_EAD") + ) + assert int((labor_force & pew_unauthorized).sum()) == 4 + + def test_worker_control_uses_pew_age_16_labor_force_universe(self) -> None: + person = _person_table( + [ + _noncitizen(A_AGE=15, A_LFSR=1), + _noncitizen(A_AGE=16, A_LFSR=3), + ] + ) + output = _run(person, workers=0.001) + assert output.loc[0, "ssn_card_type"] == "NONE" + assert output.loc[1, "ssn_card_type"] == "NON_CITIZEN_VALID_EAD" + + def test_invalid_labor_force_status_is_refused(self) -> None: + with pytest.raises(SourceRuntimeError, match="A_LFSR"): + _run(_person_table([_noncitizen(A_LFSR=6)])) + + def test_worker_control_counts_tps_before_spilling_residual_workers( + self, + ) -> None: + person = _person_table( + [ + _noncitizen( + PENATVTY=373, + PEINUSYR=24, + A_LFSR=1, + ), + _noncitizen(A_LFSR=1), + ] + ) + output = _run( + person, + workers=1.0, + humanitarian=_humanitarian_block(tps={"venezuela": 1.0}), + ) + assert output.loc[0, "immigration_status_str"] == "TPS" + # Pew includes TPS holders in its unauthorized estimate. The TPS row + # therefore exhausts the one-person labor-force control, so the other + # residual worker must spill to EAD rather than remain undocumented. + assert output.loc[1, "ssn_card_type"] == "NON_CITIZEN_VALID_EAD" + assert output.loc[1, "immigration_status_str"] == ("LEGAL_PERMANENT_RESIDENT") + def test_below_control_counts_spill_nothing(self) -> None: - person = _person_table([_noncitizen(WSAL_VAL=10_000.0), _noncitizen(A_HSCOL=2)]) + person = _person_table( + [_noncitizen(A_LFSR=1, WSAL_VAL=10_000.0), _noncitizen(A_HSCOL=2)] + ) output = _run(person, workers=50.0, students=50.0) assert (output["ssn_card_type"] == "NONE").all() @@ -273,16 +357,16 @@ def test_student_spill_leaves_undocumented_students_at_control(self) -> None: def test_weights_drive_the_spill_amounts(self) -> None: person = _person_table( [ - _noncitizen(WSAL_VAL=10_000.0, person_weight=6.0), - _noncitizen(WSAL_VAL=10_000.0, person_weight=6.0), + _noncitizen(A_LFSR=1, WSAL_VAL=10_000.0, person_weight=6.0), + _noncitizen(A_LFSR=1, WSAL_VAL=10_000.0, person_weight=6.0), ] ) output = _run(person, workers=6.0) assert set(output["ssn_card_type"]) == {"NON_CITIZEN_VALID_EAD", "NONE"} def test_indicator_holders_never_flip_to_undocumented(self) -> None: - # The total undocumented population is emergent: a short count is - # never topped up from people with legal-status indicators. + # The total Pew-defined unauthorized population is emergent: a short + # count is never topped up from people with legal-status indicators. person = _person_table( [_noncitizen(), _noncitizen(CAID=1, person_household_id=1)] ) @@ -312,14 +396,18 @@ class TestImmigrationStatusTags: def test_daca_statutory_cohort_among_ead_holders(self) -> None: # Arrived 2005 (code 19) aged 10 → age at entry < 16, now 29, EAD via # worker spill with a zero control. - person = _person_table([_noncitizen(PEINUSYR=19, A_AGE=29, WSAL_VAL=20_000.0)]) + person = _person_table( + [_noncitizen(PEINUSYR=19, A_AGE=29, A_LFSR=1, WSAL_VAL=20_000.0)] + ) output = _run(person, workers=0.001) assert output.loc[0, "ssn_card_type"] == "NON_CITIZEN_VALID_EAD" assert output.loc[0, "immigration_status_str"] == "DACA" def test_ead_outside_daca_cohort_is_lpr(self) -> None: # Arrived 2015 as an adult: fails the DACA arrival test. - person = _person_table([_noncitizen(PEINUSYR=24, A_AGE=40, WSAL_VAL=20_000.0)]) + person = _person_table( + [_noncitizen(PEINUSYR=24, A_AGE=40, A_LFSR=1, WSAL_VAL=20_000.0)] + ) output = _run(person, workers=0.001) assert output.loc[0, "ssn_card_type"] == "NON_CITIZEN_VALID_EAD" assert output.loc[0, "immigration_status_str"] == "LEGAL_PERMANENT_RESIDENT" @@ -346,7 +434,7 @@ def test_undocumented_tag_matches_none_ssn_exactly(self) -> None: [ _noncitizen(), _noncitizen(CAID=1), - _noncitizen(WSAL_VAL=10_000.0), + _noncitizen(A_LFSR=1, WSAL_VAL=10_000.0), {"PRCITSHP": 1}, ] ) @@ -362,10 +450,15 @@ def test_emitted_values_stay_inside_engine_enum_domains(self) -> None: for overrides in ( {}, {"CAID": 1}, - {"WSAL_VAL": 10_000.0}, + {"A_LFSR": 1, "WSAL_VAL": 10_000.0}, {"A_HSCOL": 2}, {"PENATVTY": 327}, - {"PEINUSYR": 19, "A_AGE": 25, "WSAL_VAL": 5_000.0}, + { + "PEINUSYR": 19, + "A_AGE": 25, + "A_LFSR": 1, + "WSAL_VAL": 5_000.0, + }, ) ] + [{"PRCITSHP": 1}] @@ -725,7 +818,15 @@ def test_daca_cohort_is_excluded_from_humanitarian_draws(self) -> None: # Arrived 2000-2001 aged ~7: the DACA statutory cohort. The worker # spill gives the EAD card and the DACA tag survives TPS targeting. person = _person_table( - [_noncitizen(PENATVTY=312, PEINUSYR=17, A_AGE=30, WSAL_VAL=20_000.0)] + [ + _noncitizen( + PENATVTY=312, + PEINUSYR=17, + A_AGE=30, + A_LFSR=1, + WSAL_VAL=20_000.0, + ) + ] ) output = _run( person, @@ -790,18 +891,33 @@ def test_humanitarian_spill_interaction_preserves_worker_control(self) -> None: # TPS extraction removes a residual worker before the spill, so the # remaining undocumented workers still land on the control exactly. person = _person_table( - [_noncitizen(PENATVTY=373, PEINUSYR=28, WSAL_VAL=10_000.0)] - + [_noncitizen(WSAL_VAL=10_000.0) for _ in range(10)] + [ + _noncitizen( + PENATVTY=373, + PEINUSYR=28, + A_LFSR=1, + WSAL_VAL=10_000.0, + ) + ] + + [_noncitizen(A_LFSR=1, WSAL_VAL=10_000.0) for _ in range(10)] ) output = _run( person, workers=4.0, humanitarian=_humanitarian_block(tps={"venezuela": 1.0}), ) - workers = output["WSAL_VAL"] > 0 - undocumented_workers = ((output["ssn_card_type"] == "NONE") & workers).sum() + workers = output["A_LFSR"].isin([1, 2, 3, 4]) + pew_workers = workers & output["immigration_status_str"].isin( + [ + "UNDOCUMENTED", + "DACA", + "PAROLED_ONE_YEAR", + "DEPORTATION_WITHHELD", + "TPS", + ] + ) assert output.loc[0, "immigration_status_str"] == "TPS" - assert undocumented_workers == 4 + assert pew_workers.sum() == 4 def test_missing_humanitarian_block_is_refused(self) -> None: spec_mapping = { @@ -897,7 +1013,9 @@ def test_humanitarian_clones_stay_consistent(self) -> None: class TestDeterminism: def _worker_pool(self) -> pd.DataFrame: - return _person_table([_noncitizen(WSAL_VAL=10_000.0) for _ in range(20)]) + return _person_table( + [_noncitizen(A_LFSR=1, WSAL_VAL=10_000.0) for _ in range(20)] + ) def test_partial_household_lineage_uses_complete_legacy_alternative(self) -> None: person = _person_table([_noncitizen(), _noncitizen()]) @@ -931,6 +1049,7 @@ def test_different_seeds_select_different_ead_holders(self) -> None: def test_source_identity_keys_make_clones_consistent(self) -> None: rows = [ _noncitizen( + A_LFSR=1, WSAL_VAL=10_000.0, person_id=index + 1, source_year=2024, @@ -1133,7 +1252,7 @@ def test_with_us_immigration_inputs_writes_both_columns(self) -> None: rows = ( [{"PRCITSHP": 1} for _ in range(93)] + [_noncitizen(CAID=1) for _ in range(2)] - + [_noncitizen(WSAL_VAL=10_000.0) for _ in range(12)] + + [_noncitizen(A_LFSR=1, WSAL_VAL=10_000.0) for _ in range(12)] + [_noncitizen() for _ in range(5)] ) frame = _us_frame(rows, household_weights=[1e6] * len(rows)) @@ -1143,7 +1262,7 @@ def test_with_us_immigration_inputs_writes_both_columns(self) -> None: assert column in person.columns assert set(person["ssn_card_type"]) <= set(SSN_CARD_TYPE_VALUES) assert (person.loc[person["PRCITSHP"] == 1, "ssn_card_type"] == "CITIZEN").all() - # 12M weighted undocumented workers against the 8.3M Pew control: + # 12M weighted unauthorized workers against the 9.7M Pew control: # some spill to EAD, the rest stay undocumented. assert (person["ssn_card_type"] == "NON_CITIZEN_VALID_EAD").any() assert (person["ssn_card_type"] == "NONE").any() @@ -1226,7 +1345,10 @@ def _composition_frame( + [("NON_CITIZEN_VALID_EAD", "LEGAL_PERMANENT_RESIDENT")] * ead + [("NONE", "UNDOCUMENTED")] * none + [ - ("OTHER_NON_CITIZEN", status) + ( + ("NON_CITIZEN_VALID_EAD" if status == "DACA" else "OTHER_NON_CITIZEN"), + status, + ) for status, count in (humanitarian or {}).items() for _ in range(count) ] @@ -1237,6 +1359,7 @@ def _composition_frame( "ASYLEE": {"PENATVTY": 207, "PEINUSYR": 26}, "DEPORTATION_WITHHELD": {"PENATVTY": 303, "PEINUSYR": 24}, "TPS": {"PENATVTY": 373, "PEINUSYR": 24}, + "DACA": {"PENATVTY": 303, "PEINUSYR": 19, "A_AGE": 29}, } for ssn, status in values: rows.append( @@ -1322,13 +1445,13 @@ def test_fails_when_non_citizen_share_implausible(self) -> None: def test_gate_reads_packaged_controls_by_default(self) -> None: packaged = us_immigration_controls() - assert packaged.undocumented.workers == 8_300_000 + assert packaged.undocumented.workers == 9_700_000 assert packaged.humanitarian_target("refugee") == 160_000 gate = us_immigration_composition_gate(_us_frame([{"PRCITSHP": 1}])) assert not gate.passed controls = gate.details["controls"] - assert controls["undocumented_workers"] == 8_300_000 - assert controls["undocumented_population_anchor"] == 11_000_000 + assert controls["undocumented_workers"] == 9_700_000 + assert controls["undocumented_population_anchor"] == 14_000_000 stocks = controls["humanitarian_status_stocks"] assert stocks["paroled_one_year:afghanistan"]["target"] == 73_566 assert stocks["refugee"] == { @@ -1359,6 +1482,60 @@ def test_gate_passes_with_in_band_humanitarian_masses(self) -> None: assert achieved["refugee"]["relative"] == 1.0 assert achieved["tps"]["target"] == 6.0 + def test_gate_counts_temporary_protections_in_pew_population(self) -> None: + gate = us_immigration_composition_gate( + _composition_frame( + none=10, + humanitarian={"DACA": 5, "PAROLED_ONE_YEAR": 10, "TPS": 10}, + ), + controls=_plausible_controls(paroled_one_year=10.0, tps=10.0), + ) + assert gate.passed, gate.failures + assert gate.details["pew_unauthorized_population"] == 35.0 + + def test_gate_counts_only_residual_ead_cuban_haitian_entrants(self) -> None: + frame = _composition_frame() + person = frame.table("person") + ead_row = person.index[person["ssn_card_type"].eq("NON_CITIZEN_VALID_EAD")][0] + documented_row = person.index[person["ssn_card_type"].eq("OTHER_NON_CITIZEN")][ + 0 + ] + for row in (ead_row, documented_row): + person.loc[row, "immigration_status_str"] = "CUBAN_HAITIAN_ENTRANT" + person.loc[row, "PENATVTY"] = 327 + person.loc[row, "PEINUSYR"] = 24 + + gate = us_immigration_composition_gate( + frame, + controls=_plausible_controls(), + ) + assert gate.passed, gate.failures + assert gate.details["pew_unauthorized_population"] == 31.0 + assert gate.details["pew_unauthorized_paired_status"] == { + "immigration_status_str": "CUBAN_HAITIAN_ENTRANT", + "ssn_card_type": "NON_CITIZEN_VALID_EAD", + } + + def test_gate_excludes_refugees_and_asylees_from_pew_population(self) -> None: + controls = ImmigrationControls( + undocumented=UndocumentedControls( + workers=20.0, + students=5.0, + population_anchor=10.0, + sources=_plausible_controls().undocumented.sources, + ), + humanitarian=_humanitarian_draws(refugee=10.0, asylee=10.0), + ) + gate = us_immigration_composition_gate( + _composition_frame( + none=10, + humanitarian={"REFUGEE": 10, "ASYLEE": 10}, + ), + controls=controls, + ) + assert gate.passed, gate.failures + assert gate.details["pew_unauthorized_population"] == 10.0 + def test_gate_fails_when_humanitarian_category_collapses(self) -> None: gate = us_immigration_composition_gate( _composition_frame(), diff --git a/packages/microcosm-build/tests/test_us_immigration_pool_scaling.py b/packages/microcosm-build/tests/test_us_immigration_pool_scaling.py index 2d9ffe6e4b..45820b87da 100644 --- a/packages/microcosm-build/tests/test_us_immigration_pool_scaling.py +++ b/packages/microcosm-build/tests/test_us_immigration_pool_scaling.py @@ -88,7 +88,7 @@ def _stage_spec() -> SourceStageSpec: def _immigration_frame() -> Frame: rows: list[dict[str, object]] = [] - rows.extend({"PRCITSHP": 5, "WSAL_VAL": 10_000.0} for _ in range(4)) + rows.extend({"PRCITSHP": 5, "A_LFSR": 1, "WSAL_VAL": 10_000.0} for _ in range(4)) rows.extend({"PRCITSHP": 5, "A_HSCOL": 2} for _ in range(3)) rows.extend( { @@ -107,6 +107,7 @@ def _immigration_frame() -> Frame: "A_MARITL": 7, "A_SPOUSE": 0, "A_HSCOL": 0, + "A_LFSR": 7, "WSAL_VAL": 0.0, "SEMP_VAL": 0.0, "MCARE": 2, @@ -197,8 +198,8 @@ def test_stage_weight_scale_allocates_every_absolute_control_to_source_share( == 2.0 ) - standalone_workers = standalone_person["WSAL_VAL"].gt(0) - pooled_workers = pooled_person["WSAL_VAL"].gt(0) + standalone_workers = standalone_person["A_LFSR"].isin([1, 2, 3, 4]) + pooled_workers = pooled_person["A_LFSR"].isin([1, 2, 3, 4]) assert ( _weighted_count( standalone, diff --git a/packages/microcosm-build/tests/test_us_late_producer_dag.py b/packages/microcosm-build/tests/test_us_late_producer_dag.py index 1e5b0a05fa..babf598858 100644 --- a/packages/microcosm-build/tests/test_us_late_producer_dag.py +++ b/packages/microcosm-build/tests/test_us_late_producer_dag.py @@ -1033,6 +1033,7 @@ def test_every_origin_exclusive_raw_input_has_its_native_scope() -> None: } asec_person_raw_columns = { "A_HSCOL", + "A_LFSR", "A_MJOCC", "CAID", "CHAMPVA", @@ -1128,10 +1129,10 @@ def test_every_origin_exclusive_raw_input_has_its_native_scope() -> None: assert observed == Counter( {(*key, required_scope[key]): count for key, count in expected_counts.items()} ) - assert sum(observed.values()) == 107 + assert sum(observed.values()) == 108 assert sum(count for key, count in observed.items() if key[2] == "acs_source") == 46 assert ( - sum(count for key, count in observed.items() if key[2] == "asec_source") == 61 + sum(count for key, count in observed.items() if key[2] == "asec_source") == 62 ) assert receipts[("household", "TYPEHUGQ")] == set() @@ -1341,8 +1342,7 @@ def test_source_numeric_input_audit_is_fully_executable() -> None: "A_MARITL", "A_SPOUSE", "A_HSCOL", - "WSAL_VAL", - "SEMP_VAL", + "A_LFSR", "MCARE", "CAID", "IHSFLG", diff --git a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py index 830ab61287..7080f9851e 100644 --- a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py @@ -2928,7 +2928,7 @@ def capture_equality(expected: object, actual: object) -> None: "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "97922fe1d5c5737c7d51e8af1c77fcad97d564935e5229fbe10b7ef203a123e0", + "spec_sha256": "ae83c7a32a4a2970070f71b95aa7de771a56ab435f71004c11ee2e6b181fe07b", }, } diff --git a/packages/microcosm-build/tests/test_us_plan.py b/packages/microcosm-build/tests/test_us_plan.py index b893f8cbac..309ab0e8fe 100644 --- a/packages/microcosm-build/tests/test_us_plan.py +++ b/packages/microcosm-build/tests/test_us_plan.py @@ -1255,8 +1255,8 @@ class TestBaseStageSourceClosure: #: Raw columns restored from pinned official sidecars because the frozen #: census_cps inputs never carried them (LKWEEKS only for income year - #: 2022; ED_VAL and PAW_TYP for every pooled year). - SIDECAR_RESTORED_COLUMNS = frozenset({"LKWEEKS", "ED_VAL", "PAW_TYP"}) + #: 2022; ED_VAL, A_LFSR, and PAW_TYP for every pooled year). + SIDECAR_RESTORED_COLUMNS = frozenset({"LKWEEKS", "ED_VAL", "A_LFSR", "PAW_TYP"}) #: Release-time stage constants whose inputs are produced inside the #: fiscal-refresh release tool, not the base builder (org_wages consumes diff --git a/packages/microcosm-build/tests/test_us_puf_support_base_builder.py b/packages/microcosm-build/tests/test_us_puf_support_base_builder.py index 7f2d44ddd1..0884dd0e8e 100644 --- a/packages/microcosm-build/tests/test_us_puf_support_base_builder.py +++ b/packages/microcosm-build/tests/test_us_puf_support_base_builder.py @@ -231,6 +231,7 @@ def _education_source() -> pd.DataFrame: "A_LINENO": [1, 2, 1], "PERIDNUM": [f"{value:022d}" for value in (1, 2, 3)], "ED_VAL": [0.0, 500.0, 1_000.0], + "A_LFSR": [1, 3, 7], } ) source.attrs["source_audit"] = {2022: {"rows": 3}} @@ -830,11 +831,13 @@ def test_raw_stage_copy_adds_only_exact_source_mappings( assert frame_identity(source) == before assert "LKWEEKS" not in source.table("person") assert "ED_VAL" not in source.table("person") + assert "A_LFSR" not in source.table("person") assert "PAW_TYP" not in source.table("person") assert raw.table("person")["LKWEEKS"].tolist() == [7.0, -1.0, 12.0] assert raw.table("person")["ED_VAL"].tolist() == [0.0, 500.0, 1_000.0] + assert raw.table("person")["A_LFSR"].tolist() == [1, 3, 7] assert raw.table("person")["PAW_TYP"].tolist() == [0, 1, 2] - assert set(mappings) == {"ED_VAL", "LKWEEKS", "PAW_TYP"} + assert set(mappings) == {"A_LFSR", "ED_VAL", "LKWEEKS", "PAW_TYP"} assert all( mapping["operation"] == "exact_source_join" and mapping["join_keys"] == ["source_year", "PERIDNUM"] @@ -843,6 +846,49 @@ def test_raw_stage_copy_adds_only_exact_source_mappings( builder.assert_operator_free_source_frame(raw, label="raw-stage fixture") +def test_direct_builder_attaches_measured_labor_force_status_without_mutation() -> None: + builder = _load_support_builder_module() + source = _raw_asec_frame() + before = frame_identity(source) + + attached = builder._with_asec_labor_force_status_source( + source, + _education_source(), + ) + + assert frame_identity(source) == before + assert "A_LFSR" not in source.table("person") + assert attached.table("person")["A_LFSR"].tolist() == [1, 3, 7] + assert np.array_equal( + attached.weights_for("household").values, + source.weights_for("household").values, + ) + + +def test_direct_immigration_path_uses_labor_force_status_ephemerally( + monkeypatch: pytest.MonkeyPatch, +) -> None: + builder = _load_support_builder_module() + source = _raw_asec_frame() + observed: list[list[int]] = [] + + def immigration(frame: Frame, *, seed: int, time_period: int) -> Frame: + observed.append(frame.table("person")["A_LFSR"].tolist()) + return frame + + monkeypatch.setattr(builder, "with_us_immigration_inputs", immigration) + result = builder._with_us_immigration_inputs_from_asec_source( + source, + _education_source(), + seed=7, + time_period=2024, + ) + + assert observed == [[1, 3, 7]] + assert "A_LFSR" not in source.table("person") + assert "A_LFSR" not in result.table("person") + + def test_pooled_source_stage_dual_exports_without_changing_legacy_checkpoints( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, @@ -891,10 +937,14 @@ def add_age(frame: Frame, **_kwargs) -> Frame: "with_us_energy_subsidy_input", "with_us_retirement_contribution_inputs", "with_us_retirement_distribution_inputs", - "with_us_immigration_inputs", ) for name in identity_transforms: monkeypatch.setattr(builder, name, lambda value, **_kwargs: value) + monkeypatch.setattr( + builder, + "_with_us_immigration_inputs_from_asec_source", + lambda value, _source, **_kwargs: value, + ) passing_gate = SimpleNamespace(passed=True, failures=(), details={}) for name in ( "us_relationship_inputs_signal_gate", @@ -917,6 +967,7 @@ def add_age(frame: Frame, **_kwargs) -> Frame: assert source_checkpoint.path.read_bytes() == baseline_source.path.read_bytes() assert "ED_VAL" not in source_checkpoint.frame.table("person") assert "LKWEEKS" not in source_checkpoint.frame.table("person") + assert "A_LFSR" not in source_checkpoint.frame.table("person") assert "PAW_TYP" not in source_checkpoint.frame.table("person") raw_path = args.checkpoint_dir / builder.ASEC_RAW_STAGE_CHECKPOINT_FILENAME @@ -924,6 +975,7 @@ def add_age(frame: Frame, **_kwargs) -> Frame: assert raw_metadata["stage"] == "raw_source_mapping" assert raw.table("person")["LKWEEKS"].tolist() == [7.0, -1.0, 12.0] assert raw.table("person")["ED_VAL"].tolist() == [0.0, 500.0, 1_000.0] + assert raw.table("person")["A_LFSR"].tolist() == [1, 3, 7] assert raw.table("person")["PAW_TYP"].tolist() == [0, 1, 2] assert "age" not in raw.table("person") assert [path.name for path in args.checkpoint_dir.glob("*.frame.h5")] == [ @@ -938,6 +990,7 @@ def add_age(frame: Frame, **_kwargs) -> Frame: assert enriched.table("person")["age"].tolist() == [31, 29, 50] assert "ED_VAL" not in enriched.table("person") assert "LKWEEKS" not in enriched.table("person") + assert "A_LFSR" not in enriched.table("person") assert "PAW_TYP" not in enriched.table("person") assert sorted(path.name for path in args.checkpoint_dir.glob("*.frame.h5")) == [ "000_source_construction.frame.h5", @@ -975,6 +1028,7 @@ def test_completed_source_stage_repairs_raw_auxiliary_without_rewriting_legacy( assert context_path.read_bytes() == expected_context repaired, _metadata = builder.load_asec_raw_stage_checkpoint(raw_path) assert repaired.table("person")["ED_VAL"].tolist() == [0.0, 500.0, 1_000.0] + assert repaired.table("person")["A_LFSR"].tolist() == [1, 3, 7] assert repaired.table("person")["PAW_TYP"].tolist() == [0, 1, 2] @@ -1030,10 +1084,14 @@ def test_source_and_preclone_stages_round_trip_design_weight_kind( "with_us_energy_subsidy_input", "with_us_retirement_contribution_inputs", "with_us_retirement_distribution_inputs", - "with_us_immigration_inputs", ) for name in identity_transforms: monkeypatch.setattr(builder, name, lambda value, **_kwargs: value) + monkeypatch.setattr( + builder, + "_with_us_immigration_inputs_from_asec_source", + lambda value, _source, **_kwargs: value, + ) passing_gate = SimpleNamespace(passed=True, failures=(), details={}) for name in ( "us_relationship_inputs_signal_gate", @@ -2185,8 +2243,8 @@ def fake_retirement_distributions( ) monkeypatch.setattr( builder, - "with_us_immigration_inputs", - lambda frame, *, seed, time_period: frame, + "_with_us_immigration_inputs_from_asec_source", + lambda frame, _source, *, seed, time_period: frame, ) monkeypatch.setattr( builder, diff --git a/packages/microcosm-build/tests/test_us_spec_bundle.py b/packages/microcosm-build/tests/test_us_spec_bundle.py index 72cd7a3ee4..915a7e2165 100644 --- a/packages/microcosm-build/tests/test_us_spec_bundle.py +++ b/packages/microcosm-build/tests/test_us_spec_bundle.py @@ -190,7 +190,7 @@ def _load_generator_module(): LEGACY_COMPATIBILITY_SHA256 = { "source_stages.json": ( - "413174820cfb57f85783831dbbf445f007d06128d31562a1234e998e627bd3aa" + "788b815f748abb2061c41efb0cec4cc4952435dbcb4448ecf21ccac85a5ccec2" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" @@ -382,11 +382,11 @@ def test_constant_derived_domain_counts_are_complete( assert len(compiled_schedule["waves"]) == 6 assert ( compiled_schedule["schedule_sha256"] - == "b7e40a89f5100c34b748ce960c209807b98d0f5e2c5834bb4c250aec733a37dc" + == "88bc9243a3518982ae951c3de21bd55877e296ce4fcb183b9bee420d3a684b10" ) assert ( compiled_schedule["payload_sha256"] - == "6260cc59c337c94587354800a368e40128f70158fff3a190a3c9c44f735f1b0d" + == "324b8e495920af091cb7c14f78b461b2a10fb3fedeb4ce706d68ae6b4c2e3b6b" ) assert len(take_up["programs"]) == 17 diff --git a/tools/build_us_puf_support_base.py b/tools/build_us_puf_support_base.py index 5bdd07fe51..a5632ac717 100644 --- a/tools/build_us_puf_support_base.py +++ b/tools/build_us_puf_support_base.py @@ -46,6 +46,7 @@ ASEC_2023_WEEKS_UNEMPLOYED_SOURCE_YEAR, ASEC_2023_WEEKS_UNEMPLOYED_ZIP_URL, ASEC_EDUCATION_ASSISTANCE_ARCHIVES, + ASEC_LABOR_FORCE_STATUS_COLUMN, ASEC_RAW_STAGE_ARTIFACT_KIND, ASEC_RAW_STAGE_CHECKPOINT_FILENAME, ASEC_RAW_STAGE_OPERATOR_STATUS, @@ -61,6 +62,7 @@ PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS, PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS, PUF_TAX_DETAIL_SUPPORT_CHANNEL, + US_IMMIGRATION_OUTPUT_COLUMNS, US_PUF_SUPPORT_FIT_NAME, US_SOURCE_MANIFEST, US_SUPPORT_SPINE_SPEC, @@ -75,6 +77,7 @@ fetch_asec_2023_weeks_unemployed_source, fill_asec_2022_weeks_unemployed_source, fill_asec_education_assistance_source, + fill_asec_labor_force_status_source, fill_asec_public_assistance_type_source, finalize_puf_e01000_reconciliation, impute_us_housing_assistance_to_puf_support, @@ -322,8 +325,8 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: help=( "Optional INCOME_YEAR=PATH mapping to a local copy of the " "SHA-pinned official ASEC survey archive (zip or extracted " - "pppub member) restoring that pooled income year's ED_VAL and " - "PAW_TYP (income year YYYY maps to the survey-year YYYY+1 " + "pppub member) restoring that pooled income year's ED_VAL, " + "A_LFSR, and PAW_TYP (income year YYYY maps to the survey-year YYYY+1 " "archive). Years without a mapping are fetched from the " "official Census archive and verified against the same pins." ), @@ -870,6 +873,80 @@ def _write_policyengine_dataset( ) +def _with_asec_labor_force_status_source( + frame: Frame, + source: pd.DataFrame, +) -> Frame: + """Attach measured A_LFSR without mutating the source-construction frame.""" + + tables = {entity: frame.table(entity).copy(deep=True) for entity in frame.entities} + tables["person"] = fill_asec_labor_force_status_source(tables["person"], source) + return Frame( + tables, + frame.schema, + { + entity: Weights( + frame.weights_for(entity).values.copy(), + frame.weights_for(entity).kind, + ) + for entity in frame.weighted_entities + }, + frame.strata.copy(deep=True), + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +def _with_us_immigration_inputs_from_asec_source( + frame: Frame, + source: pd.DataFrame, + *, + seed: int, + time_period: int, +) -> Frame: + """Run immigration with measured A_LFSR as an ephemeral raw input.""" + + person = frame.table("person") + if set(US_IMMIGRATION_OUTPUT_COLUMNS).issubset(person.columns): + return with_us_immigration_inputs( + frame, + seed=seed, + time_period=time_period, + ) + carried_labor_force_status = ASEC_LABOR_FORCE_STATUS_COLUMN in person + staged = ( + frame + if carried_labor_force_status + else _with_asec_labor_force_status_source(frame, source) + ) + result = with_us_immigration_inputs( + staged, + seed=seed, + time_period=time_period, + ) + if carried_labor_force_status: + return result + + tables = { + entity: result.table(entity).copy(deep=True) for entity in result.entities + } + tables["person"] = tables["person"].drop(columns=[ASEC_LABOR_FORCE_STATUS_COLUMN]) + return Frame( + tables, + result.schema, + { + entity: Weights( + result.weights_for(entity).values.copy(), + result.weights_for(entity).kind, + ) + for entity in result.weighted_entities + }, + result.strata.copy(deep=True), + mass_log=result.mass_log, + metadata=result.metadata, + ) + + def _run_all( args: argparse.Namespace, *, @@ -1027,8 +1104,9 @@ def _run_all( seed=args.seed, time_period=args.target_year, ) - base = with_us_immigration_inputs( + base = _with_us_immigration_inputs_from_asec_source( base, + education_assistance_source, seed=args.seed, time_period=args.target_year, ) @@ -1918,6 +1996,7 @@ def _asec_raw_source_mapping_frame( weeks_source, ) person = fill_asec_education_assistance_source(person, education_source) + person = fill_asec_labor_force_status_source(person, education_source) person = fill_asec_public_assistance_type_source( person, public_assistance_type_source, @@ -1962,6 +2041,14 @@ def _asec_raw_source_mapping_frame( "operation": "exact_source_join", "source_pins": education_pins, }, + "A_LFSR": { + "audit": dict(education_source.attrs.get("source_audit", {})), + "column": "A_LFSR", + "entity": "person", + "join_keys": ["source_year", "PERIDNUM"], + "operation": "exact_source_join", + "source_pins": education_pins, + }, "LKWEEKS": { "audit": dict(weeks_source.attrs.get("source_audit", {})), "column": "LKWEEKS", @@ -1978,8 +2065,8 @@ def _asec_raw_source_mapping_frame( } ], }, - # PAW_TYP lives in the same pinned survey-year person members as - # ED_VAL, so the mapping reuses those archive pins (microcosm#591). + # PAW_TYP and A_LFSR live in the same pinned survey-year person members + # as ED_VAL, so their mappings reuse those archive pins. "PAW_TYP": { "audit": dict(public_assistance_type_source.attrs.get("source_audit", {})), "column": "PAW_TYP", @@ -2002,6 +2089,10 @@ def _pre_clone_enrichment_stage( _asec_education_source_paths(args), income_years=_pooled_income_years(args), ) + education_assistance_source = load_asec_education_assistance_sources( + _asec_education_source_paths(args), + income_years=_pooled_income_years(args), + ) base = derive_us_cps_carried_inputs( raw_base, public_assistance_type_source=public_assistance_type_source, @@ -2120,8 +2211,9 @@ def _pre_clone_enrichment_stage( seed=args.seed, time_period=args.target_year, ) - base = with_us_immigration_inputs( + base = _with_us_immigration_inputs_from_asec_source( base, + education_assistance_source, seed=args.seed, time_period=args.target_year, ) diff --git a/tools/generate_us_bundle_from_constants.py b/tools/generate_us_bundle_from_constants.py index 542b75c2de..31c4de9b36 100644 --- a/tools/generate_us_bundle_from_constants.py +++ b/tools/generate_us_bundle_from_constants.py @@ -122,7 +122,7 @@ def _domain_builders() -> dict[str, Callable[[], object]]: # frozen files, is the forward YAML -> legacy-payload path. FROZEN_LEGACY_RESOURCE_SHA256 = { "source_stages.json": ( - "413174820cfb57f85783831dbbf445f007d06128d31562a1234e998e627bd3aa" + "788b815f748abb2061c41efb0cec4cc4952435dbcb4448ecf21ccac85a5ccec2" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" diff --git a/tools/spec_engine_coverage.py b/tools/spec_engine_coverage.py index b5d5a78657..b2eb3317d6 100644 --- a/tools/spec_engine_coverage.py +++ b/tools/spec_engine_coverage.py @@ -41,7 +41,7 @@ REPORT_SCHEMA_VERSION = 3 EXPECTED_POINTER_INVENTORY_SHA256 = ( - "29c8dbe6c5335217b3d5ac7fb239a75c48b33362637611dcf297597eca20352f" + "faef18fd543699ba65a91f3ffaf6fdf38a807ffdc88bc3debfeb970881050d62" ) DEFAULT_REPORT_PATH = ( Path(__file__).resolve().parents[1] diff --git a/tools/us_bundle_generation/imputation.py b/tools/us_bundle_generation/imputation.py index c045387eab..5f569cdb6a 100644 --- a/tools/us_bundle_generation/imputation.py +++ b/tools/us_bundle_generation/imputation.py @@ -2172,7 +2172,7 @@ def _assert_invariants( late_authored_output_count, len(tolerated_receipts), ) - expected_graph_counts = (2751, 92, 227, 35, 0, 213) + expected_graph_counts = (2750, 92, 227, 35, 0, 213) if graph_counts != expected_graph_counts: raise RuntimeError( "US producer graph input/output/absence counts changed: " From e6db756b2538152e86521dd0a935dcbf5a7afa6a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 29 Aug 2026 02:16:43 +0200 Subject: [PATCH 12/49] Refresh seed hashes and multispine fixture --- .../microcosm-build/tests/test_spec_engine_country_bundles.py | 4 ++-- packages/microcosm-build/tests/test_us_multispine_pool.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index 949bd7fd9c..913b10d5f2 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -32,7 +32,7 @@ [ ( "be", - "b34cbb236a7373c520028976a043f241a9979cece43365024f26beea635b7cc5", + "16118de0f95b7821a7ede20104d1466235d6900c665d5220b2a68a0a52007515", { "household.household_id", "person.person_id", @@ -42,7 +42,7 @@ ), ( "uk", - "07d59c3edcb2b66bc83bb2650f66ae713b9718cb98ec3135bbc8b4ba15ba53b3", + "c4e77decba4be76d920af1c61c1244c365132bea0e77324c1a1f8f868475b990", { "benunit.benunit_id", "household.household_id", diff --git a/packages/microcosm-build/tests/test_us_multispine_pool.py b/packages/microcosm-build/tests/test_us_multispine_pool.py index 665bdd41e9..9afc509b06 100644 --- a/packages/microcosm-build/tests/test_us_multispine_pool.py +++ b/packages/microcosm-build/tests/test_us_multispine_pool.py @@ -1847,6 +1847,7 @@ def _producer_dtype_source_frame() -> Frame: person["PRCITSHP"] = [1, 5, 1, 5] person["PEINUSYR"] = [0, 24, 0, 24] person["PENATVTY"] = [57, 303, 57, 303] + person["A_LFSR"] = [1, 0, 1, 0] person["A_SPOUSE"] = 0 person["CAID"] = 2 person["IHSFLG"] = 2 From 96c88e6817102eef1ae56ab83012b2d01fd48636 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 29 Aug 2026 02:50:21 +0200 Subject: [PATCH 13/49] Refresh semantic hash golden vector --- packages/microcosm-build/tests/test_spec_engine_loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/microcosm-build/tests/test_spec_engine_loader.py b/packages/microcosm-build/tests/test_spec_engine_loader.py index 89825c56c7..4d66b25165 100644 --- a/packages/microcosm-build/tests/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/test_spec_engine_loader.py @@ -236,7 +236,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # Pin the domain separator, normalization rules, schema-set receipt, and # exact normative projection as one reviewable golden vector. assert first.spec_sha256 == ( - "095210345d1b1c29189a79289f5dfe05f070bbb1f012a77ff6c371b610210e3d" + "8b72d260c755e46b4a07043ee81949ff79196e9b459cd9ae1096e99eeed2f4e4" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") From 85e3627eb84bb16a5bb5951cc39849397915d17c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 4 Sep 2026 08:21:21 -0400 Subject: [PATCH 14/49] Merge main and re-pin the attested identities the merge moved Resolves the six pin collisions against current main: BE/UK envelope digests, the loader golden vector, the US spec digest in the multispine tool test, the nine inventory digests plus the producer-input count the A_LFSR column adds, and the regenerated coverage evidence. The h5_io test is a real three-way merge (branch kept its stacked-pool immigration receipt tests; main's failed-gate fixture helper replaces the inlined block). Re-applies the branch's seed-group entries and site counts on main's inventory module. Refs #767 Co-Authored-By: Claude Fable 5.1 --- docs/evidence/spec-engine/us-f0-coverage.json | 135 +++-- .../build/spec_engine/inventory_coverage.py | 51 +- .../tests/test_spec_engine_country_bundles.py | 4 +- .../tests/test_spec_engine_loader.py | 2 +- .../tests/test_us_multispine_pool_h5_io.py | 484 ++++++++++++++-- .../tests/test_us_multispine_pool_tool.py | 541 ++++++++++++++++-- 6 files changed, 1036 insertions(+), 181 deletions(-) diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index 5365a086aa..fa51156959 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -4,9 +4,9 @@ "version": 1 }, "country": "us", - "documentation_sha256": "4b39450dbdb8dafb83c3b627123b8026c6f82c660b66fe76f341a67c4f37c77b", + "documentation_sha256": "02ac100a7aaa1770930dd6005b2d7f26550560e769c4ffb8d642b76f1774dd84", "field_usage": { - "authored_normative_field_count": 32384, + "authored_normative_field_count": 32521, "claim_count": 49, "claims": [ { @@ -220,8 +220,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 97, - "pointer_sha256": "cabaaa3d96c534f5ec37f51811c3f549d0ed20791e2892c868888c0da6d7d80e", + "pointer_count": 115, + "pointer_sha256": "545f626e7b9c6aed25a5ae8dd3156c96db2cd35e5637ec1ebe443432d1684277", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/spec~1imputation.yaml/transfer_execution", @@ -274,8 +274,8 @@ ], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 24488, - "pointer_sha256": "6382a3ed07016414c31462b8bedb24590d92b792405a89c927d85fa1bd2a8347", + "pointer_count": 24554, + "pointer_sha256": "18140dbc554c2a17a673fec5ae5839f504d4b9980c5d8f1c1ccb545342977cb4", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/spec~1imputation.yaml/producer_graph", @@ -514,8 +514,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 1702, - "pointer_sha256": "be3e24b53955b9dabccfc7e779212b1e97daf0c0992be3c183e391bf695ed4ee", + "pointer_count": 1715, + "pointer_sha256": "78227f34c3f4ef946c06693dec9177dd906069f98016854bfd4d39c4cd4c51fa", "rationale": null, "relative_sink_prefix": "/source_manifest/stages", "source_prefix": "/authored/spec~1sources.yaml/stages", @@ -530,8 +530,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 88, - "pointer_sha256": "ae011455154bae0df3913ca9a056058a909d913439b292cb46c11f63c7d0d9a3", + "pointer_count": 89, + "pointer_sha256": "8d281f8f1e2f9914684150b0424ff816d0ec09ed0d9053ea5a64e877a86cc35b", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/spec~1spine.yaml/pipeline_contract", @@ -544,8 +544,8 @@ "legacy_sinks": [], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 277, - "pointer_sha256": "dfa7ae701f62f7b1f06954798d29cfba3fc53a0be21e1751646d1f5bafd6071f", + "pointer_count": 316, + "pointer_sha256": "bb848f38e068858ac23e964a4ffbbf5542253da50515c72fe7753d8c56912486", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/spec~1spine.yaml/seed_site_bindings", @@ -736,8 +736,8 @@ "legacy_sinks": [], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 824, - "pointer_sha256": "7537385c3fd399a2dbb7dcd8ed7cf1ff2481ed336db621eafcbfd741d5792f40", + "pointer_count": 1032, + "pointer_sha256": "6d4af2d988ee87ed38a044c1007e854176af17be4782e802d86d960f10376f7a", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/resolved/seed_protocol", @@ -750,8 +750,8 @@ "legacy_sinks": [], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 277, - "pointer_sha256": "368fc19a8f07c4abf38ebcf2fbc4414d33c403c0f1f9881c2d2e3f5d6160feb6", + "pointer_count": 316, + "pointer_sha256": "199ec98045871070860164b522892327ab840b48c0c6a53972ebc0ff0d7d091f", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/resolved/seed_site_bindings", @@ -772,21 +772,21 @@ "verifier": "vintages" } ], - "configuration_field_count": 42154, - "consumed_field_count": 42154, + "configuration_field_count": 42538, + "consumed_field_count": 42538, "generation0_effect_counts": { - "legacy_behavior": 38476, - "no_generation0_effect": 3678 + "legacy_behavior": 38574, + "no_generation0_effect": 3964 }, "mode_counts": { - "compiler_semantic": 27715, + "compiler_semantic": 28067, "front_end_validation": 348, "identity_only": 103, - "legacy_behavior": 13988 + "legacy_behavior": 14020 }, "multiple_primary_use_field_count": 0, - "pointer_inventory_sha256": "3fc6b9480ea81b9635bd0db56e180c2daf32a5cd2006a70d350586c570f96754", - "resolved_binding_field_count": 9770, + "pointer_inventory_sha256": "faef18fd543699ba65a91f3ffaf6fdf38a807ffdc88bc3debfeb970881050d62", + "resolved_binding_field_count": 10017, "unused_field_count": 0 }, "inventory_coverage": { @@ -810,14 +810,14 @@ "primary_targets": 65, "producer_authored_outputs": 92, "producer_compiled_outputs": 227, - "producer_inputs": 2744, + "producer_inputs": 2750, "producer_nodes": 38, "producer_virtual_resources": 75, "release_rungs": 5, "resolved_references": 334, - "seed_owner_bindings": 112, + "seed_owner_bindings": 125, "seed_owner_rows": 54, - "seed_sites": 53, + "seed_sites": 66, "seed_streams": 14, "source_operators": 16, "source_stages": 37, @@ -1074,11 +1074,11 @@ "legacy_adapter.stacked_checkpoint_static_components" ], "expected": { - "schedule_sha256": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f" + "schedule_sha256": "88bc9243a3518982ae951c3de21bd55877e296ce4fcb183b9bee420d3a684b10" }, "failures": [], "observed": { - "schedule_sha256": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f" + "schedule_sha256": "88bc9243a3518982ae951c3de21bd55877e296ce4fcb183b9bee420d3a684b10" }, "status": "covered" }, @@ -1250,14 +1250,14 @@ "expected": { "edges": 71, "nodes": 38, - "schedule_sha256": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", + "schedule_sha256": "88bc9243a3518982ae951c3de21bd55877e296ce4fcb183b9bee420d3a684b10", "waves": 6 }, "failures": [], "observed": { "edges": 71, "nodes": 38, - "schedule_sha256": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", + "schedule_sha256": "88bc9243a3518982ae951c3de21bd55877e296ce4fcb183b9bee420d3a684b10", "waves": 6 }, "status": "covered" @@ -1275,11 +1275,11 @@ ], "expected": { "relation": "source rows preserved exactly", - "rows": 2744 + "rows": 2750 }, "failures": [], "observed": { - "rows": 2744 + "rows": 2750 }, "status": "covered" }, @@ -1321,7 +1321,7 @@ "expected": "complete execution-row and transition-authority object", "failures": [], "observed": { - "sha256": "503428f6e9d98f19ed3a6ada5bc9883ae44c1b5dc27f09e60b7ac98895a99bc0" + "sha256": "b15bf2dcf709cf6c7922d4b76e1d64c14a6ec2f9aa1185e6770505d798333c76" }, "status": "covered" }, @@ -1338,12 +1338,12 @@ ], "expected": { "nodes": 38, - "sha256": "271a7bb8d0b3f97ff344e0b7e68184fa74738a6585c24fc8781793db669f388b" + "sha256": "014f90315324f72f45ece928bccc44f34b4d1e95c0582f1d7d73620f482da15f" }, "failures": [], "observed": { "nodes": 38, - "sha256": "271a7bb8d0b3f97ff344e0b7e68184fa74738a6585c24fc8781793db669f388b" + "sha256": "014f90315324f72f45ece928bccc44f34b4d1e95c0582f1d7d73620f482da15f" }, "status": "covered" }, @@ -1361,12 +1361,12 @@ ], "expected": { "producer_count": 38, - "sha256": "afebb6725373abf5b8dd4fdb77bf2814cb6fcc569cb606c0c30963a8f65c0bab" + "sha256": "db175a952340b3ff2774215f320c4d19592c06b9959442639a6c561055f2442d" }, "failures": [], "observed": { "producer_count": 38, - "sha256": "afebb6725373abf5b8dd4fdb77bf2814cb6fcc569cb606c0c30963a8f65c0bab" + "sha256": "db175a952340b3ff2774215f320c4d19592c06b9959442639a6c561055f2442d" }, "status": "covered" }, @@ -1533,7 +1533,7 @@ "expected": { "groups": 12, "partition": "disjoint and exhaustive", - "sites": 53 + "sites": 66 }, "failures": [], "observed": { @@ -1550,6 +1550,19 @@ "pregnancy_assignment", "wic_claim_assignment", "snap_discretionary_exemption_assignment", + "immigration_humanitarian_paroled_one_year_afghanistan_assignment", + "immigration_humanitarian_paroled_one_year_ukraine_assignment", + "immigration_humanitarian_paroled_one_year_nicaragua_assignment", + "immigration_humanitarian_paroled_one_year_venezuela_assignment", + "immigration_humanitarian_refugee_assignment", + "immigration_humanitarian_asylee_assignment", + "immigration_humanitarian_deportation_withheld_assignment", + "immigration_humanitarian_tps_venezuela_assignment", + "immigration_humanitarian_tps_el_salvador_assignment", + "immigration_humanitarian_tps_honduras_assignment", + "immigration_humanitarian_tps_nicaragua_assignment", + "immigration_humanitarian_tps_nepal_assignment", + "immigration_humanitarian_tps_other_designated_assignment", "immigration_ead_workers_assignment", "immigration_ead_students_assignment", "ssi_take_up_assignment", @@ -1616,7 +1629,7 @@ "sipp_tip_training_cap" ] }, - "sites": 53 + "sites": 66 }, "status": "covered" }, @@ -1656,13 +1669,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "6b4902d9a640dd459942e588ae2cc7fb937f1c42ad814365c7c1616c98ce1b68", - "protocol_sha256": "e63bbfa0f05302a672acd9914e3652c151cc7ee690182d8823dfb85329b1e911" + "map_sha256": "d2112f002b039e67f4e4e0dcb1e7bd64e41cc794a9e1904a6d9ee59413ef4653", + "protocol_sha256": "c579d1be2c9e884b5d5fcb554a8d3ffff62fc94365cdef9a5ffac34e2bf42c00" }, "failures": [], "observed": { - "map_sha256": "6b4902d9a640dd459942e588ae2cc7fb937f1c42ad814365c7c1616c98ce1b68", - "protocol_sha256": "e63bbfa0f05302a672acd9914e3652c151cc7ee690182d8823dfb85329b1e911" + "map_sha256": "d2112f002b039e67f4e4e0dcb1e7bd64e41cc794a9e1904a6d9ee59413ef4653", + "protocol_sha256": "c579d1be2c9e884b5d5fcb554a8d3ffff62fc94365cdef9a5ffac34e2bf42c00" }, "status": "covered" }, @@ -1677,7 +1690,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "e63bbfa0f05302a672acd9914e3652c151cc7ee690182d8823dfb85329b1e911", + "implementation_sha256": "c579d1be2c9e884b5d5fcb554a8d3ffff62fc94365cdef9a5ffac34e2bf42c00", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1698,7 +1711,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "e63bbfa0f05302a672acd9914e3652c151cc7ee690182d8823dfb85329b1e911", + "implementation_sha256": "c579d1be2c9e884b5d5fcb554a8d3ffff62fc94365cdef9a5ffac34e2bf42c00", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1732,12 +1745,12 @@ ], "expected": { "relation": "all site fields preserved exactly", - "sites": 53 + "sites": 66 }, "failures": [], "observed": { - "sha256": "a0366cd518f42a240b589af3ef6580aac44ad7ed70969bcc669b365d315146ab", - "sites": 53 + "sha256": "5c22dfb6f3cd7c23c527e054ba6f5040ca2eb4235df8659e3f7c61ebe2ae6c4f", + "sites": 66 }, "status": "covered" }, @@ -1752,12 +1765,12 @@ "compiler_ir.seed_stream_map.sites.owners" ], "expected": { - "bindings": 112, - "coverage": "all 53 sites" + "bindings": 125, + "coverage": "all 66 sites" }, "failures": [], "observed": { - "bindings": 112 + "bindings": 125 }, "status": "covered" }, @@ -1775,12 +1788,12 @@ "legacy_adapter.imputation.late_producer_resource_semantics" ], "expected": { - "sha256": "cd5ba8924d64da5425ee14cca82a774e3f4b2bb5aabe06df291cc3cc457287a9", + "sha256": "b82d911a0263ebc1b6d5d8f82bb1076fd8d779bea49072ec42f02a8fe027cbe3", "stages": 37 }, "failures": [], "observed": { - "sha256": "cd5ba8924d64da5425ee14cca82a774e3f4b2bb5aabe06df291cc3cc457287a9", + "sha256": "b82d911a0263ebc1b6d5d8f82bb1076fd8d779bea49072ec42f02a8fe027cbe3", "stages": 37 }, "status": "covered" @@ -1840,11 +1853,11 @@ "legacy_adapter.stacked_checkpoint_static_components" ], "expected": { - "sha256": "e660a8ce42b69a39d29c5f0ec37264bc69d61b03f27adc386336ec8889531bb2" + "sha256": "406b2cf93a7fb94dc63f1a24a52cfb20362ab83195eb8ebf61d4ef43072280ec" }, "failures": [], "observed": { - "sha256": "e660a8ce42b69a39d29c5f0ec37264bc69d61b03f27adc386336ec8889531bb2" + "sha256": "406b2cf93a7fb94dc63f1a24a52cfb20362ab83195eb8ebf61d4ef43072280ec" }, "status": "covered" }, @@ -1887,7 +1900,7 @@ "alpha", "zeta" ], - "sha256": "b88f2d9c0f6f92c6cd81eb14d6b126afe59577b8bb392b394b2c6fbbafd195c5" + "sha256": "a53da3c26dfaf8f400fb74ef92f79555dd0c152319aa50317b52c5603d0af7b4" }, "failures": [], "observed": { @@ -1911,7 +1924,7 @@ "alpha", "zeta" ], - "sha256": "b88f2d9c0f6f92c6cd81eb14d6b126afe59577b8bb392b394b2c6fbbafd195c5" + "sha256": "a53da3c26dfaf8f400fb74ef92f79555dd0c152319aa50317b52c5603d0af7b4" }, "status": "covered" }, @@ -1975,7 +1988,7 @@ "take_up_contract", "us_qbi_reconciliation_contract" ], - "sha256": "04899daa491e8f089899c9df64cdb2ed44d61d11da2b4c733db6f20f38a1668a" + "sha256": "fbc3631977c43ba558f89b3f36c1b8d0d3958a593f93198c0c5b5833cd18999b" }, "status": "covered" }, @@ -2599,7 +2612,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "a6bc79878eb6f64637b9f3eceeea6cc2b050c0e5b8f9aca446179258940c44f2" + "spec_sha256": "ea5cc4f31ac7cd67a0d01bccf9e1454755c879f045a813741c84d17cbddd796d" } }, "report_schema_version": 3, @@ -2609,7 +2622,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "a6bc79878eb6f64637b9f3eceeea6cc2b050c0e5b8f9aca446179258940c44f2" + "spec_sha256": "ea5cc4f31ac7cd67a0d01bccf9e1454755c879f045a813741c84d17cbddd796d" }, "status": "pass" } diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 1c38f83573..d69bfd09db 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -296,6 +296,19 @@ "pregnancy_assignment", "wic_claim_assignment", "snap_discretionary_exemption_assignment", + "immigration_humanitarian_paroled_one_year_afghanistan_assignment", + "immigration_humanitarian_paroled_one_year_ukraine_assignment", + "immigration_humanitarian_paroled_one_year_nicaragua_assignment", + "immigration_humanitarian_paroled_one_year_venezuela_assignment", + "immigration_humanitarian_refugee_assignment", + "immigration_humanitarian_asylee_assignment", + "immigration_humanitarian_deportation_withheld_assignment", + "immigration_humanitarian_tps_venezuela_assignment", + "immigration_humanitarian_tps_el_salvador_assignment", + "immigration_humanitarian_tps_honduras_assignment", + "immigration_humanitarian_tps_nicaragua_assignment", + "immigration_humanitarian_tps_nepal_assignment", + "immigration_humanitarian_tps_other_designated_assignment", "immigration_ead_workers_assignment", "immigration_ead_students_assignment", "ssi_take_up_assignment", @@ -348,20 +361,20 @@ EXPECTED_HASHES = { "acs_group_predictors": "a927bb7ecf3e84f54c93583ab79318654514ac546aefafba67da5285615fbd60", "acs_person_predictors": "878c788a6f037d7aca12b3586ea034eff04f3034ffa11935a736493042551f25", - "authority": "e660a8ce42b69a39d29c5f0ec37264bc69d61b03f27adc386336ec8889531bb2", + "authority": "406b2cf93a7fb94dc63f1a24a52cfb20362ab83195eb8ebf61d4ef43072280ec", "early_families": "4aa9f736fd76e83955477ad1667e58f48f264783f05bdc7f0102cd32d61323bd", - "full_checkpoint": "b88f2d9c0f6f92c6cd81eb14d6b126afe59577b8bb392b394b2c6fbbafd195c5", + "full_checkpoint": "a53da3c26dfaf8f400fb74ef92f79555dd0c152319aa50317b52c5603d0af7b4", "gap_fill_schedule": "1c31f9868f7884347cc19cf1ff65da43f950b9114941a715bab168246db414a7", - "graph_nodes": "271a7bb8d0b3f97ff344e0b7e68184fa74738a6585c24fc8781793db669f388b", + "graph_nodes": "014f90315324f72f45ece928bccc44f34b4d1e95c0582f1d7d73620f482da15f", "geography_assignment": "f49425ca8734ac559c73cf44f6458d86d3162a48956b98a27e6e758959361585", "late_families": "d91f9ff0eb52f43e7b6eed3d5c58c37abe1620c3a11021da15dae9c10e16d382", - "late_resource_semantics": "afebb6725373abf5b8dd4fdb77bf2814cb6fcc569cb606c0c30963a8f65c0bab", - "late_schedule": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", + "late_resource_semantics": "db175a952340b3ff2774215f320c4d19592c06b9959442639a6c561055f2442d", + "late_schedule": "88bc9243a3518982ae951c3de21bd55877e296ce4fcb183b9bee420d3a684b10", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "987b501c695e31f45521c4a178528f75ab3df22c09bc407b182213b2de99ee57", - "seed_map": "6b4902d9a640dd459942e588ae2cc7fb937f1c42ad814365c7c1616c98ce1b68", - "seed_protocol": "e63bbfa0f05302a672acd9914e3652c151cc7ee690182d8823dfb85329b1e911", - "source_manifest": "cd5ba8924d64da5425ee14cca82a774e3f4b2bb5aabe06df291cc3cc457287a9", + "seed_map": "d2112f002b039e67f4e4e0dcb1e7bd64e41cc794a9e1904a6d9ee59413ef4653", + "seed_protocol": "c579d1be2c9e884b5d5fcb554a8d3ffff62fc94365cdef9a5ffac34e2bf42c00", + "source_manifest": "b82d911a0263ebc1b6d5d8f82bb1076fd8d779bea49072ec42f02a8fe027cbe3", "take_up": "fa186daea0f8dd641cc470e41d1a2953f887d45282ec990201298f47bedf8d4d", "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", } @@ -429,14 +442,14 @@ "primary_targets": 65, "producer_authored_outputs": 92, "producer_compiled_outputs": 227, - "producer_inputs": 2_744, + "producer_inputs": 2_750, "producer_nodes": 38, "producer_virtual_resources": 75, "release_rungs": 5, "resolved_references": 334, - "seed_owner_bindings": 112, + "seed_owner_bindings": 125, "seed_owner_rows": 54, - "seed_sites": 53, + "seed_sites": 66, "seed_streams": 14, "source_operators": 16, "source_stages": 37, @@ -1005,7 +1018,7 @@ def add( "producer_inputs_exact", clauses={ "producer input rows differ": inputs_exact, - "input row count differs": input_count == 2744, + "input row count differs": input_count == 2750, }, homes=("/imputation/producer_graph/nodes/*/inputs",), consumers=( @@ -1013,7 +1026,7 @@ def add( "compiler_ir.node_slices", ), observed={"rows": input_count}, - expected={"rows": 2744, "relation": "source rows preserved exactly"}, + expected={"rows": 2750, "relation": "source rows preserved exactly"}, ) outputs_exact = set(expected_outputs) == set(compiled_by_id) and all( _json_equal( @@ -1773,7 +1786,7 @@ def add( "seed_site_definitions_exact", clauses={ "seed site definitions differ": site_definitions_exact, - "seed site count differs": len(protocol_sites) == 53, + "seed site count differs": len(protocol_sites) == 66, }, homes=("/bundle/seed_protocol",), consumers=("compiler_ir.seed_stream_map.sites", "compiler_ir.node_slices"), @@ -1781,7 +1794,7 @@ def add( "sites": len(protocol_sites), "sha256": sha256_json([site.to_wire() for site in protocol.sites]), }, - expected={"sites": 53, "relation": "all site fields preserved exactly"}, + expected={"sites": 66, "relation": "all site fields preserved exactly"}, ) binding_by_site = {binding.site: binding for binding in spec.seed_site_bindings} binding_exact = set(binding_by_site) == set(compiled_sites) and all( @@ -1796,7 +1809,7 @@ def add( "owner binding count differs": sum( len(site.owners) for site in compiled_sites.values() ) - == 112, + == 125, "one or more sites have no owner": all( site.owners for site in compiled_sites.values() ), @@ -1806,7 +1819,7 @@ def add( observed={ "bindings": sum(len(site.owners) for site in compiled_sites.values()) }, - expected={"bindings": 112, "coverage": "all 53 sites"}, + expected={"bindings": 125, "coverage": "all 66 sites"}, ) expected_owner_sites: dict[tuple[str, str], list[str]] = {} for site in protocol.sites: @@ -1850,7 +1863,7 @@ def add( "seed groups overlap": groups_disjoint, "seed groups do not cover the protocol exactly": set(grouped_ids) == set(protocol_sites), - "seed group cardinality differs": len(grouped_ids) == 53, + "seed group cardinality differs": len(grouped_ids) == 66, }, homes=("/bundle/seed_protocol", "/spine/seed_site_bindings"), consumers=("compiler_ir.seed_stream_map",), @@ -1860,7 +1873,7 @@ def add( }, expected={ "groups": len(EXPECTED_SEED_GROUPS), - "sites": 53, + "sites": 66, "partition": "disjoint and exhaustive", }, ) diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index 3298e907dd..5b7f763136 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -25,7 +25,7 @@ "spine", "vintages", } -AM_SPEC_SHA256 = "b128d14f8e6351d745a16ef4537d5c1b8d71d11b9d96cb8a65f1a4fb953f13ed" +AM_SPEC_SHA256 = "6bb0cff6a0ac9d077573126f8f4c13868534e5135dec328917e253dd58f1e089" @pytest.mark.parametrize( @@ -45,7 +45,7 @@ ), ( "be", - "8c6018eaf518b239a5625f8df8bf783ce4723a03704d100f3dbf4c2bb7765772", + "ecfaa9088326d08db364fca3c76c5016706a8b7ca418098a65a1e32b5b896a48", { "household.household_id", "person.person_id", diff --git a/packages/microcosm-build/tests/test_spec_engine_loader.py b/packages/microcosm-build/tests/test_spec_engine_loader.py index a7effd5068..006aef4842 100644 --- a/packages/microcosm-build/tests/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/test_spec_engine_loader.py @@ -236,7 +236,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # Pin the domain separator, normalization rules, schema-set receipt, and # exact normative projection as one reviewable golden vector. assert first.spec_sha256 == ( - "7a0fbb0a2a16aff28a9ac6e205e6679b013c580171db87b39597a0b14a97e423" + "6b3f486cb6047b66212668de686618445db06c0c63dc6ee2266c79350a5090ac" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") diff --git a/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py b/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py index 6f51e3f5e9..3096322a57 100644 --- a/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py +++ b/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py @@ -8,8 +8,9 @@ import pandas as pd import pytest -import microcosm.build.us_runtime.acs_transfer as acs_transfer_module +import microcosm.build.us_runtime.acs_transfer as acs_transfer_runtime import microcosm.build.us_runtime.h5_io as h5_io +import microcosm.build.us_runtime.immigration as immigration_runtime import microcosm.build.us_runtime.post_transfer_calibration as post_transfer_calibration_runtime import microcosm.build.us_runtime.stacked_spine as stacked_spine_module from microcosm.build.frame_checkpoint import ( @@ -45,6 +46,10 @@ write_nullable_us_h5, ) from microcosm.build.us_runtime.support_provenance import ( + BASE_ASEC_SUPPORT_CHANNEL, + spine_assembly_manifest, + spine_provenance_counts, + support_channel_column, support_clone_index_column, support_source_id_column, ) @@ -97,10 +102,7 @@ def _pool_frame() -> Frame: ) -def _pool_frame_with_object_strings_on_every_entity( - *, - stacked: bool = False, -) -> Frame: +def _pool_frame_with_object_strings_on_every_entity() -> Frame: """Match the assembled pool's object-backed source-string shape.""" frame = _pool_frame() @@ -118,21 +120,6 @@ def _pool_frame_with_object_strings_on_every_entity( ), ) tables[entity] = table - if stacked: - household = tables["household"].copy() - household["puma"] = ["0600101", "0600102", "0600102"] - household["congressional_district_geoid"] = np.asarray( - [601, 601, 601], - dtype=np.int64, - ) - household["county_fips"] = ["06001", "06001", "06001"] - household[support_source_id_column("household")] = np.asarray( - [10, 20, 20], dtype=np.int64 - ) - household[support_clone_index_column("household")] = np.asarray( - [0, 0, 1], dtype=np.int64 - ) - tables["household"] = household return Frame( tables, frame.schema, @@ -143,6 +130,181 @@ def _pool_frame_with_object_strings_on_every_entity( ) +def _stacked_pool_frame_with_live_immigration( + *, + include_zero_weight_clone: bool = False, +) -> tuple[Frame, dict[str, object]]: + """Build a persisted-shape final stack and its real immigration receipt.""" + + controls = immigration_runtime.us_immigration_controls() + positive_draws = tuple(draw for draw in controls.humanitarian if draw.target > 0) + acs_evidence = { + "paroled_one_year:afghanistan": (200, 2021, "NONE", "UNDOCUMENTED"), + "paroled_one_year:ukraine": (164, 2022, "NONE", "UNDOCUMENTED"), + "paroled_one_year:nicaragua": (315, 2023, "NONE", "UNDOCUMENTED"), + "paroled_one_year:venezuela": (373, 2022, "NONE", "UNDOCUMENTED"), + "refugee": ( + 412, + 2022, + "OTHER_NON_CITIZEN", + "LEGAL_PERMANENT_RESIDENT", + ), + "asylee": ( + 207, + 2020, + "OTHER_NON_CITIZEN", + "LEGAL_PERMANENT_RESIDENT", + ), + "tps:venezuela": (373, 2015, "NONE", "UNDOCUMENTED"), + "tps:el_salvador": (312, 2001, "NONE", "UNDOCUMENTED"), + "tps:honduras": (314, 1998, "NONE", "UNDOCUMENTED"), + "tps:nicaragua": (315, 1998, "NONE", "UNDOCUMENTED"), + "tps:nepal": (229, 2015, "NONE", "UNDOCUMENTED"), + "tps:other_designated": (248, 2020, "NONE", "UNDOCUMENTED"), + } + expected_labels = {draw.label for draw in positive_draws} + if set(acs_evidence) != expected_labels: + raise AssertionError( + "Stacked H5 immigration evidence must exactly cover the positive " + "canonical humanitarian draws." + ) + + row_count = 1 + len(positive_draws) + int(include_zero_weight_clone) + ids = np.arange(1, row_count + 1, dtype=np.int64) + channels = np.asarray( + [BASE_ASEC_SUPPORT_CHANNEL] + + [stacked_spine_module.ACS_STACKED_SUPPORT_CHANNEL] * len(positive_draws) + + ([BASE_ASEC_SUPPORT_CHANNEL] if include_zero_weight_clone else []), + dtype=object, + ) + clone_indices = np.zeros(row_count, dtype=np.int64) + source_ids = ids.copy() + if include_zero_weight_clone: + clone_indices[-1] = 1 + source_ids[-1] = ids[0] + + person_rows: list[dict[str, object]] = [ + { + "PRCITSHP": 1, + "PENATVTY": 57, + "PEINUSYR": 0, + "CIT": np.nan, + "POBP": np.nan, + "YOEP": np.nan, + "A_AGE": 50, + "age": 50, + "ssn_card_type": "CITIZEN", + "immigration_status_str": "CITIZEN", + } + ] + for draw in positive_draws: + birth_country, arrival_year, ssn_card_type, immigration_status = acs_evidence[ + draw.label + ] + person_rows.append( + { + "PRCITSHP": np.nan, + "PENATVTY": np.nan, + "PEINUSYR": np.nan, + "CIT": 5, + "POBP": birth_country, + "YOEP": arrival_year, + "A_AGE": np.nan, + "age": 50, + "ssn_card_type": ssn_card_type, + "immigration_status_str": immigration_status, + } + ) + if include_zero_weight_clone: + person_rows.append(dict(person_rows[0])) + + person = pd.DataFrame(person_rows) + person.insert(0, "PERIDNUM", pd.Series([f"person-{value}" for value in ids])) + person.insert(1, "person_id", ids) + for entity in US_SCHEMA.group_entities: + person[US_SCHEMA.membership_column(entity)] = ids + person["nullable_input"] = np.resize( + np.asarray([True, None, False], dtype=object), + row_count, + ) + person["is_incapable_of_self_care"] = True + person["tax_unit_role_input"] = "DEPENDENT" + person[support_channel_column("person")] = channels + person[support_source_id_column("person")] = source_ids + person[support_clone_index_column("person")] = clone_indices + + household_weights = np.asarray( + [1.0] + + [float(draw.target) for draw in positive_draws] + + ([0.0] if include_zero_weight_clone else []), + dtype=np.float64, + ) + mutable = channels == stacked_spine_module.ACS_STACKED_SUPPORT_CHANNEL + reconciled_person, reconciliation = ( + immigration_runtime.reconcile_us_immigration_humanitarian_transfer( + person, + weights=household_weights, + mutable_rows=mutable, + seed=0, + time_period=2024, + controls=controls, + ) + ) + + tables = {"person": reconciled_person} + for entity in US_SCHEMA.group_entities: + table = pd.DataFrame({US_SCHEMA.id_column(entity): ids}) + table.insert( + 0, + f"{entity}_source_label", + pd.Series([f"{entity}-{value}" for value in ids], dtype=object), + ) + table[support_channel_column(entity)] = channels + table[support_source_id_column(entity)] = source_ids + table[support_clone_index_column(entity)] = clone_indices + tables[entity] = table + + household = tables["household"] + household["puma"] = "0600101" + household["congressional_district_geoid"] = np.full( + row_count, + 601, + dtype=np.int64, + ) + household["county_fips"] = "06001" + + for index, spec in enumerate( + post_transfer_calibration_runtime.POST_TRANSFER_CALIBRATION_SPECS.values() + ): + tables[spec.entity][spec.target] = ( + 10.0 + index + np.arange(row_count, dtype=np.float64) + ) + + assembly_tables = { + entity: table.loc[table[support_clone_index_column(entity)].eq(0)] + for entity, table in tables.items() + } + metadata = spine_assembly_manifest( + assembly_tables, + channels=( + BASE_ASEC_SUPPORT_CHANNEL, + stacked_spine_module.ACS_STACKED_SUPPORT_CHANNEL, + ), + ) + frame = Frame( + tables, + US_SCHEMA, + { + "household": Weights( + household_weights, + WeightKind.IMPORTANCE, + ) + }, + metadata=metadata, + ) + return frame, reconciliation + + def _semantic_string_columns(table: pd.DataFrame) -> tuple[str, ...]: return tuple( column @@ -630,24 +792,54 @@ def _write_ready_pool( tmp_path: Path, *, stacked: bool = False, + include_zero_weight_clone: bool = False, sample_fraction: float = 1.0, ) -> Path: run_id = "fixture-publication" pool_path = tmp_path / "pool.h5" diagnostics_path = tmp_path / "pool.agreement.json" manifest_path = tmp_path / "pool.manifest.json" + gate_names = ( + ( + "us_stacked_completeness", + "us_by_origin_battery", + "immigration_composition", + ) + if stacked + else ("us_spine_agreement",) + ) agreement_gate = { "passed": True, "gates": { - "us_spine_agreement": { + name: { "passed": True, "failures": [], "details": {"fixture": True}, } + for name in gate_names }, } + if include_zero_weight_clone and not stacked: + raise ValueError("Only the stacked H5 fixture may include a clone row.") schema_version = US_MULTISPINE_POOL_MANIFEST_SCHEMA_VERSION if stacked else 4 - pool_frame = _pool_frame_with_object_strings_on_every_entity(stacked=stacked) + if stacked: + pool_frame, immigration_reconciliation = ( + _stacked_pool_frame_with_live_immigration( + include_zero_weight_clone=include_zero_weight_clone, + ) + ) + dag = _canonical_stacked_late_dag_receipt( + pool_frame, + immigration_reconciliation=immigration_reconciliation, + ) + provenance_counts = spine_provenance_counts( + pool_frame, + boundary="stacked H5 fixture provenance counts", + ) + else: + pool_frame = _pool_frame_with_object_strings_on_every_entity() + dag = None + provenance_counts = {"household": {"rows": pool_frame.n("household")}} geography_assignment = ( _fixture_geography_assignment(pool_frame.table("household")) if stacked @@ -725,7 +917,7 @@ def _write_ready_pool( }, }, "agreement_gate": agreement_gate, - "provenance_counts": {"household": {"rows": 3}}, + "provenance_counts": provenance_counts, "pool_h5": { "path": str(pool_path.resolve()), "sha256": _sha256(pool_path), @@ -740,7 +932,7 @@ def _write_ready_pool( }, } if stacked: - dag = _canonical_stacked_late_dag_receipt() + assert dag is not None assert geography_assignment is not None sampling, stack_manifest = _fixture_stacked_sampling(sample_fraction) transition_authority = ( @@ -828,28 +1020,44 @@ def _write_gate_failed_pool(tmp_path: Path) -> Path: def _canonical_late_calibration_owner_receipt( + frame: Frame, spec: post_transfer_calibration_runtime.PostTransferCalibrationSpec, ) -> dict[str, object]: - values = np.asarray( - [10.0, 20.0, 30.0, 40.0, 50.0, 100.0, 200.0, 300.0, 400.0, 500.0] - ) - weights = np.asarray([2.0, 3.0, 5.0, 5.0, 5.0, 4.0, 4.0, 4.0, 4.0, 4.0]) - entity_ids = np.arange(1, len(values) + 1) - reference = np.asarray([True] * 5 + [False] * 5) - recipient = ~reference + table = frame.table(spec.entity) + channel = table[support_channel_column(spec.entity)].astype(str) + clone_index = pd.to_numeric( + table[support_clone_index_column(spec.entity)], + errors="raise", + ) + reference = (channel.eq(BASE_ASEC_SUPPORT_CHANNEL) & clone_index.eq(0)).to_numpy( + dtype=bool + ) + recipient = ( + channel.eq(stacked_spine_module.ACS_STACKED_SUPPORT_CHANNEL) & clone_index.eq(0) + ).to_numpy(dtype=bool) + weights = np.asarray( + frame.resolve_weights(spec.entity).values, + dtype=np.float64, + ) + entity_ids = table[frame.schema.entity_id_column(spec.entity)].to_numpy(copy=False) constrained = spec.special_constraint != "none" - result = post_transfer_calibration_runtime.calibrate_post_transfer_values( - values, - weights, - entity_ids, - spec=spec, + application = post_transfer_calibration_runtime.apply_post_transfer_calibration( + frame, + entity=spec.entity, + family=spec.family, + target=spec.target, reference_rows=reference, recipient_rows=recipient, mutable_rows=recipient, allowed_carrier_rows=recipient if constrained else None, addition_candidate_rows=recipient if constrained else None, ) - calibration = result.receipt + result_values = application.frame.table(spec.entity)[spec.target].to_numpy( + dtype=np.float64, + copy=True, + ) + table[spec.target] = result_values + calibration = application.receipt scope = calibration["scope"] constraint: dict[str, object] = {"constraint": spec.special_constraint} if spec.special_constraint == "adult_care_qualifying_one_per_tax_unit": @@ -890,13 +1098,13 @@ def _canonical_late_calibration_owner_receipt( ), "reference_output_values_sha256": ( stacked_spine_module._post_transfer_float64_sha256( - result.values[reference], + result_values[reference], boundary="synthetic reference calibration output", ) ), "recipient_output_values_sha256": ( stacked_spine_module._post_transfer_float64_sha256( - result.values[recipient], + result_values[recipient], boundary="synthetic recipient calibration output", ) ), @@ -922,7 +1130,7 @@ def _canonical_late_calibration_owner_receipt( def _canonical_pregnancy_structural_receipt() -> dict[str, object]: - policy = acs_transfer_module.acs_transfer_execution_contract_identity( + policy = acs_transfer_runtime.acs_transfer_execution_contract_identity( targets=("is_pregnant",), derive_schedule_d=False, )["structural_target_policies"]["is_pregnant"] @@ -950,7 +1158,11 @@ def _canonical_pregnancy_structural_receipt() -> dict[str, object]: } -def _canonical_stacked_late_dag_receipt() -> dict[str, object]: +def _canonical_stacked_late_dag_receipt( + frame: Frame, + *, + immigration_reconciliation: dict[str, object], +) -> dict[str, object]: """Build a signed fixture receipt over the live canonical contracts.""" schedule = stacked_spine_module.CANONICAL_US_LATE_PRODUCER_SCHEDULE @@ -1000,17 +1212,80 @@ def _canonical_stacked_late_dag_receipt() -> dict[str, object]: "sha256" ] ) + immigration_reconciliation = stacked_spine_module._json_ready( + immigration_reconciliation + ) + immigration_recipient_rows = int(immigration_reconciliation["mutable_rows"]) + immigration_producer_rows = int(immigration_reconciliation["immutable_rows"]) group_receipts: dict[str, object] = {} for group in stacked_spine_module.CANONICAL_US_LATE_TRANSFER_GROUPS: + is_immigration_group = ( + group.entity == "person" and group.family == "source_operator_immigration" + ) group_targets = { f"{group.entity}/{group.family}/{target}": { - "authorized_null_rows": 0, - "imputed_rows": 0, + **( + { + "producer_roles": ["asec_source"], + "producer_rows": immigration_producer_rows, + } + if is_immigration_group + else {} + ), + "authorized_null_rows": ( + immigration_recipient_rows if is_immigration_group else 0 + ), + "imputed_rows": ( + immigration_recipient_rows if is_immigration_group else 0 + ), "unmodeled_rows": 0, "residual_null_rows": 0, } for target in group.targets } + if is_immigration_group: + required_predictors, _optional_predictors = ( + stacked_spine_module._acs_pattern_predictor_authority( + entity=group.entity, + family_targets=group.targets, + ) + ) + pattern = acs_transfer_runtime.AcsTransferPattern( + name=acs_transfer_runtime._pattern_name(0, ()), + observed_optional_predictors=(), + predictors=required_predictors, + seed=0, + weight_kind="design", + donor_rows=immigration_producer_rows, + recipient_rows=immigration_recipient_rows, + target_regimes=tuple( + (model_target, "positive_only") + for model_target in acs_transfer_runtime._model_target_names( + group.targets + ) + ), + ) + for target in group.targets: + key = f"{group.entity}/{group.family}/{target}" + record = acs_transfer_runtime.AcsImputedInput( + column=target, + entity=group.entity, + family=group.family, + donor_spine="synthetic_stacked_h5_fixture", + donor_channel="asec", + predictors=pattern.predictors, + seed=pattern.seed, + weight_kind=pattern.weight_kind, + patterns=(pattern,), + imputed_recipient_rows=immigration_recipient_rows, + reconciliation=immigration_reconciliation, + ) + group_targets[key]["qrf_pattern_evidence"] = ( + stacked_spine_module._acs_imputed_pattern_evidence(record) + ) + group_targets[key]["post_transfer_reconciliation"] = dict( + immigration_reconciliation + ) pregnancy_key = f"{group.entity}/{group.family}/is_pregnant" if pregnancy_key in group_targets: group_targets[pregnancy_key]["structural_policy"] = ( @@ -1019,7 +1294,7 @@ def _canonical_stacked_late_dag_receipt() -> dict[str, object]: calibrated_keys = sorted(set(group_targets) & set(late_specs)) for key in calibrated_keys: group_targets[key]["post_transfer_calibration"] = ( - _canonical_late_calibration_owner_receipt(late_specs[key]) + _canonical_late_calibration_owner_receipt(frame, late_specs[key]) ) group_receipts[group.name] = { "producer": group.name, @@ -1463,8 +1738,58 @@ def test_ready_stacked_pool_loader_binds_terminal_gate_aliases( frame, manifest, _ = load_simulation_ready_us_multispine_pool(manifest_path) + positive_draws = tuple( + draw + for draw in immigration_runtime.us_immigration_controls().humanitarian + if draw.target > 0 + ) + expected_rows = 1 + len(positive_draws) + expected_weights = np.asarray( + [1.0, *(float(draw.target) for draw in positive_draws)], + dtype=np.float64, + ) + assert all(frame.n(entity) == expected_rows for entity in US_SCHEMA.entities) + np.testing.assert_array_equal( + frame.weights_for("household").values, + expected_weights, + ) + person = frame.table("person") + assert person.loc[0, "PRCITSHP"] == 1 + assert pd.isna(person.loc[0, "CIT"]) + assert person.loc[0, "immigration_status_str"] == "CITIZEN" + assert person.loc[1:, "CIT"].eq(5).all() + assert person.loc[1:, ["POBP", "YOEP"]].notna().all().all() + for entity in US_SCHEMA.group_entities: + np.testing.assert_array_equal( + person[US_SCHEMA.membership_column(entity)], + frame.table(entity)[US_SCHEMA.id_column(entity)], + ) + for draw in positive_draws: + emitted = immigration_runtime.us_immigration_humanitarian_draw_mask( + frame, + draw, + ) + assert int(emitted.sum()) == 1 + assert float(frame.resolve_weights("person").values[emitted].sum()) == float( + draw.target + ) + for entity in US_SCHEMA.entities: + assert manifest["provenance_counts"][entity] == { + "rows": expected_rows, + "by_source_channel": { + BASE_ASEC_SUPPORT_CHANNEL: 1, + stacked_spine_module.ACS_STACKED_SUPPORT_CHANNEL: len(positive_draws), + }, + "by_clone_index": {"0": expected_rows}, + "by_source_channel_and_clone_index": { + BASE_ASEC_SUPPORT_CHANNEL: {"0": 1}, + stacked_spine_module.ACS_STACKED_SUPPORT_CHANNEL: { + "0": len(positive_draws) + }, + }, + } assert manifest["terminal_gates"] == manifest["agreement_gate"] - assert manifest["schema_version"] == 9 + assert manifest["schema_version"] == 10 assert ( manifest["pool_h5"]["materializer_version"] == US_MULTISPINE_POOL_H5_MATERIALIZER_VERSION @@ -1610,6 +1935,61 @@ def test_ready_stacked_pool_loader_rejects_boolean_sampling_aliases( load_simulation_ready_us_multispine_pool(manifest_path) +def test_ready_stacked_pool_loader_replays_immigration_status_against_h5( + tmp_path: Path, +) -> None: + """An ordinary H5 re-hash cannot forge the sealed live immigration proof.""" + + pytest.importorskip("tables") + manifest_path = _write_ready_pool(tmp_path, stacked=True) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + pool_path = Path(manifest["pool_h5"]["path"]) + with pd.HDFStore(pool_path, mode="a") as store: + person = h5_io.read_frame_table(store, "person") + parole_rows = person["immigration_status_str"].eq("PAROLED_ONE_YEAR") + assert parole_rows.any() + person.loc[parole_rows.idxmax(), "immigration_status_str"] = ( + "LEGAL_PERMANENT_RESIDENT" + ) + h5_io.put_frame_table( + store, + "person", + person, + preferred_format="fixed", + ) + + # Re-sign only the ordinary byte-level artifact envelope. The immutable + # late-transition receipt still describes the untampered live population. + manifest["pool_h5"]["sha256"] = _sha256(pool_path) + manifest["pool_h5"]["size_bytes"] = pool_path.stat().st_size + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + + with pytest.raises( + ValueError, + match="differs from the live ASEC/ACS weighted population", + ): + load_simulation_ready_us_multispine_pool(manifest_path) + + +def test_ready_stacked_pool_loader_requires_immigration_terminal_gate( + tmp_path: Path, +) -> None: + pytest.importorskip("tables") + manifest_path = _write_ready_pool(tmp_path, stacked=True) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + diagnostics_path = Path(manifest["agreement_diagnostics"]["path"]) + diagnostics = json.loads(diagnostics_path.read_text(encoding="utf-8")) + for payload in (manifest, diagnostics): + payload["terminal_gates"]["gates"].pop("immigration_composition") + payload["agreement_gate"]["gates"].pop("immigration_composition") + diagnostics_path.write_text(json.dumps(diagnostics), encoding="utf-8") + manifest["agreement_diagnostics"]["sha256"] = _sha256(diagnostics_path) + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + + with pytest.raises(ValueError, match="canonical terminal gate set"): + load_simulation_ready_us_multispine_pool(manifest_path) + + def test_ready_stacked_pool_loader_binds_h5_cd_vintage_attrs( tmp_path: Path, ) -> None: @@ -1678,7 +2058,11 @@ def test_ready_stacked_pool_loader_rejects_divergent_clone_geography( tmp_path: Path, ) -> None: pytest.importorskip("tables") - manifest_path = _write_ready_pool(tmp_path, stacked=True) + manifest_path = _write_ready_pool( + tmp_path, + stacked=True, + include_zero_weight_clone=True, + ) manifest = json.loads(manifest_path.read_text(encoding="utf-8")) pool_path = Path(manifest["pool_h5"]["path"]) clone_column = support_clone_index_column("household") @@ -1720,7 +2104,11 @@ def test_scoring_pool_loader_authenticates_failed_stacked_terminal_receipt( ) ) - assert frame.n("household") == 3 + expected_rows = 1 + sum( + draw.target > 0 + for draw in immigration_runtime.us_immigration_controls().humanitarian + ) + assert frame.n("household") == expected_rows assert loaded_manifest["status"] == "gate_failed" assert loaded_manifest["simulation_ready"] is False assert loaded_manifest["terminal_gates"]["passed"] is False @@ -1747,7 +2135,7 @@ def test_scoring_pool_loader_authenticates_failed_stacked_terminal_receipt( "failure_count": 1, "failures": [ { - "gate": "us_spine_agreement", + "gate": "us_stacked_completeness", "message": "fixture terminal failure", } ], diff --git a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py index a9049e406a..150ac29260 100644 --- a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py @@ -22,6 +22,7 @@ import pytest import microcosm.build.us_runtime.acs_transfer as acs_transfer_module +import microcosm.build.us_runtime.immigration as immigration_module import microcosm.build.us_runtime.multispine_pool as multispine_pool_module import microcosm.build.us_runtime.post_transfer_calibration as post_transfer_calibration_runtime import microcosm.build.us_runtime.stacked_spine as stacked_spine_module @@ -185,6 +186,86 @@ def _many_household_source_frame( ) +def _with_household_design_weights(frame: Frame, values: np.ndarray) -> Frame: + """Return a source fixture with replacement household design weights.""" + + weights = np.asarray(values, dtype=np.float64) + assert weights.shape == (len(frame.table("household")),) + return Frame( + {entity: frame.table(entity).copy(deep=True) for entity in frame.entities}, + frame.schema, + {"household": Weights(weights, frame.weights_for("household").kind)}, + frame.strata.copy(deep=True), + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +def _humanitarian_weighted_source_frames( + *, + asec_count: int, + acs_count: int, + sample_fraction: float, + sample_seed: int, + state_fips: str | None = None, + puma: str | None = None, +) -> tuple[Frame, Frame]: + """Shape sampled survey weights to the live humanitarian controls. + + The stacked assembly normalizes each sampled arm back to its source mass, + then splits the ASEC anchor mass equally across ASEC and ACS. Setting the + ASEC source mass to twice the humanitarian target total and the selected + ACS weights to the individual targets therefore makes both assembled arms + exact without changing the frozen assembly manifest later in the fixture + pipeline. + """ + + controls = immigration_module.us_immigration_controls() + positive_draws = tuple(draw for draw in controls.humanitarian if draw.target > 0) + target_weights = np.asarray( + [draw.target for draw in positive_draws], + dtype=np.float64, + ) + target_mass = float(target_weights.sum()) + asec = _many_household_source_frame( + count=asec_count, + state_fips=state_fips, + ) + acs = _many_household_source_frame( + count=acs_count, + measured_offset=1_000.0, + state_fips=state_fips, + puma=puma, + ) + sampled_acs, _receipt = stacked_spine_module.sample_acs_households( + acs, + fraction=sample_fraction, + seed=sample_seed, + ) + selected_ids = sampled_acs.table("household")["household_id"].to_numpy( + dtype=np.int64 + ) + assert len(selected_ids) == len(target_weights) + + asec = _with_household_design_weights( + asec, + np.full(asec_count, 2.0 * target_mass / asec_count, dtype=np.float64), + ) + acs_household_ids = acs.table("household")["household_id"].to_numpy(dtype=np.int64) + acs_weights = np.ones(acs_count, dtype=np.float64) + id_to_position = { + int(household_id): position + for position, household_id in enumerate(acs_household_ids) + } + for household_id, target_weight in zip( + selected_ids, + target_weights, + strict=True, + ): + acs_weights[id_to_position[int(household_id)]] = target_weight + return asec, _with_household_design_weights(acs, acs_weights) + + def _replace_person( frame: Frame, person: pd.DataFrame, @@ -1222,6 +1303,283 @@ def _canonical_late_calibration_owner_receipt( return owner +_HUMANITARIAN_ACS_EVIDENCE = { + "paroled_one_year:afghanistan": (200, 2021), + "paroled_one_year:ukraine": (164, 2022), + "paroled_one_year:nicaragua": (315, 2023), + "paroled_one_year:venezuela": (373, 2022), + "refugee": (412, 2022), + "asylee": (207, 2016), + "deportation_withheld": (501, 2015), + "tps:venezuela": (373, 2015), + "tps:el_salvador": (312, 2001), + "tps:honduras": (314, 1998), + "tps:nicaragua": (315, 1998), + "tps:nepal": (229, 2015), + "tps:other_designated": (205, 2015), +} + + +def _with_reconciled_immigration_fixture( + frame: Frame, +) -> tuple[Frame, dict[str, object]]: + """Build and reconcile one live ACS household per positive manifest draw.""" + + controls = immigration_module.us_immigration_controls() + positive_draws = tuple(draw for draw in controls.humanitarian if draw.target > 0) + expected_labels = {draw.label for draw in controls.humanitarian} + assert set(_HUMANITARIAN_ACS_EVIDENCE) == expected_labels + row_count = 1 + len(positive_draws) + ids = np.arange(1, row_count + 1, dtype=np.int64) + expected_channels = {"asec": 1, "acs": len(positive_draws)} + preserve_live_lineage = all( + len(frame.table(entity)) == row_count + and support_channel_column(entity) in frame.table(entity) + and frame.table(entity)[support_channel_column(entity)] + .astype(str) + .value_counts() + .to_dict() + == expected_channels + for entity in frame.entities + ) + + tables: dict[str, pd.DataFrame] = {} + for entity in frame.entities: + source = frame.table(entity) + assert not source.empty + if preserve_live_lineage: + table = source.copy(deep=True) + else: + table = pd.concat([source.iloc[[0]]] * row_count, ignore_index=True) + table[f"{entity}_id"] = ids + table[support_channel_column(entity)] = np.asarray( + ["asec", *(["acs"] * len(positive_draws))], + dtype=object, + ) + table[support_clone_index_column(entity)] = np.zeros( + row_count, + dtype=np.int64, + ) + tables[entity] = table + + person = tables[frame.schema.person_entity] + if not preserve_live_lineage: + for entity in frame.schema.group_entities: + person[f"person_{entity}_id"] = ids + person["person_id"] = ids + person_channels = person[support_channel_column(frame.schema.person_entity)].astype( + str + ) + asec_rows = person_channels.eq("asec").to_numpy(dtype=bool) + acs_rows = person_channels.eq("acs").to_numpy(dtype=bool) + assert int(asec_rows.sum()) == 1 + assert int(acs_rows.sum()) == len(positive_draws) + # Keep legacy TPS arrivals outside the DACA age-at-entry cohort so each + # row remains eligible for the single humanitarian draw it represents. + person["A_AGE"] = np.full(row_count, 60.0) + if "age" in person: + person["age"] = np.full(row_count, 60.0) + if "source_year" in person and not preserve_live_lineage: + person["source_year"] = np.full(row_count, 2024, dtype=np.int64) + if "source_household_id" in person and not preserve_live_lineage: + person["source_household_id"] = ids + if "source_person_id" in person and not preserve_live_lineage: + person["source_person_id"] = ids + if "PERIDNUM" in person and not preserve_live_lineage: + person["PERIDNUM"] = pd.Series(ids.astype(str), dtype="string") + + acs_evidence = [_HUMANITARIAN_ACS_EVIDENCE[draw.label] for draw in positive_draws] + for column in ("PRCITSHP", "PENATVTY", "PEINUSYR", "CIT", "POBP", "YOEP"): + person[column] = np.full(row_count, np.nan, dtype=np.float64) + person.loc[asec_rows, ["PRCITSHP", "PENATVTY", "PEINUSYR"]] = ( + 1.0, + 57.0, + 0.0, + ) + person.loc[acs_rows, "CIT"] = 5.0 + person.loc[acs_rows, "POBP"] = [item[0] for item in acs_evidence] + person.loc[acs_rows, "YOEP"] = [item[1] for item in acs_evidence] + person["ssn_card_type"] = pd.Series( + np.where(asec_rows, "CITIZEN", "OTHER_NON_CITIZEN"), + index=person.index, + dtype="string", + ) + person["immigration_status_str"] = pd.Series( + np.where(asec_rows, "CITIZEN", "LEGAL_PERMANENT_RESIDENT"), + index=person.index, + dtype="string", + ) + + household = tables["household"] + existing_household_weights = pd.Series( + np.asarray(frame.weights_for("household").values, dtype=np.float64), + index=frame.table("household")["household_id"].to_numpy(dtype=np.int64), + ) + asec_household_id = int(person.loc[asec_rows, "person_household_id"].iloc[0]) + household_id_to_weight = { + asec_household_id: ( + float(existing_household_weights.loc[asec_household_id]) + if preserve_live_lineage + else 1.0 + ), + **{ + int(household_id): float(draw.target) + for household_id, draw in zip( + person.loc[acs_rows, "person_household_id"], + positive_draws, + strict=True, + ) + }, + } + household_weights = household["household_id"].map(household_id_to_weight) + assert household_weights.notna().all() + assert tuple(frame.weighted_entities) == ("household",) + weights = { + "household": Weights( + household_weights.to_numpy(dtype=np.float64), + frame.weights_for("household").kind, + ) + } + source_strata = np.asarray(frame.strata, dtype=object) + strata = ( + frame.strata + if preserve_live_lineage + else pd.Series(np.resize(source_strata, row_count), dtype=object) + ) + metadata = dict(frame.metadata) + assembly_key = stacked_spine_module.SPINE_ASSEMBLY_MANIFEST_KEY + assembly = metadata.get(assembly_key) + if not preserve_live_lineage and isinstance(assembly, Mapping): + assembly = copy.deepcopy(dict(assembly)) + declared_channels = tuple(assembly["channels"]) + assembly["native_row_counts"] = { + entity: { + channel: int(table[support_channel_column(entity)].eq(channel).sum()) + for channel in declared_channels + } + for entity, table in tables.items() + } + metadata[assembly_key] = assembly + stacked_key = stacked_spine_module.STACKED_SPINE_MANIFEST_KEY + stacked = metadata.get(stacked_key) + if not preserve_live_lineage and isinstance(stacked, Mapping): + stacked = copy.deepcopy(dict(stacked)) + household_channels = household[support_channel_column("household")].astype(str) + live_weights = household_weights.to_numpy(dtype=np.float64) + live_masses = { + channel: float( + live_weights[household_channels.eq(channel).to_numpy()].sum() + ) + for channel in ("asec", "acs") + } + total_mass = float(live_weights.sum()) + shares = { + channel: live_masses[channel] / total_mass for channel in ("asec", "acs") + } + sample_receipts = { + channel: dict(sample) + for channel, sample in stacked["survey_samples"].items() + } + mass_anchor = str(stacked["mass_anchor_channel"]) + normalized_masses = { + channel: total_mass if channel == mass_anchor else live_masses[channel] + for channel in ("asec", "acs") + } + for channel, sample in sample_receipts.items(): + sampled_mass = float(sample["sampled_household_mass"]) + normalized_mass = normalized_masses[channel] + sample.update( + { + "incoming_household_mass": normalized_mass, + "normalization_factor": normalized_mass / sampled_mass, + "normalized_household_mass": normalized_mass, + } + ) + stacked["survey_samples"] = sample_receipts + stacked["household_mass_shares"] = shares + harmonization = { + channel: dict(arm) + for channel, arm in stacked["weight_harmonization"].items() + } + for channel, arm in harmonization.items(): + incoming_mass = normalized_masses[channel] + allocated_mass = live_masses[channel] + arm.update( + { + "share": shares[channel], + "incoming_mass": incoming_mass, + "allocated_mass": allocated_mass, + "declared_allocation": shares[channel] * total_mass, + "scale_factor": allocated_mass / incoming_mass, + } + ) + stacked["weight_harmonization"] = harmonization + metadata[stacked_key] = stacked + reconciled_frame = Frame( + tables, + frame.schema, + weights, + strata, + mass_log=frame.mass_log, + metadata=metadata, + ) + reconciled_person, receipt = ( + immigration_module.reconcile_us_immigration_humanitarian_transfer( + reconciled_frame.table(frame.schema.person_entity), + weights=np.asarray( + reconciled_frame.resolve_weights(frame.schema.person_entity).values, + dtype=np.float64, + ), + mutable_rows=acs_rows, + seed=0, + ) + ) + return _replace_person(reconciled_frame, reconciled_person), receipt + + +def _immigration_fixture_qrf_evidence( + *, + target: str, + family_targets: tuple[str, ...], + donor_rows: int, + recipient_rows: int, +) -> dict[str, object]: + """Return canonical joint-codec QRF evidence for one paired output leaf.""" + + required_predictors, _optional_predictors = ( + stacked_spine_module._acs_pattern_predictor_authority( + entity="person", + family_targets=family_targets, + ) + ) + pattern = acs_transfer_module.AcsTransferPattern( + name=acs_transfer_module._pattern_name(0, ()), + observed_optional_predictors=(), + predictors=required_predictors, + seed=0, + weight_kind=WeightKind.DESIGN.value, + donor_rows=donor_rows, + recipient_rows=recipient_rows, + target_regimes=tuple( + (model_target, "positive_only") + for model_target in acs_transfer_module._model_target_names(family_targets) + ), + ) + record = acs_transfer_module.AcsImputedInput( + column=target, + entity="person", + family="source_operator_immigration", + donor_spine="synthetic_pool_tool_fixture", + donor_channel="asec", + predictors=required_predictors, + seed=0, + weight_kind=WeightKind.DESIGN.value, + patterns=(pattern,), + imputed_recipient_rows=recipient_rows, + ) + return stacked_spine_module._acs_imputed_pattern_evidence(record) + + def _canonical_pregnancy_structural_receipt() -> dict[str, object]: policy = acs_transfer_module.acs_transfer_execution_contract_identity( targets=("is_pregnant",), @@ -1256,7 +1614,12 @@ def _canonical_late_transfer_receipt( *, authority: Mapping[str, object] | None = None, frame: Frame | None = None, + immigration_reconciliation: Mapping[str, object] | None = None, ) -> dict[str, object]: + if immigration_reconciliation is None: + _fixture_frame, immigration_reconciliation = ( + _with_reconciled_immigration_fixture(_source_frame()) + ) canonical_family = { (entity, target): family for entity, families in ( @@ -1278,12 +1641,36 @@ def _canonical_late_transfer_receipt( ] ) for group in pool_tool.CANONICAL_US_LATE_TRANSFER_GROUPS: + is_immigration_group = ( + group.entity == "person" and group.family == "source_operator_immigration" + ) + immigration_mutable_rows = int(immigration_reconciliation["mutable_rows"]) + immigration_immutable_rows = int(immigration_reconciliation["immutable_rows"]) group_targets = { f"{group.entity}/{group.family}/{target}": { - "authorized_null_rows": 0, - "imputed_rows": 0, + "authorized_null_rows": immigration_mutable_rows + if is_immigration_group + else 0, + "imputed_rows": immigration_mutable_rows if is_immigration_group else 0, "unmodeled_rows": 0, "residual_null_rows": 0, + **( + { + "qrf_pattern_evidence": ( + _immigration_fixture_qrf_evidence( + target=target, + family_targets=group.targets, + donor_rows=immigration_immutable_rows, + recipient_rows=immigration_mutable_rows, + ) + ), + "post_transfer_reconciliation": copy.deepcopy( + immigration_reconciliation + ), + } + if is_immigration_group + else {} + ), } for target in group.targets } @@ -1346,6 +1733,7 @@ def _canonical_late_dag_receipt( authority: Mapping[str, object] | None = None, output_frame_sha256: str = "f" * 64, frame: Frame | None = None, + immigration_reconciliation: Mapping[str, object] | None = None, ) -> dict[str, object]: schedule = stacked_spine_module.CANONICAL_US_LATE_PRODUCER_SCHEDULE schedule_receipt = pool_tool._json_ready( @@ -1389,6 +1777,7 @@ def _canonical_late_dag_receipt( pool_tool, authority=authority, frame=frame, + immigration_reconciliation=immigration_reconciliation, ) input_frame_sha256 = "e" * 64 previous_sha256 = stacked_spine_module._late_execution_genesis_sha256( @@ -1610,8 +1999,9 @@ def _authorized_late_impute_fixture( *, authority: Mapping[str, object] | None = None, ) -> tuple[Frame, dict[str, object], str]: - """Bind one structurally signed synthetic DAG proof to a live fixture frame.""" + """Bind one structurally signed fixture DAG proof to its live frame.""" + frame, immigration_reconciliation = _with_reconciled_immigration_fixture(frame) tables = {entity: frame.table(entity).copy(deep=True) for entity in frame.entities} for entity, table in tables.items(): if support_channel_column(entity) not in table: @@ -1656,6 +2046,7 @@ def _authorized_late_impute_fixture( authority=authority, output_frame_sha256=stacked_spine_module._late_frame_content_sha256(frame), frame=frame, + immigration_reconciliation=immigration_reconciliation, ) authorized, transition_authority_sha256 = ( stacked_spine_module._bind_late_producer_transition_authority(frame, dag) @@ -1686,15 +2077,17 @@ def _install_stacked_entrypoint_stubs( verified = _verified_inputs_fixture(pool_tool, tmp_path / "pins") source_manifest = pool_tool.load_acs_source_manifest() puf_donor = pd.DataFrame({"fixture": np.arange(7)}) + asec_source, acs_source = _humanitarian_weighted_source_frames( + asec_count=100, + acs_count=1_200, + sample_fraction=0.01, + sample_seed=578, + state_fips="06" if real_geography_assignment else None, + puma="0600100" if real_geography_assignment else None, + ) loaded = pool_tool._LoadedInputs( - asec=_many_household_source_frame( - state_fips="06" if real_geography_assignment else None, - ), - acs=_many_household_source_frame( - measured_offset=1_000.0, - state_fips="06" if real_geography_assignment else None, - puma="0600100" if real_geography_assignment else None, - ), + asec=asec_source, + acs=acs_source, acs_rent_donor=pd.DataFrame({"fixture": [1.0]}), puf_donor=puf_donor, asec_raw_stage_checkpoint={"artifact": "fixture-raw-stage"}, @@ -1835,8 +2228,8 @@ def gap_fill(frame: Frame, **kwargs): ) assert counts == { ("person", "strike_benefits"): { - "authorized_null_rows": 1, - "recipient_rows": 1, + "authorized_null_rows": 12, + "recipient_rows": 12, "donor_rows": 1, } } @@ -1947,9 +2340,12 @@ def late_producer_dag(frame: Frame, **kwargs: object): "family": group.family, "ordered_targets": list(group.targets), } + late_base, immigration_reconciliation = _with_reconciled_immigration_fixture( + primary_puf_result.frame + ) late_tables = { - entity: primary_puf_result.frame.table(entity).copy(deep=True) - for entity in primary_puf_result.frame.entities + entity: late_base.table(entity).copy(deep=True) + for entity in late_base.entities } for ( spec @@ -1977,22 +2373,17 @@ def late_producer_dag(frame: Frame, **kwargs: object): index=late_person.index, dtype="string", ) - late_tables.update( - { - name: primary_puf_result.frame.link(name) - for name in primary_puf_result.frame.links - } - ) + late_tables.update({name: late_base.link(name) for name in late_base.links}) late_frame = Frame( late_tables, - primary_puf_result.frame.schema, + late_base.schema, { - entity: primary_puf_result.frame.weights_for(entity) - for entity in primary_puf_result.frame.weighted_entities + entity: late_base.weights_for(entity) + for entity in late_base.weighted_entities }, - primary_puf_result.frame.strata, - mass_log=primary_puf_result.frame.mass_log, - metadata=primary_puf_result.frame.metadata, + late_base.strata, + mass_log=late_base.mass_log, + metadata=late_base.metadata, ) dag_receipt = _canonical_late_dag_receipt( pool_tool, @@ -2001,6 +2392,7 @@ def late_producer_dag(frame: Frame, **kwargs: object): late_frame ), frame=late_frame, + immigration_reconciliation=immigration_reconciliation, ) authorized_frame, transition_authority_sha256 = ( stacked_spine_module._bind_late_producer_transition_authority( @@ -2092,6 +2484,18 @@ def battery( monkeypatch.setattr(pool_tool, "stacked_completeness_gate", completeness) monkeypatch.setattr(pool_tool, "by_origin_battery", battery) + + def immigration(_frame: Frame) -> GateResult: + order.append("immigration") + if terminal == "immigration_red": + return GateResult( + name="fixture_immigration", + passed=False, + failures=("fixture immigration terminal failure",), + ) + return GateResult(name="fixture_immigration", passed=True) + + monkeypatch.setattr(pool_tool, "us_immigration_composition_gate", immigration) real_publish = pool_tool._write_stacked_outputs def publish(*args, **kwargs): @@ -2152,6 +2556,7 @@ def test_stacked_operator_target_requires_preparation_before_activation_authorit [ ("success", 0, "iterating"), ("red", 1, "failed"), + ("immigration_red", 1, "failed"), ("error", None, "failed"), ], ) @@ -2214,6 +2619,7 @@ def test_stacked_tool_entrypoint_fixture_e2e_emits_one_logbook_row_at_every_term "simulate", "completeness", "battery", + "immigration", "publish", ] # Exported rows must never embed host-absolute paths; pytest tmp @@ -2232,6 +2638,28 @@ def test_stacked_tool_entrypoint_fixture_e2e_emits_one_logbook_row_at_every_term manifest = json.loads( (tmp_path / "stacked-pool.manifest.json").read_text(encoding="utf-8") ) + if terminal == "immigration_red": + assert manifest["status"] == "gate_failed" + assert manifest["simulation_ready"] is False + assert manifest["terminal_gates"]["passed"] is False + assert ( + manifest["terminal_gates"]["gates"]["fixture_battery"]["passed"] is True + ) + assert manifest["terminal_gates"]["gates"]["fixture_immigration"] == { + "passed": False, + "failures": ["fixture immigration terminal failure"], + "details": {}, + } + assert row.gate_verdicts["fixture_battery"]["verdict"] == "passed" + assert row.gate_verdicts["fixture_immigration"]["verdict"] == "failed" + immigration_receipt = json.loads( + _receipt_file_from_reference( + row.gate_verdicts["fixture_immigration"]["receipt"] + ).read_text(encoding="utf-8") + ) + assert immigration_receipt["terminal_gates"]["gates"][ + "fixture_immigration" + ]["failures"] == ["fixture immigration terminal failure"] assert manifest["schema_version"] == pool_tool.POOL_MANIFEST_SCHEMA_VERSION assert manifest["pipeline"] == "us-stacked-pool" assert manifest["pool_h5"]["materializer_version"] == ( @@ -2300,13 +2728,14 @@ def test_stacked_tool_entrypoint_fixture_e2e_emits_one_logbook_row_at_every_term "materialize_multispine_agreement_outputs", "stacked_completeness_gate", "by_origin_battery", + "us_immigration_composition_gate", ] assert manifest["sampling"] == { **manifest["sampling"], "sample_fraction": 0.01, "fraction_token": "f001", "sample_seed": 578, - "realized_households": {"asec": 1, "acs": 1}, + "realized_households": {"asec": 1, "acs": 12}, } @@ -2499,7 +2928,7 @@ def capture_equality(expected: object, actual: object) -> None: "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "a6bc79878eb6f64637b9f3eceeea6cc2b050c0e5b8f9aca446179258940c44f2", + "spec_sha256": "ea5cc4f31ac7cd67a0d01bccf9e1454755c879f045a813741c84d17cbddd796d", }, } @@ -2702,7 +3131,7 @@ def test_constants_adapter_fixture_checkpoints_are_byte_identical_and_only_recei "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "5378bb9189aec96f50da22aac71e5bd2c3d919e9795f6ef2147e0bc9c739dd8e", + "spec_sha256": "ea202eebdfe30a68323af5cdce6abcc16b94c9f3f18f44ae5369fe7c1d9a5702", } def run_fixture(root: Path, *, config_authority: str) -> dict[str, object]: @@ -3246,6 +3675,7 @@ def fail_publication(*_args, **_kwargs) -> None: assert set(row.gate_verdicts) == { "fixture_completeness", "fixture_battery", + "fixture_immigration", "pipeline_error", } terminal_path = _receipt_file_from_reference( @@ -3518,7 +3948,7 @@ def test_legacy_checkpoint_identity_excludes_stacked_late_producer_schedule( assert changed == current -def test_stacked_checkpoint_identity_binds_v12_semantic_contracts( +def test_stacked_checkpoint_identity_binds_v13_semantic_contracts( pool_tool: ModuleType, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, @@ -3551,7 +3981,7 @@ def identity() -> dict[str, object]: current = identity() pool_code = current["pool_code"] - assert current["materializer_version"] == 12 + assert current["materializer_version"] == 13 assert current["stacked_authority"]["version"] == 11 assert current["geography_assignment"] == ( pool_tool._stacked_geography_assignment_contract() @@ -3574,6 +4004,7 @@ def identity() -> dict[str, object]: "materialize_multispine_agreement_outputs", "stacked_completeness_gate", "by_origin_battery", + "us_immigration_composition_gate", ] assert pool_code["late_producer_schedule"] == pool_tool._json_ready( pool_tool.us_late_producer_schedule_receipt() @@ -3779,7 +4210,7 @@ def changed_source_stage_binding( ) ) - assert current["materializer_version"] == stale_qrf["materializer_version"] == 12 + assert current["materializer_version"] == stale_qrf["materializer_version"] == 13 assert stale_qrf["pool_code"]["primary_qrf_checkpoint_schema_version"] == 5 assert ( pool_tool._discover_stacked_checkpoint_identity( @@ -3827,7 +4258,7 @@ def changed_source_stage_binding( assert "checkpoint base identity is stale" in capsys.readouterr().out -def test_pool_envelope_v7_preserves_stacked_bank_identity_but_rejects_v6( +def test_pool_envelope_v8_preserves_stacked_bank_identity_but_rejects_v7( pool_tool: ModuleType, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, @@ -3860,7 +4291,7 @@ def identity() -> dict[str, object]: legacy.setattr( pool_tool, "POOL_STAGE_CHECKPOINT_MATERIALIZER_VERSION", - 6, + 7, ) assert identity() == current_identity legacy_store = pool_tool._PoolStageCheckpointStore( @@ -3881,7 +4312,7 @@ def identity() -> dict[str, object]: ) capsys.readouterr() - assert pool_tool.POOL_STAGE_CHECKPOINT_MATERIALIZER_VERSION == 7 + assert pool_tool.POOL_STAGE_CHECKPOINT_MATERIALIZER_VERSION == 8 assert identity() == current_identity current_store = pool_tool._PoolStageCheckpointStore( checkpoint_root, @@ -4034,7 +4465,7 @@ def test_qbi_receipt_route_resolution_rejects_wrong_or_ambiguous_paths( ) -@pytest.mark.parametrize("legacy_version", (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11)) +@pytest.mark.parametrize("legacy_version", (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12)) def test_legacy_stacked_materializer_checkpoint_is_not_discovered( pool_tool: ModuleType, monkeypatch: pytest.MonkeyPatch, @@ -4088,7 +4519,7 @@ def test_legacy_stacked_materializer_checkpoint_is_not_discovered( ) ) - assert pool_tool._STACKED_CHECKPOINT_MATERIALIZER_VERSION == 12 + assert pool_tool._STACKED_CHECKPOINT_MATERIALIZER_VERSION == 13 assert ( pool_tool._discover_stacked_checkpoint_identity( checkpoint_root, @@ -4107,9 +4538,15 @@ def test_stacked_resume_rejects_noncanonical_post_puf_transfer_receipt( pool_tool: ModuleType, tmp_path: Path, ) -> None: + asec_source, acs_source = _humanitarian_weighted_source_frames( + asec_count=10, + acs_count=120, + sample_fraction=0.10, + sample_seed=578, + ) stack = pool_tool.assemble_stacked_spine( - _many_household_source_frame(), - _many_household_source_frame(measured_offset=1_000.0), + asec_source, + acs_source, sample_fraction=0.10, sample_seed=578, ) @@ -4230,12 +4667,14 @@ def test_stacked_entrypoint_resumes_each_checkpoint_boundary( "simulate", "completeness", "battery", + "immigration", "publish", ] assert simulated_resume_order == [ "build_stacked_pool", "completeness", "battery", + "immigration", "publish", ] assert transferred_resume_order == [ @@ -4246,6 +4685,7 @@ def test_stacked_entrypoint_resumes_each_checkpoint_boundary( "simulate", "completeness", "battery", + "immigration", "publish", ] assert assembled_resume_order == [ @@ -4260,6 +4700,7 @@ def test_stacked_entrypoint_resumes_each_checkpoint_boundary( "simulate", "completeness", "battery", + "immigration", "publish", ] final_rows = [ @@ -4492,8 +4933,8 @@ def deterministic_fixture_h5( outputs = pool_tool._output_paths(output, checkpoint_root=checkpoint_root) manifest = pool_tool._read_json_object(outputs.manifest) diagnostics = pool_tool._read_json_object(outputs.agreement_diagnostics) - assert pool_tool.POOL_MANIFEST_SCHEMA_VERSION == 9 - assert pool_tool.POOL_STAGE_CHECKPOINT_MATERIALIZER_VERSION == 7 + assert pool_tool.POOL_MANIFEST_SCHEMA_VERSION == 10 + assert pool_tool.POOL_STAGE_CHECKPOINT_MATERIALIZER_VERSION == 8 assert manifest["schema_version"] == 4 assert diagnostics["schema_version"] == 4 assert "materializer_version" not in manifest["pool_h5"] @@ -5802,7 +6243,7 @@ def test_pool_checkpoint_store_round_trips_nullable_boolean_families( manifest = pool_tool._read_json_object( cold_store.checkpoint_manifest_path(stage) ) - assert manifest["materializer_version"] == 7 + assert manifest["materializer_version"] == 8 loaded = pool_tool.load_frame_checkpoint(path).frame if stage == "assembled": assert "fixture_declared_boolean" not in loaded.person @@ -5823,11 +6264,11 @@ def test_pool_checkpoint_store_round_trips_nullable_boolean_families( assert resumed.frame.person["fixture_declared_boolean"].isna().sum() == 1 -def test_simulated_v7_checkpoint_accepts_both_string_encodings_without_rewrite( +def test_simulated_v8_checkpoint_accepts_both_string_encodings_without_rewrite( pool_tool: ModuleType, tmp_path: Path, ) -> None: - """V7 authenticates both physical string encodings as one logical frame.""" + """V8 authenticates both physical string encodings as one logical frame.""" pytest.importorskip("h5py") checkpoint_root = tmp_path / "checkpoints" @@ -5839,7 +6280,7 @@ def test_simulated_v7_checkpoint_accepts_both_string_encodings_without_rewrite( loaded = pool_tool.load_frame_checkpoint(checkpoint_path) canonical_v2_bytes = checkpoint_path.read_bytes() canonical_identity = loaded.metadata["identity"] - assert loaded.metadata["materializer_version"] == 7 + assert loaded.metadata["materializer_version"] == 8 assert any( column["dtype"] == str(CANONICAL_STRING_DTYPE) for columns in loaded.metadata["frame_schema"]["entities"].values() @@ -5870,7 +6311,7 @@ def test_simulated_v7_checkpoint_accepts_both_string_encodings_without_rewrite( banked_v2_bytes = checkpoint_path.read_bytes() assert banked_v2_bytes != canonical_v2_bytes assert legacy_metadata["identity"] == canonical_identity - assert legacy_metadata["materializer_version"] == 7 + assert legacy_metadata["materializer_version"] == 8 assert any( column["dtype"] == "object" for columns in legacy_metadata["frame_schema"]["entities"].values() @@ -6245,7 +6686,7 @@ def test_tail_support_contract_identity_mutation_rebuilds_pool_checkpoints( assert changed_store.load_deepest() is None -@pytest.mark.parametrize("legacy_version", (1, 2, 3, 4, 5, 6)) +@pytest.mark.parametrize("legacy_version", (1, 2, 3, 4, 5, 6, 7)) def test_legacy_pool_materializer_artifacts_fail_closed_with_named_receipts( pool_tool: ModuleType, monkeypatch: pytest.MonkeyPatch, @@ -6278,9 +6719,9 @@ def test_legacy_pool_materializer_artifacts_fail_closed_with_named_receipts( assert manifest["identity"]["materializer_version"] == legacy_version capsys.readouterr() - assert pool_tool.POOL_STAGE_CHECKPOINT_MATERIALIZER_VERSION == 7 + assert pool_tool.POOL_STAGE_CHECKPOINT_MATERIALIZER_VERSION == 8 current_store = _checkpoint_fixture_store(pool_tool, checkpoint_root) - assert current_store.base_identity["materializer_version"] == 7 + assert current_store.base_identity["materializer_version"] == 8 assert current_store.load_deepest() is None output = capsys.readouterr().out From 8fc4c7973ffd92ef1e4f7da1dac5ae86024f849f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 4 Sep 2026 08:28:41 -0400 Subject: [PATCH 15/49] Give the failed-gate pool fixture the canonical terminal gate set The branch's scoring loader authenticates the canonical stacked terminal gates, so main's minimal fixture gate is refused; the helper keeps its API and adopts the branch's construction (real gates, completeness flipped). The denied-pool test now derives the fixture's household count from the positive canonical humanitarian draws instead of main's three-row literal. Refs #767 Co-Authored-By: Claude Fable 5.1 --- .../tests/test_us_multispine_pool_h5_io.py | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py b/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py index 3096322a57..57d24df435 100644 --- a/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py +++ b/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py @@ -988,15 +988,14 @@ def _write_gate_failed_pool(tmp_path: Path) -> Path: manifest = json.loads(manifest_path.read_text(encoding="utf-8")) diagnostics_path = Path(manifest["agreement_diagnostics"]["path"]) diagnostics = json.loads(diagnostics_path.read_text(encoding="utf-8")) - failed_gate = { + # The scoring loader authenticates the canonical terminal gate set, so the + # failed fixture must carry the real gates with one of them flipped. + failed_gate = json.loads(json.dumps(manifest["terminal_gates"])) + failed_gate["passed"] = False + failed_gate["gates"]["us_stacked_completeness"] = { "passed": False, - "gates": { - "us_spine_agreement": { - "passed": False, - "failures": ["fixture terminal failure"], - "details": {"fixture": False}, - } - }, + "failures": ["fixture terminal failure"], + "details": {"fixture": False}, } manifest.update( { @@ -2172,7 +2171,14 @@ def test_denied_gate_failed_pool_is_available_only_for_scoring( frame, loaded_manifest, _ = load_authenticated_us_multispine_pool_for_scoring( manifest_path ) - assert frame.n("household") == 3 + # The stacked fixture carries one base row plus one row per positive + # canonical humanitarian draw (see _stacked_pool_frame_with_live_immigration). + positive_draws = [ + draw + for draw in immigration_runtime.us_immigration_controls().humanitarian + if draw.target > 0 + ] + assert frame.n("household") == 1 + len(positive_draws) assert loaded_manifest["status"] == "gate_failed" refused_loaders = ( From fdfa6a20583f78725c348b2bb3888a934ef86810 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 4 Sep 2026 08:39:12 -0400 Subject: [PATCH 16/49] Pin the Armenian bundle digest alongside BE/UK on the merged tree Main added the am country bundle this week; the envelope digests move with the shared seed protocol, so all three pins are recomputed by country rather than by position. Refs #767 Co-Authored-By: Claude Fable 5.1 --- .../tests/test_spec_engine_country_bundles.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index 5b7f763136..ac30c9ed6a 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -25,7 +25,7 @@ "spine", "vintages", } -AM_SPEC_SHA256 = "6bb0cff6a0ac9d077573126f8f4c13868534e5135dec328917e253dd58f1e089" +AM_SPEC_SHA256 = "4699b27f6604c7ab3f9e2f82a6e74dd4ea89094dc84829d90af91473572cc07a" @pytest.mark.parametrize( @@ -45,7 +45,7 @@ ), ( "be", - "ecfaa9088326d08db364fca3c76c5016706a8b7ca418098a65a1e32b5b896a48", + "6bb0cff6a0ac9d077573126f8f4c13868534e5135dec328917e253dd58f1e089", { "household.household_id", "person.person_id", @@ -55,7 +55,7 @@ ), ( "uk", - "d732d3aa599a5c911f1900419142ddff6a1771b6b95c38e06b4c0e513619ead6", + "ecfaa9088326d08db364fca3c76c5016706a8b7ca418098a65a1e32b5b896a48", { "benunit.benunit_id", "household.household_id", From b8152de78f0d9d2417a31b6773077e2f50d367a0 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 18 Sep 2026 09:22:19 -0400 Subject: [PATCH 17/49] Re-pin the merged tree's attested identities for the third main sync The 552-commit merge combines this branch's humanitarian draws with main's portable QRF worker identity: shared version constants land as authority 12 / outer materializer 13 / pool-stage materializer 8 / manifest schema 10 with both bindings documented; the transition- authority validation now threads worker authentication and replays the branch's post-PUF transfer evidence; the h5_io fixture keeps the per-frame immigration receipt (main's env-keyed cache cannot key it). Every closed-world census is recomputed on the merged bundle: 42,540 configuration fields, seven pointer claims, the inventory digest vector, coverage evidence, loader golden vector, source_stages attestation, US spec digest, and the schedule/payload receipt shas the ordering doc quotes. Refs #767 Co-Authored-By: Claude Fable 5 --- docs/evidence/spec-engine/us-f0-coverage.json | 135 ++++++++++-------- docs/us-multispine-operator-ordering.md | 4 +- .../build/spec_engine/field_usage.py | 34 ++--- .../build/spec_engine/inventory_coverage.py | 51 ++++--- .../src/microcosm/build/us/spec/sources.yaml | 107 +++++++++++--- .../tests/test_spec_engine_coverage_tool.py | 16 +-- .../tests/test_spec_engine_field_usage.py | 16 +-- .../tests/test_spec_engine_legacy_adapter.py | 2 +- .../tests/test_spec_engine_loader.py | 2 +- ...spec_engine_stacked_authority_semantics.py | 2 +- .../tests/test_us_bundle_core_contracts.py | 2 +- .../tests/test_us_multispine_pool_tool.py | 2 +- .../tests/test_us_spec_bundle.py | 4 +- tools/generate_us_bundle_from_constants.py | 2 +- tools/spec_engine_coverage.py | 2 +- 15 files changed, 238 insertions(+), 143 deletions(-) diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index db6d790c71..a1edaeb857 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -4,9 +4,9 @@ "version": 1 }, "country": "us", - "documentation_sha256": "177ac92f6c70d9eca47fbce8642ecf0d8d80eb9a02745824bfa91fc39089d102", + "documentation_sha256": "5beb9497d74b7407fbb3fff888007d089423122db504171ce5e4b337a79e3dcd", "field_usage": { - "authored_normative_field_count": 32384, + "authored_normative_field_count": 32521, "claim_count": 49, "claims": [ { @@ -220,8 +220,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 97, - "pointer_sha256": "cabaaa3d96c534f5ec37f51811c3f549d0ed20791e2892c868888c0da6d7d80e", + "pointer_count": 115, + "pointer_sha256": "545f626e7b9c6aed25a5ae8dd3156c96db2cd35e5637ec1ebe443432d1684277", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/spec~1imputation.yaml/transfer_execution", @@ -274,8 +274,8 @@ ], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 24488, - "pointer_sha256": "6382a3ed07016414c31462b8bedb24590d92b792405a89c927d85fa1bd2a8347", + "pointer_count": 24554, + "pointer_sha256": "18140dbc554c2a17a673fec5ae5839f504d4b9980c5d8f1c1ccb545342977cb4", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/spec~1imputation.yaml/producer_graph", @@ -514,8 +514,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 1702, - "pointer_sha256": "be3e24b53955b9dabccfc7e779212b1e97daf0c0992be3c183e391bf695ed4ee", + "pointer_count": 1715, + "pointer_sha256": "78227f34c3f4ef946c06693dec9177dd906069f98016854bfd4d39c4cd4c51fa", "rationale": null, "relative_sink_prefix": "/source_manifest/stages", "source_prefix": "/authored/spec~1sources.yaml/stages", @@ -530,8 +530,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 88, - "pointer_sha256": "ae011455154bae0df3913ca9a056058a909d913439b292cb46c11f63c7d0d9a3", + "pointer_count": 89, + "pointer_sha256": "8d281f8f1e2f9914684150b0424ff816d0ec09ed0d9053ea5a64e877a86cc35b", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/spec~1spine.yaml/pipeline_contract", @@ -544,8 +544,8 @@ "legacy_sinks": [], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 277, - "pointer_sha256": "dfa7ae701f62f7b1f06954798d29cfba3fc53a0be21e1751646d1f5bafd6071f", + "pointer_count": 316, + "pointer_sha256": "bb848f38e068858ac23e964a4ffbbf5542253da50515c72fe7753d8c56912486", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/spec~1spine.yaml/seed_site_bindings", @@ -736,8 +736,8 @@ "legacy_sinks": [], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 826, - "pointer_sha256": "7ff2d5d1c2fd8026d17a57244f969dc0e9625a9e47304b693ad15df329041282", + "pointer_count": 1034, + "pointer_sha256": "079a78782c7ca36c89160eccb36a17227222c892261d7910ea55b6e97ccc5b9e", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/resolved/seed_protocol", @@ -750,8 +750,8 @@ "legacy_sinks": [], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 277, - "pointer_sha256": "368fc19a8f07c4abf38ebcf2fbc4414d33c403c0f1f9881c2d2e3f5d6160feb6", + "pointer_count": 316, + "pointer_sha256": "199ec98045871070860164b522892327ab840b48c0c6a53972ebc0ff0d7d091f", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/resolved/seed_site_bindings", @@ -772,21 +772,21 @@ "verifier": "vintages" } ], - "configuration_field_count": 42156, - "consumed_field_count": 42156, + "configuration_field_count": 42540, + "consumed_field_count": 42540, "generation0_effect_counts": { - "legacy_behavior": 38476, - "no_generation0_effect": 3680 + "legacy_behavior": 38574, + "no_generation0_effect": 3966 }, "mode_counts": { - "compiler_semantic": 27717, + "compiler_semantic": 28069, "front_end_validation": 348, "identity_only": 103, - "legacy_behavior": 13988 + "legacy_behavior": 14020 }, "multiple_primary_use_field_count": 0, - "pointer_inventory_sha256": "2c0423a08dc16bf5f142134804a1a6545f887e32ca5c991c856a2f8bcfaea0f0", - "resolved_binding_field_count": 9772, + "pointer_inventory_sha256": "370e1958448fed48a635701439905d2145c3ae5bf3878adc1406a56a381a303b", + "resolved_binding_field_count": 10019, "unused_field_count": 0 }, "inventory_coverage": { @@ -810,14 +810,14 @@ "primary_targets": 65, "producer_authored_outputs": 92, "producer_compiled_outputs": 227, - "producer_inputs": 2744, + "producer_inputs": 2750, "producer_nodes": 38, "producer_virtual_resources": 75, "release_rungs": 5, "resolved_references": 334, - "seed_owner_bindings": 112, + "seed_owner_bindings": 125, "seed_owner_rows": 54, - "seed_sites": 53, + "seed_sites": 66, "seed_streams": 14, "source_operators": 16, "source_stages": 37, @@ -1074,11 +1074,11 @@ "legacy_adapter.stacked_checkpoint_static_components" ], "expected": { - "schedule_sha256": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f" + "schedule_sha256": "88bc9243a3518982ae951c3de21bd55877e296ce4fcb183b9bee420d3a684b10" }, "failures": [], "observed": { - "schedule_sha256": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f" + "schedule_sha256": "88bc9243a3518982ae951c3de21bd55877e296ce4fcb183b9bee420d3a684b10" }, "status": "covered" }, @@ -1250,14 +1250,14 @@ "expected": { "edges": 71, "nodes": 38, - "schedule_sha256": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", + "schedule_sha256": "88bc9243a3518982ae951c3de21bd55877e296ce4fcb183b9bee420d3a684b10", "waves": 6 }, "failures": [], "observed": { "edges": 71, "nodes": 38, - "schedule_sha256": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", + "schedule_sha256": "88bc9243a3518982ae951c3de21bd55877e296ce4fcb183b9bee420d3a684b10", "waves": 6 }, "status": "covered" @@ -1275,11 +1275,11 @@ ], "expected": { "relation": "source rows preserved exactly", - "rows": 2744 + "rows": 2750 }, "failures": [], "observed": { - "rows": 2744 + "rows": 2750 }, "status": "covered" }, @@ -1321,7 +1321,7 @@ "expected": "complete execution-row and transition-authority object", "failures": [], "observed": { - "sha256": "9a09b49855b3dc4c37ac86c3b91293b6944a50f4448354701b18db8f659ae121" + "sha256": "e7f16e205ae8c7604d5b021f046724441533222c89f42ab7765620e9b8a36091" }, "status": "covered" }, @@ -1338,12 +1338,12 @@ ], "expected": { "nodes": 38, - "sha256": "40cd51ffdfe2e9d9d08d48c08e8ded9de1e4b134783bab05c4abc6ad5c72ca1e" + "sha256": "2c71f35d54330f13c96783e0252ed9354ff3b4c0edd7e30205886c87ab7ec078" }, "failures": [], "observed": { "nodes": 38, - "sha256": "40cd51ffdfe2e9d9d08d48c08e8ded9de1e4b134783bab05c4abc6ad5c72ca1e" + "sha256": "2c71f35d54330f13c96783e0252ed9354ff3b4c0edd7e30205886c87ab7ec078" }, "status": "covered" }, @@ -1361,12 +1361,12 @@ ], "expected": { "producer_count": 38, - "sha256": "895ffd419de464a761669a5c3bf68db66e3d1a273c47631de80778a6c7f73b57" + "sha256": "fda05279f38b371b38faa44b87d3a64aa34f1a4cc7f93bec7e059ca6418176ff" }, "failures": [], "observed": { "producer_count": 38, - "sha256": "895ffd419de464a761669a5c3bf68db66e3d1a273c47631de80778a6c7f73b57" + "sha256": "fda05279f38b371b38faa44b87d3a64aa34f1a4cc7f93bec7e059ca6418176ff" }, "status": "covered" }, @@ -1533,7 +1533,7 @@ "expected": { "groups": 12, "partition": "disjoint and exhaustive", - "sites": 53 + "sites": 66 }, "failures": [], "observed": { @@ -1550,6 +1550,19 @@ "pregnancy_assignment", "wic_claim_assignment", "snap_discretionary_exemption_assignment", + "immigration_humanitarian_paroled_one_year_afghanistan_assignment", + "immigration_humanitarian_paroled_one_year_ukraine_assignment", + "immigration_humanitarian_paroled_one_year_nicaragua_assignment", + "immigration_humanitarian_paroled_one_year_venezuela_assignment", + "immigration_humanitarian_refugee_assignment", + "immigration_humanitarian_asylee_assignment", + "immigration_humanitarian_deportation_withheld_assignment", + "immigration_humanitarian_tps_venezuela_assignment", + "immigration_humanitarian_tps_el_salvador_assignment", + "immigration_humanitarian_tps_honduras_assignment", + "immigration_humanitarian_tps_nicaragua_assignment", + "immigration_humanitarian_tps_nepal_assignment", + "immigration_humanitarian_tps_other_designated_assignment", "immigration_ead_workers_assignment", "immigration_ead_students_assignment", "ssi_take_up_assignment", @@ -1616,7 +1629,7 @@ "sipp_tip_training_cap" ] }, - "sites": 53 + "sites": 66 }, "status": "covered" }, @@ -1656,13 +1669,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "86cecb19281446d2b505fb975c822c60a604e18f44baded5dc9d62d508a8197e", - "protocol_sha256": "55d6b907dd44613a5498dd7156f1ce9e78758835255bc7c336b4c327bb40e9bf" + "map_sha256": "674f1b222d31c925a9914791bb23fcfcfb6039c56261ae47a13cf735b29de393", + "protocol_sha256": "7883cc956c11fe43fb0dedbe0bedb02b9ad804696f5551ee5d2c7a250b0abdbe" }, "failures": [], "observed": { - "map_sha256": "86cecb19281446d2b505fb975c822c60a604e18f44baded5dc9d62d508a8197e", - "protocol_sha256": "55d6b907dd44613a5498dd7156f1ce9e78758835255bc7c336b4c327bb40e9bf" + "map_sha256": "674f1b222d31c925a9914791bb23fcfcfb6039c56261ae47a13cf735b29de393", + "protocol_sha256": "7883cc956c11fe43fb0dedbe0bedb02b9ad804696f5551ee5d2c7a250b0abdbe" }, "status": "covered" }, @@ -1677,7 +1690,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "55d6b907dd44613a5498dd7156f1ce9e78758835255bc7c336b4c327bb40e9bf", + "implementation_sha256": "7883cc956c11fe43fb0dedbe0bedb02b9ad804696f5551ee5d2c7a250b0abdbe", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1698,7 +1711,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "55d6b907dd44613a5498dd7156f1ce9e78758835255bc7c336b4c327bb40e9bf", + "implementation_sha256": "7883cc956c11fe43fb0dedbe0bedb02b9ad804696f5551ee5d2c7a250b0abdbe", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1732,12 +1745,12 @@ ], "expected": { "relation": "all site fields preserved exactly", - "sites": 53 + "sites": 66 }, "failures": [], "observed": { - "sha256": "a0366cd518f42a240b589af3ef6580aac44ad7ed70969bcc669b365d315146ab", - "sites": 53 + "sha256": "5c22dfb6f3cd7c23c527e054ba6f5040ca2eb4235df8659e3f7c61ebe2ae6c4f", + "sites": 66 }, "status": "covered" }, @@ -1752,12 +1765,12 @@ "compiler_ir.seed_stream_map.sites.owners" ], "expected": { - "bindings": 112, - "coverage": "all 53 sites" + "bindings": 125, + "coverage": "all 66 sites" }, "failures": [], "observed": { - "bindings": 112 + "bindings": 125 }, "status": "covered" }, @@ -1775,12 +1788,12 @@ "legacy_adapter.imputation.late_producer_resource_semantics" ], "expected": { - "sha256": "d6849714bb9d6f64c000193a871c324b6b82850100db6cb3196f5f6e417dc494", + "sha256": "9df42433ba92c5cad75f1df2394f8b64fb25ed027b5a5e6ad383874345112afa", "stages": 37 }, "failures": [], "observed": { - "sha256": "d6849714bb9d6f64c000193a871c324b6b82850100db6cb3196f5f6e417dc494", + "sha256": "9df42433ba92c5cad75f1df2394f8b64fb25ed027b5a5e6ad383874345112afa", "stages": 37 }, "status": "covered" @@ -1840,11 +1853,11 @@ "legacy_adapter.stacked_checkpoint_static_components" ], "expected": { - "sha256": "9d4a9672a0f03039b1fe874b9fe21ed575be0d29f14afc396d03cdf5c809bdd2" + "sha256": "5730bd1c7410a59baa5a85ced86d9dcf0f901b3fa1c56617953199ad1a180eaa" }, "failures": [], "observed": { - "sha256": "9d4a9672a0f03039b1fe874b9fe21ed575be0d29f14afc396d03cdf5c809bdd2" + "sha256": "5730bd1c7410a59baa5a85ced86d9dcf0f901b3fa1c56617953199ad1a180eaa" }, "status": "covered" }, @@ -1887,7 +1900,7 @@ "alpha", "zeta" ], - "sha256": "c66844ca246fb0277e648691a91fe7bea2a21c26980b398017af90f3128521c5" + "sha256": "9c2af34c0779efe4cb7fc7d25c37e5b318e7d886dda4cedcadc5acda0cf9b010" }, "failures": [], "observed": { @@ -1911,7 +1924,7 @@ "alpha", "zeta" ], - "sha256": "c66844ca246fb0277e648691a91fe7bea2a21c26980b398017af90f3128521c5" + "sha256": "9c2af34c0779efe4cb7fc7d25c37e5b318e7d886dda4cedcadc5acda0cf9b010" }, "status": "covered" }, @@ -1975,7 +1988,7 @@ "take_up_contract", "us_qbi_reconciliation_contract" ], - "sha256": "7fc5e7d72ba465cae5b7cc80f042c1b82c756b0a74c1ba4ebc1ed3d99e88d10d" + "sha256": "cabf628f4d2ee3cdd05a7a843da0ba39e6873d0761e7c983e3c2c4c917b46178" }, "status": "covered" }, @@ -2599,7 +2612,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "e058159f89a01b6541e98ee57ffb0169eeff8ed2ea4fb1c664b3f3fb0c60a675" + "spec_sha256": "7f1d3106448c57baf7205ee4c024cab986de0ed7faef7f98f296d7c79c792587" } }, "report_schema_version": 3, @@ -2609,7 +2622,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "e058159f89a01b6541e98ee57ffb0169eeff8ed2ea4fb1c664b3f3fb0c60a675" + "spec_sha256": "7f1d3106448c57baf7205ee4c024cab986de0ed7faef7f98f296d7c79c792587" }, "status": "pass" } diff --git a/docs/us-multispine-operator-ordering.md b/docs/us-multispine-operator-ordering.md index 37bfbdeea1..810086769c 100644 --- a/docs/us-multispine-operator-ordering.md +++ b/docs/us-multispine-operator-ordering.md @@ -943,9 +943,9 @@ Registry schema version 17 and execution-receipt contract version 4 bind the canonical input declarations, outputs, edges, waves, exact kind-specific virtual-resource bindings, content-hashed execution-row schema, and immutable transition authority version 2. The schedule SHA-256 is -`SCHEDULE_SHA_TODO`; +`88bc9243a3518982ae951c3de21bd55877e296ce4fcb183b9bee420d3a684b10`; the full payload SHA-256 is -`PAYLOAD_SHA_TODO`. +`d35ebb433f16e976c0d6e2210c99a36ca042a52c422c0968128678932d999735`. Reversing registry iteration produces those same bytes. The virtual-resource payload ledger is independently versioned: ACS-universe diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py index 107265aebf..499e4d8403 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py @@ -26,9 +26,9 @@ ) from .schemas import load_schema_registry -EXPECTED_AUTHORED_FIELD_COUNT = 32_384 -EXPECTED_RESOLVED_BINDING_FIELD_COUNT = 9_772 -EXPECTED_CONFIGURATION_FIELD_COUNT = 42_156 +EXPECTED_AUTHORED_FIELD_COUNT = 32_521 +EXPECTED_RESOLVED_BINDING_FIELD_COUNT = 10_019 +EXPECTED_CONFIGURATION_FIELD_COUNT = 42_540 class FieldUsageError(AssertionError): @@ -421,12 +421,12 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "e1dd7dc5123ab0f39d08ea4939d98dd09a6fdb8e7449a7ca3125fb1ddbd5b4e9", ), "imputation_producer_graph": ( - 24_488, - "6382a3ed07016414c31462b8bedb24590d92b792405a89c927d85fa1bd2a8347", + 24_554, + "18140dbc554c2a17a673fec5ae5839f504d4b9980c5d8f1c1ccb545342977cb4", ), "imputation_transfer_execution": ( - 97, - "cabaaa3d96c534f5ec37f51811c3f549d0ed20791e2892c868888c0da6d7d80e", + 115, + "545f626e7b9c6aed25a5ae8dd3156c96db2cd35e5637ec1ebe443432d1684277", ), "imputation_waiver_records": ( 70, @@ -453,12 +453,12 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "6a781915fd491d2c4b16d2b7d482f69cf362c904130093c59f9629f7a319269b", ), "resolved_seed_protocol": ( - 826, - "7ff2d5d1c2fd8026d17a57244f969dc0e9625a9e47304b693ad15df329041282", + 1_034, + "079a78782c7ca36c89160eccb36a17227222c892261d7910ea55b6e97ccc5b9e", ), "resolved_seed_site_bindings": ( - 277, - "368fc19a8f07c4abf38ebcf2fbc4414d33c403c0f1f9881c2d2e3f5d6160feb6", + 316, + "199ec98045871070860164b522892327ab840b48c0c6a53972ebc0ff0d7d091f", ), "resolved_vintage_authorities": ( 63, @@ -485,8 +485,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "d6782c5de5bbed1bdc6bf653c4a6d4aadcad4ccc72d35e1092e130fcb04680a3", ), "source_stages": ( - 1_702, - "be3e24b53955b9dabccfc7e779212b1e97daf0c0992be3c183e391bf695ed4ee", + 1_715, + "78227f34c3f4ef946c06693dec9177dd906069f98016854bfd4d39c4cd4c51fa", ), "spine_assembly_household_mass_shares": ( 2, @@ -505,16 +505,16 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "cf0000464013118955571dac2691d4cc1b900c97c6570349b489678cdf937649", ), "spine_pipeline_contract": ( - 88, - "ae011455154bae0df3913ca9a056058a909d913439b292cb46c11f63c7d0d9a3", + 89, + "8d281f8f1e2f9914684150b0424ff816d0ec09ed0d9053ea5a64e877a86cc35b", ), "spine_sampling": ( 17, "29c6c1b3243e178783e7ab139993ba3a9b42d62edd4e9e4e1f3b28688daf2c6d", ), "spine_seed_site_bindings": ( - 277, - "dfa7ae701f62f7b1f06954798d29cfba3fc53a0be21e1751646d1f5bafd6071f", + 316, + "bb848f38e068858ac23e964a4ffbbf5542253da50515c72fe7753d8c56912486", ), "spine_support_roles": ( 29, diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 0f9b254674..764dcba78f 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -296,6 +296,19 @@ "pregnancy_assignment", "wic_claim_assignment", "snap_discretionary_exemption_assignment", + "immigration_humanitarian_paroled_one_year_afghanistan_assignment", + "immigration_humanitarian_paroled_one_year_ukraine_assignment", + "immigration_humanitarian_paroled_one_year_nicaragua_assignment", + "immigration_humanitarian_paroled_one_year_venezuela_assignment", + "immigration_humanitarian_refugee_assignment", + "immigration_humanitarian_asylee_assignment", + "immigration_humanitarian_deportation_withheld_assignment", + "immigration_humanitarian_tps_venezuela_assignment", + "immigration_humanitarian_tps_el_salvador_assignment", + "immigration_humanitarian_tps_honduras_assignment", + "immigration_humanitarian_tps_nicaragua_assignment", + "immigration_humanitarian_tps_nepal_assignment", + "immigration_humanitarian_tps_other_designated_assignment", "immigration_ead_workers_assignment", "immigration_ead_students_assignment", "ssi_take_up_assignment", @@ -348,20 +361,20 @@ EXPECTED_HASHES = { "acs_group_predictors": "a927bb7ecf3e84f54c93583ab79318654514ac546aefafba67da5285615fbd60", "acs_person_predictors": "878c788a6f037d7aca12b3586ea034eff04f3034ffa11935a736493042551f25", - "authority": "9d4a9672a0f03039b1fe874b9fe21ed575be0d29f14afc396d03cdf5c809bdd2", + "authority": "5730bd1c7410a59baa5a85ced86d9dcf0f901b3fa1c56617953199ad1a180eaa", "early_families": "4aa9f736fd76e83955477ad1667e58f48f264783f05bdc7f0102cd32d61323bd", - "full_checkpoint": "c66844ca246fb0277e648691a91fe7bea2a21c26980b398017af90f3128521c5", + "full_checkpoint": "9c2af34c0779efe4cb7fc7d25c37e5b318e7d886dda4cedcadc5acda0cf9b010", "gap_fill_schedule": "1c31f9868f7884347cc19cf1ff65da43f950b9114941a715bab168246db414a7", - "graph_nodes": "40cd51ffdfe2e9d9d08d48c08e8ded9de1e4b134783bab05c4abc6ad5c72ca1e", + "graph_nodes": "2c71f35d54330f13c96783e0252ed9354ff3b4c0edd7e30205886c87ab7ec078", "geography_assignment": "f49425ca8734ac559c73cf44f6458d86d3162a48956b98a27e6e758959361585", "late_families": "d91f9ff0eb52f43e7b6eed3d5c58c37abe1620c3a11021da15dae9c10e16d382", - "late_resource_semantics": "895ffd419de464a761669a5c3bf68db66e3d1a273c47631de80778a6c7f73b57", - "late_schedule": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", + "late_resource_semantics": "fda05279f38b371b38faa44b87d3a64aa34f1a4cc7f93bec7e059ca6418176ff", + "late_schedule": "88bc9243a3518982ae951c3de21bd55877e296ce4fcb183b9bee420d3a684b10", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "987b501c695e31f45521c4a178528f75ab3df22c09bc407b182213b2de99ee57", - "seed_map": "86cecb19281446d2b505fb975c822c60a604e18f44baded5dc9d62d508a8197e", - "seed_protocol": "55d6b907dd44613a5498dd7156f1ce9e78758835255bc7c336b4c327bb40e9bf", - "source_manifest": "d6849714bb9d6f64c000193a871c324b6b82850100db6cb3196f5f6e417dc494", + "seed_map": "674f1b222d31c925a9914791bb23fcfcfb6039c56261ae47a13cf735b29de393", + "seed_protocol": "7883cc956c11fe43fb0dedbe0bedb02b9ad804696f5551ee5d2c7a250b0abdbe", + "source_manifest": "9df42433ba92c5cad75f1df2394f8b64fb25ed027b5a5e6ad383874345112afa", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", } @@ -429,14 +442,14 @@ "primary_targets": 65, "producer_authored_outputs": 92, "producer_compiled_outputs": 227, - "producer_inputs": 2_744, + "producer_inputs": 2_750, "producer_nodes": 38, "producer_virtual_resources": 75, "release_rungs": 5, "resolved_references": 334, - "seed_owner_bindings": 112, + "seed_owner_bindings": 125, "seed_owner_rows": 54, - "seed_sites": 53, + "seed_sites": 66, "seed_streams": 14, "source_operators": 16, "source_stages": 37, @@ -1012,7 +1025,7 @@ def add( "producer_inputs_exact", clauses={ "producer input rows differ": inputs_exact, - "input row count differs": input_count == 2744, + "input row count differs": input_count == 2750, }, homes=("/imputation/producer_graph/nodes/*/inputs",), consumers=( @@ -1020,7 +1033,7 @@ def add( "compiler_ir.node_slices", ), observed={"rows": input_count}, - expected={"rows": 2744, "relation": "source rows preserved exactly"}, + expected={"rows": 2750, "relation": "source rows preserved exactly"}, ) outputs_exact = set(expected_outputs) == set(compiled_by_id) and all( _json_equal( @@ -1778,7 +1791,7 @@ def add( "seed_site_definitions_exact", clauses={ "seed site definitions differ": site_definitions_exact, - "seed site count differs": len(protocol_sites) == 53, + "seed site count differs": len(protocol_sites) == 66, }, homes=("/bundle/seed_protocol",), consumers=("compiler_ir.seed_stream_map.sites", "compiler_ir.node_slices"), @@ -1786,7 +1799,7 @@ def add( "sites": len(protocol_sites), "sha256": sha256_json([site.to_wire() for site in protocol.sites]), }, - expected={"sites": 53, "relation": "all site fields preserved exactly"}, + expected={"sites": 66, "relation": "all site fields preserved exactly"}, ) binding_by_site = {binding.site: binding for binding in spec.seed_site_bindings} binding_exact = set(binding_by_site) == set(compiled_sites) and all( @@ -1801,7 +1814,7 @@ def add( "owner binding count differs": sum( len(site.owners) for site in compiled_sites.values() ) - == 112, + == 125, "one or more sites have no owner": all( site.owners for site in compiled_sites.values() ), @@ -1811,7 +1824,7 @@ def add( observed={ "bindings": sum(len(site.owners) for site in compiled_sites.values()) }, - expected={"bindings": 112, "coverage": "all 53 sites"}, + expected={"bindings": 125, "coverage": "all 66 sites"}, ) expected_owner_sites: dict[tuple[str, str], list[str]] = {} for site in protocol.sites: @@ -1855,7 +1868,7 @@ def add( "seed groups overlap": groups_disjoint, "seed groups do not cover the protocol exactly": set(grouped_ids) == set(protocol_sites), - "seed group cardinality differs": len(grouped_ids) == 53, + "seed group cardinality differs": len(grouped_ids) == 66, }, homes=("/bundle/seed_protocol", "/spine/seed_site_bindings"), consumers=("compiler_ir.seed_stream_map",), @@ -1865,7 +1878,7 @@ def add( }, expected={ "groups": len(EXPECTED_SEED_GROUPS), - "sites": 53, + "sites": 66, "partition": "disjoint and exhaustive", }, ) diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml index a67ee2a18b..832afef7b0 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml @@ -99,7 +99,7 @@ sources: stage_asset: id: source_stages path: microcosm.build.us/source_stages.json - sha256: 7935d891ed16eac618f04813a7b8a3ceb9a971c8b4c4273683594ba61e7dd631 + sha256: b3ad16b1adb77a2cac6deae6c4d0f1779e4cf36a3f18820d9887a5434daaff1a stage_manifest: version: 1 country: us @@ -1884,19 +1884,21 @@ stages: formula-owned employment_income_last_year; the signed self-employment leaf and availability flag persist. - stage: immigration_status survey: CPS ASEC + published unauthorized-population estimates - source: https://www.pewresearch.org/short-reads/2024/07/22/what-we-know-about-unauthorized-immigrants-living-in-the-us/ + source: https://www.pewresearch.org/race-and-ethnicity/2025/08/21/u-s-unauthorized-immigrant-population-reached-a-record-14-million-in-2023/ grain: person artifacts: - kind: public_microdata format: census_asec vintage: build_year locator: Census CPS ASEC person file citizenship (PRCITSHP), entry year (PEINUSYR), nativity (PENATVTY), - and program-participation fields + labor-force status (A_LFSR), and program-participation fields - kind: administrative_table format: published_estimate vintage: latest_available locator: Pew Research Center unauthorized-immigrant population and worker estimates; Higher Ed Immigration - Portal undocumented-student estimates + Portal undocumented-student estimates; OHSS refugee and asylee annual flow reports; DHS OAW/U4U + parole reports and CBP CHNV releases; CRS RS20844 TPS designations table; EOIR adjudication statistics + via CRS IN12501 operations: - kind: read_table table: person @@ -1905,29 +1907,96 @@ stages: seed_from_build_config: true time_period_from_build_config: true undocumented_workers: - target: 8300000 - source: https://www.pewresearch.org/short-reads/2024/07/22/what-we-know-about-unauthorized-immigrants-living-in-the-us/ + target: 9700000 + source: https://www.pewresearch.org/race-and-ethnicity/2025/08/21/u-s-unauthorized-immigrant-population-reached-a-record-14-million-in-2023/ undocumented_students: target: 408000 source: https://www.higheredimmigrationportal.org/research/undocumented-students-in-higher-education-updated-march-2021/ undocumented_population_anchor: - value: 11000000 - source: https://www.pewresearch.org/short-reads/2024/07/22/what-we-know-about-unauthorized-immigrants-living-in-the-us/ + value: 14000000 + source: https://www.pewresearch.org/race-and-ethnicity/2025/08/21/u-s-unauthorized-immigrant-population-reached-a-record-14-million-in-2023/ + humanitarian_status_stocks: + paroled_one_year: + afghanistan: + target: 73566 + source: https://www.dhs.gov/sites/default/files/2023-06/PLCY%20-%20AFG%202503%20Update%202_0.pdf + ukraine: + target: 158000 + source: https://www.dhs.gov/sites/default/files/2024-12/2024_1104_dmo_plcy_uniting_for_ukraine_process_overview_and_assessment.pdf + nicaragua: + target: 93070 + source: https://www.cbp.gov/newsroom/national-media-release/cbp-releases-december-2024-monthly-update + venezuela: + target: 117330 + source: https://www.cbp.gov/newsroom/national-media-release/cbp-releases-december-2024-monthly-update + refugee: + target: 160000 + source: https://ohss.dhs.gov/topics/immigration/refugees/annual-flow-report/fy-24-refugees-flow-report + asylee: + target: 155000 + source: https://ohss.dhs.gov/topics/immigration/asylees/annual-flow-report/fy24-asylees-flow-report + deportation_withheld: + target: 0 + source: https://crsreports.congress.gov/product/pdf/IN/IN12501 + tps: + venezuela: + target: 605015 + source: https://www.everycrsreport.com/reports/RS20844.html + el_salvador: + target: 170125 + source: https://www.everycrsreport.com/reports/RS20844.html + honduras: + target: 51225 + source: https://www.everycrsreport.com/reports/RS20844.html + nicaragua: + target: 2910 + source: https://www.everycrsreport.com/reports/RS20844.html + nepal: + target: 7160 + source: https://www.everycrsreport.com/reports/RS20844.html + other_designated: + target: 21215 + source: https://www.everycrsreport.com/reports/RS20844.html outputs: - ssn_card_type - immigration_status_str notes: 'Citizenship is measured (PRCITSHP), not imputed. Non-citizens with any ASEC-UA legal-status - indicator (Van Hook et al., SSRN 4662801) become OTHER_NON_CITIZEN. The CPS carries no work-authorization - variable, so among residual non-citizens, workers and students spill to NON_CITIZEN_VALID_EAD in deterministic - seeded order until the remaining undocumented worker count matches Pew''s 8.3M and the undocumented - student count matches the Higher Ed Immigration Portal''s ~408k — the only forced margins. The total - undocumented population is emergent (≈13M on the 2024 ASEC, within the range of published 2023-24 - estimates) and is gated against Pew''s 11.0M 2022 anchor with a coarse plausibility band rather than - forced; reconciling the level is the calibration lane''s job. Immigration-status tags carry only statutory - tests the data supports: DACA (arrival cohort among EAD holders) and CUBAN_HAITIAN_ENTRANT (nativity - plus post-1980 arrival); other documented non-citizens stay LEGAL_PERMANENT_RESIDENT, the modal true - status — blanket REFUGEE/TPS labels are deliberately not fabricated because they would mislabel millions - and over-grant refugee-class benefit exemptions.' + indicator (Van Hook et al., SSRN 4662801) become OTHER_NON_CITIZEN. Humanitarian/temporary-protection + statuses (microcosm #767) are drawn between the indicators and the EAD spill, sequentially and mutually + exclusively in decreasing order of target precision — PAROLED_ONE_YEAR per program origin (OAW Afghans + 73,566 as of 2022-03-31; U4U Ukrainians over 158,000 as of 2023-09-30; CHNV Nicaraguans 93,070 and + Venezuelans 117,330 paroled through 2024-12), REFUGEE (FY2023+FY2024 admissions, 60,050+100,060 OHSS + — refugees must apply for LPR adjustment after one year, so the not-yet-adjusted stock is about the + trailing two years), ASYLEE (FY2022-FY2024 grants 35,080+51,410+68,060=154,550 OHSS, rounded to 155,000; + 88% of 2015-22 affirmative adult grantees had adjusted by end-2024, so about three trailing years), + then TPS per country (CRS RS20844 Table 1 as of 2025-03-31, with required-arrival cutoffs binding + at PEINUSYR granularity: El Salvador <=2001, Honduras/Nicaragua <=1999, Nepal <=2015). Candidates + are restricted by program origin countries (Census Appendix I PENATVTY codes) and arrival windows; + Cuba/Haiti-born are excluded everywhere (the CUBAN_HAITIAN_ENTRANT class is the better statutory label + and keeps H.R.1 eligibility — Haiti TPS 330,735 rides that label), the DACA statutory cohort is excluded + (dual TPS/DACA holders stay DACA), and Ukraine/Afghanistan TPS registrants (101,150/8,105) are carried + by the parole draw since they are overwhelmingly the same U4U/OAW parolees. Parole targets are gross + admissions: cohort members who later won asylum keep the parole label, which is channel-equivalent + under H.R.1 (both classes lose Medicaid/SNAP/PTC on identical dates). DEPORTATION_WITHHELD carries + an explicit zero target: EOIR grants about 2.2k withholdings a year (CRS IN12501: under 1% of 666,177 + FY2024 decisions) and publishes no stock, so the accumulated population is below the CPS''s resolution + for a defensible draw. CONDITIONAL_ENTRANT is deliberately not emitted (the INA 203(a)(7) class closed + in 1980). REFUGEE/ASYLEE draw from the indicator-documented pool only; parole/TPS may draw residual-pool + people, who then receive NON_CITIZEN_VALID_EAD SSN cards (both programs confer employment authorization). + Pew''s 2025 report supersedes its earlier series and estimates a coherent 2023 pair of 14.0M unauthorized + immigrants and 9.7M in the labor force. Its modeled universe is broader than the engine''s UNDOCUMENTED + enum: it retains DACA recipients, parolees, TPS holders, similar temporary protections, and the residual-EAD + subset of the engine''s broad CUBAN_HAITIAN_ENTRANT class, while excluding refugees, people already + granted asylum, and assumed-documented Cuban/Haitian entrants. The worker EAD spill binds that modeled + universe to 9.7M using actual ASEC labor-force status (A_LFSR codes 1-4 at ages 16+) restored from + the same pinned official ASEC archives by exact PERIDNUM join, never prior-year WSAL_VAL/SEMP_VAL + earnings. The student spill binds the corresponding broad residual universe to the Higher Ed Immigration + Portal''s ~408k students. The total modeled Pew-defined unauthorized population is emergent and gated + against the 14.0M 2023 anchor with a coarse band; per-category humanitarian masses are gated against + their cited targets with a coarse band (the ASEC undercovers 2022-24 arrivals — the Ukraine parole + pool saturates below its admin count); reconciling levels is the calibration lane''s job. DACA (arrival + cohort among EAD holders) and CUBAN_HAITIAN_ENTRANT (nativity plus post-1980 arrival) keep their statutory + tests; other documented non-citizens stay LEGAL_PERMANENT_RESIDENT, the modal true status.' - stage: hours_worked survey: Census CPS ASEC source: https://www.census.gov/programs-surveys/cps.html diff --git a/packages/microcosm-build/tests/test_spec_engine_coverage_tool.py b/packages/microcosm-build/tests/test_spec_engine_coverage_tool.py index 0f3ac73c14..932636266a 100644 --- a/packages/microcosm-build/tests/test_spec_engine_coverage_tool.py +++ b/packages/microcosm-build/tests/test_spec_engine_coverage_tool.py @@ -88,22 +88,22 @@ def test_us_coverage_is_exact_complete_and_honest( assert_coverage_complete(coverage_report) assert coverage_report["status"] == "pass" fields = coverage_report["field_usage"] - assert fields["configuration_field_count"] == 42_156 - assert fields["authored_normative_field_count"] == 32_384 - assert fields["resolved_binding_field_count"] == 9_772 - assert fields["consumed_field_count"] == 42_156 + assert fields["configuration_field_count"] == 42_540 + assert fields["authored_normative_field_count"] == 32_521 + assert fields["resolved_binding_field_count"] == 10_019 + assert fields["consumed_field_count"] == 42_540 assert fields["unused_field_count"] == 0 assert fields["multiple_primary_use_field_count"] == 0 assert fields["claim_count"] == 49 assert fields["mode_counts"] == { - "legacy_behavior": 13_988, - "compiler_semantic": 27_717, + "legacy_behavior": 14_020, + "compiler_semantic": 28_069, "front_end_validation": 348, "identity_only": 103, } assert fields["generation0_effect_counts"] == { - "legacy_behavior": 38_476, - "no_generation0_effect": 3_680, + "legacy_behavior": 38_574, + "no_generation0_effect": 3_966, } inventory = coverage_report["inventory_coverage"] diff --git a/packages/microcosm-build/tests/test_spec_engine_field_usage.py b/packages/microcosm-build/tests/test_spec_engine_field_usage.py index 8fae0ec2f0..752704688e 100644 --- a/packages/microcosm-build/tests/test_spec_engine_field_usage.py +++ b/packages/microcosm-build/tests/test_spec_engine_field_usage.py @@ -92,22 +92,22 @@ def _mutated_bundle( def test_exact_complete_ledger_has_one_primary_mode_per_pointer(field_ledger) -> None: - assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_156 + assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_540 assert field_ledger.source_counts == { - "authored": 32_384, - "resolved_bindings": 9_772, + "authored": 32_521, + "resolved_bindings": 10_019, } assert field_ledger.mode_counts == { - "legacy_behavior": 13_988, - "compiler_semantic": 27_717, + "legacy_behavior": 14_020, + "compiler_semantic": 28_069, "front_end_validation": 348, "identity_only": 103, } assert field_ledger.generation0_effect_counts == { - "legacy_behavior": 38_476, - "no_generation0_effect": 3_680, + "legacy_behavior": 38_574, + "no_generation0_effect": 3_966, } - assert len({field.pointer for field in field_ledger.fields}) == 42_156 + assert len({field.pointer for field in field_ledger.fields}) == 42_540 def test_eligibility_concepts_are_validation_not_generation0_behavior( diff --git a/packages/microcosm-build/tests/test_spec_engine_legacy_adapter.py b/packages/microcosm-build/tests/test_spec_engine_legacy_adapter.py index f10ceda635..67794b3e89 100644 --- a/packages/microcosm-build/tests/test_spec_engine_legacy_adapter.py +++ b/packages/microcosm-build/tests/test_spec_engine_legacy_adapter.py @@ -195,7 +195,7 @@ def test_adapter_preserves_generation_zero_identity_components( imputation = legacy_payload["imputation"] assert isinstance(imputation, dict) assert legacy_payload["stacked_authority_receipt"]["sha256"] == ( - "9d4a9672a0f03039b1fe874b9fe21ed575be0d29f14afc396d03cdf5c809bdd2" + "5730bd1c7410a59baa5a85ced86d9dcf0f901b3fa1c56617953199ad1a180eaa" ) assert imputation["late_producer_schedule_receipt"]["schedule_sha256"] == ( "88bc9243a3518982ae951c3de21bd55877e296ce4fcb183b9bee420d3a684b10" diff --git a/packages/microcosm-build/tests/test_spec_engine_loader.py b/packages/microcosm-build/tests/test_spec_engine_loader.py index 8525437344..bd4d9b3196 100644 --- a/packages/microcosm-build/tests/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/test_spec_engine_loader.py @@ -236,7 +236,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # Pin the domain separator, normalization rules, schema-set receipt, and # exact normative projection as one reviewable golden vector. assert first.spec_sha256 == ( - "a260da046550cac52594d8e7edeaec06422ea273031e97b40d1e98209d3348b5" + "cbc3321df143a3fab6b100eab503190e9b84ade88100486a087e1292f94beed6" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") diff --git a/packages/microcosm-build/tests/test_spec_engine_stacked_authority_semantics.py b/packages/microcosm-build/tests/test_spec_engine_stacked_authority_semantics.py index 82ce1e832b..b702b5ba94 100644 --- a/packages/microcosm-build/tests/test_spec_engine_stacked_authority_semantics.py +++ b/packages/microcosm-build/tests/test_spec_engine_stacked_authority_semantics.py @@ -73,7 +73,7 @@ def test_authority_projection_is_field_and_byte_identical_to_live_generation_zer assert projected == live assert stacked_identity_bytes(projected) == _canonical_bytes(live) assert projected["sha256"] == ( - "9d4a9672a0f03039b1fe874b9fe21ed575be0d29f14afc396d03cdf5c809bdd2" + "5730bd1c7410a59baa5a85ced86d9dcf0f901b3fa1c56617953199ad1a180eaa" ) assert { name: component["sha256"] for name, component in projected["components"].items() diff --git a/packages/microcosm-build/tests/test_us_bundle_core_contracts.py b/packages/microcosm-build/tests/test_us_bundle_core_contracts.py index 413230e918..af93daa068 100644 --- a/packages/microcosm-build/tests/test_us_bundle_core_contracts.py +++ b/packages/microcosm-build/tests/test_us_bundle_core_contracts.py @@ -129,7 +129,7 @@ def test_source_surface_classification_is_complete() -> None: } assert normative["stage_asset"] == { "id": "source_stages", - "sha256": "7935d891ed16eac618f04813a7b8a3ceb9a971c8b4c4273683594ba61e7dd631", + "sha256": "b3ad16b1adb77a2cac6deae6c4d0f1779e4cf36a3f18820d9887a5434daaff1a", } assert operational["stage_asset"] == { "path": "microcosm.build.us/source_stages.json" diff --git a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py index 21742582cb..9b825fc9d3 100644 --- a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py @@ -2933,7 +2933,7 @@ def capture_equality(expected: object, actual: object) -> None: "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "e058159f89a01b6541e98ee57ffb0169eeff8ed2ea4fb1c664b3f3fb0c60a675", + "spec_sha256": "7f1d3106448c57baf7205ee4c024cab986de0ed7faef7f98f296d7c79c792587", }, } diff --git a/packages/microcosm-build/tests/test_us_spec_bundle.py b/packages/microcosm-build/tests/test_us_spec_bundle.py index a81a8daea7..dc348ffd85 100644 --- a/packages/microcosm-build/tests/test_us_spec_bundle.py +++ b/packages/microcosm-build/tests/test_us_spec_bundle.py @@ -191,7 +191,7 @@ def _load_generator_module(): LEGACY_COMPATIBILITY_SHA256 = { "source_stages.json": ( - "7935d891ed16eac618f04813a7b8a3ceb9a971c8b4c4273683594ba61e7dd631" + "b3ad16b1adb77a2cac6deae6c4d0f1779e4cf36a3f18820d9887a5434daaff1a" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" @@ -401,7 +401,7 @@ def test_constant_derived_domain_counts_are_complete( ) assert ( compiled_schedule["payload_sha256"] - == "7be038d34f228d66c12b53558fc5f30c93f1b376f1058c5e4fd7e7563a88d67f" + == "d35ebb433f16e976c0d6e2210c99a36ca042a52c422c0968128678932d999735" ) assert len(take_up["programs"]) == 17 diff --git a/tools/generate_us_bundle_from_constants.py b/tools/generate_us_bundle_from_constants.py index 3df166854b..3b08ef80c1 100644 --- a/tools/generate_us_bundle_from_constants.py +++ b/tools/generate_us_bundle_from_constants.py @@ -122,7 +122,7 @@ def _domain_builders() -> dict[str, Callable[[], object]]: # frozen files, is the forward YAML -> legacy-payload path. FROZEN_LEGACY_RESOURCE_SHA256 = { "source_stages.json": ( - "7935d891ed16eac618f04813a7b8a3ceb9a971c8b4c4273683594ba61e7dd631" + "b3ad16b1adb77a2cac6deae6c4d0f1779e4cf36a3f18820d9887a5434daaff1a" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" diff --git a/tools/spec_engine_coverage.py b/tools/spec_engine_coverage.py index e8cd356603..ab17a17a8e 100644 --- a/tools/spec_engine_coverage.py +++ b/tools/spec_engine_coverage.py @@ -41,7 +41,7 @@ REPORT_SCHEMA_VERSION = 3 EXPECTED_POINTER_INVENTORY_SHA256 = ( - "2c0423a08dc16bf5f142134804a1a6545f887e32ca5c991c856a2f8bcfaea0f0" + "370e1958448fed48a635701439905d2145c3ae5bf3878adc1406a56a381a303b" ) DEFAULT_REPORT_PATH = ( Path(__file__).resolve().parents[1] From c52448d99fa92419ecec167147e17a41660dcfeb Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 18 Sep 2026 09:22:35 -0400 Subject: [PATCH 18/49] Drop the functools.cache import the removed fixture wrapper used Co-Authored-By: Claude Fable 5 --- packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py b/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py index 21b99326ba..0046faa78e 100644 --- a/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py +++ b/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py @@ -6,7 +6,6 @@ from collections.abc import Callable from copy import deepcopy from dataclasses import replace -from functools import cache from pathlib import Path import numpy as np From dcafeff789d9285dca0e8fa6ea9afa0d2947654c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 18 Sep 2026 09:36:37 -0400 Subject: [PATCH 19/49] Re-pin the live late-producer schedule component digest Co-Authored-By: Claude Fable 5 --- .../tests/test_spec_engine_stacked_authority_semantics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/microcosm-build/tests/test_spec_engine_stacked_authority_semantics.py b/packages/microcosm-build/tests/test_spec_engine_stacked_authority_semantics.py index b702b5ba94..06c59221f5 100644 --- a/packages/microcosm-build/tests/test_spec_engine_stacked_authority_semantics.py +++ b/packages/microcosm-build/tests/test_spec_engine_stacked_authority_semantics.py @@ -88,7 +88,7 @@ def test_authority_projection_is_field_and_byte_identical_to_live_generation_zer "cacc6c11e114dbae3aaa2761cc6b3fcb1191cd9b689b1c2bd096614c51ebff8b" ), "late_producer_schedule": ( - "777979b267e7307581b39d14b7232b6ee2712e3da0029fffc090396ebd849166" + "ebbce94e8cefb7afef0a2547d28f578b13c6a791576ab04081a6a09099a8c8bc" ), "metric_registry": ( "d75cb9b29f8b0a9a085471a11f4c19c32ba04cbe5419053df94ea81cbe6125a9" From 9622beb501df2d5989f1c9992fe01b657140698b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 18 Sep 2026 13:52:39 -0400 Subject: [PATCH 20/49] Forward legacy worker authentication through the transfer replay The merged transition-authority seam replays the branch's post-PUF transfer evidence for every pool, but the replay already knows the legacy mode (frozen authority snapshot, legacy schedule); the seam just dropped the parameter, so schema-9 pools authenticated by attestation were rejected as non-canonical. Main's two new worker-relocation tests also assumed the three-row fixture; the stacked fixture carries one row per positive canonical humanitarian draw, so the count is derived. Refs #767 Co-Authored-By: Claude Fable 5.1 --- .../microcosm/build/us_runtime/stacked_spine.py | 1 + .../tests/test_us_multispine_pool_h5_io.py | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py b/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py index 55803a34eb..0a0eb61df6 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py @@ -8616,6 +8616,7 @@ def validate_stacked_late_producer_transition_authority( transfer, boundary=boundary, frame=frame, + legacy_worker_authentication=legacy_worker_authentication, ) _validate_late_transition_authority( frame, diff --git a/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py b/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py index 0046faa78e..97bf32f163 100644 --- a/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py +++ b/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py @@ -2491,7 +2491,12 @@ def test_current_worker_authentication_ignores_audit_alias_relocation( manifest_path ) - assert frame.n("household") == 3 + positive_draws = [ + draw + for draw in immigration_runtime.us_immigration_controls().humanitarian + if draw.target > 0 + ] + assert frame.n("household") == 1 + len(positive_draws) assert authenticated.sha256 == pool_sha256 assert ( manifest["worker_execution_authentication"]["audit_aliases"] @@ -2554,7 +2559,12 @@ def test_scoring_loader_accepts_legacy_worker_alias_relocation_only( "compatibility_attestation_sha256": _sha256(attestation_path), "purpose": "scoring_only", } - assert frame.n("household") == 3 + positive_draws = [ + draw + for draw in immigration_runtime.us_immigration_controls().humanitarian + if draw.target > 0 + ] + assert frame.n("household") == 1 + len(positive_draws) assert manifest["status"] == "gate_failed" assert manifest["simulation_ready"] is False assert authenticated.sha256 == manifest["pool_h5"]["sha256"] From 5658ee54abfca12e92d4433573e88c50639faceb Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 19 Sep 2026 08:29:50 -0400 Subject: [PATCH 21/49] Exhaust non-overlapping spill rows before touching the other margin The student spill took the only worker when its first draw landed on a DACA-cohort row (an EAD leaves that row inside Pew's universe, so the draw closed nothing) even though a nonworking, non-cohort student could have closed the gap: 4 of 6 seeds lost the worker control, and the later worker pass cannot restore a removed worker. Each tier now runs its seeded draw and its corrective draw before the overlapping tier is considered. Also corrects the module docstring's draw order, which still described the pre-parole-first precedence. Found by the Astra agreement gate on #779. Refs #767 Co-Authored-By: Claude Fable 5.1 --- .../microcosm/build/us_runtime/immigration.py | 64 ++++++++----------- .../tests/test_us_immigration.py | 24 +++++++ 2 files changed, 52 insertions(+), 36 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py b/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py index 0bceea92c0..e7c8072e15 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py @@ -51,9 +51,11 @@ 2025-07-01, ACA §71301/§71302 effective 2026/2027 — all keyed to these enum values in policyengine-us parameters), so an LPR-only file silently zeroes every one of those channels. Instead of blankets, the stage now - draws ``REFUGEE`` / ``ASYLEE`` / ``DEPORTATION_WITHHELD`` / - ``PAROLED_ONE_YEAR`` / ``TPS`` — in that order, mutually exclusive — - from candidates whose survey signature supports the status (origin + draws ``PAROLED_ONE_YEAR`` / ``REFUGEE`` / ``ASYLEE`` / + ``DEPORTATION_WITHHELD`` / ``TPS`` — in that order (decreasing target + precision: exact per-origin program admissions first), mutually + exclusive, so a person two cohorts could claim takes the earlier + status — from candidates whose survey signature supports it (origin country x arrival window x legal-status indicators), each to a manifest-cited weighted stock target. Cuba/Haiti-born persons are excluded from every humanitarian draw (the ``CUBAN_HAITIAN_ENTRANT`` @@ -1175,13 +1177,14 @@ def _spill_pew_unauthorized_excess( ) -> None: """Spill a broad Pew-universe margin to EAD in deterministic order. - The first draw preserves the prior EAD/DACA allocation behavior by - selecting from the whole residual scope. A selected DACA or residual - Cuban/Haitian cohort row still belongs to Pew's universe, so a second pass - spills additional rows outside those retained cohorts when needed. Within - each pass, rows outside the other controlled margin are selected first; - this keeps the student and worker controls from disturbing one another - unless their overlap makes that unavoidable. + Rows outside the other controlled margin are exhausted before any + overlapping row is touched, so the student and worker controls disturb + one another only when their overlap makes that unavoidable. Within each + of those two tiers the first draw preserves the prior EAD/DACA allocation + behavior by selecting from the whole residual scope; a selected DACA or + residual Cuban/Haitian cohort row still belongs to Pew's universe, so a + corrective draw then spills rows outside those retained cohorts before + the next tier is considered. """ draws = _stable_person_draws(person, seed=seed, salt=salt) @@ -1194,38 +1197,27 @@ def current_excess() -> float: ) return float(weights[scope & included].sum()) - target - # Preserve the existing seeded EAD allocation surface. DACA selections - # do not reduce the Pew count and are compensated by the corrective pass. - initial_candidates = (ssn_codes == 0) & noncitizens & scope - initial_excess = current_excess() - if initial_excess > 0: - for priority in (~preserve_scope, preserve_scope): + # Exhaust each tier before the next: a row inside the other margin is + # spilled only once no row outside it can close the gap, because that + # margin's own pass cannot restore a row this one removed. + for priority in (~preserve_scope, preserve_scope): + # Preserve the seeded EAD allocation surface. DACA and residual + # Cuban/Haitian cohort members keep protected engine labels after an + # EAD and so remain inside Pew's estimate; only rows outside those + # cohorts reduce the count, which the corrective draw supplies. + for retained_allowed in (True, False): + if current_excess() <= 0: + return + candidates = (ssn_codes == 0) & noncitizens & scope & priority + if not retained_allowed: + candidates &= ~retained_ead_cohort selected = _select_weight_to_target( - initial_candidates & priority, + candidates, weights, draws, current_excess(), ) ssn_codes[selected] = 2 - if current_excess() <= 0: - return - - # DACA and residual Cuban/Haitian cohort members retain protected engine - # labels after EAD assignment and therefore remain inside Pew's estimate. - # Only rows outside those cohorts can close any remaining gap. - for priority in (~preserve_scope, preserve_scope): - corrective_candidates = ( - (ssn_codes == 0) & noncitizens & scope & ~retained_ead_cohort & priority - ) - selected = _select_weight_to_target( - corrective_candidates, - weights, - draws, - current_excess(), - ) - ssn_codes[selected] = 2 - if current_excess() <= 0: - return def _humanitarian_draw_candidates( diff --git a/packages/microcosm-build/tests/test_us_immigration.py b/packages/microcosm-build/tests/test_us_immigration.py index 259f1ddaf5..89e74a62df 100644 --- a/packages/microcosm-build/tests/test_us_immigration.py +++ b/packages/microcosm-build/tests/test_us_immigration.py @@ -341,6 +341,30 @@ def test_worker_control_counts_tps_before_spilling_residual_workers( assert output.loc[1, "ssn_card_type"] == "NON_CITIZEN_VALID_EAD" assert output.loc[1, "immigration_status_str"] == ("LEGAL_PERMANENT_RESIDENT") + @pytest.mark.parametrize("seed", range(6)) + def test_student_spill_exhausts_nonworkers_before_touching_a_worker( + self, seed: int + ) -> None: + # Three residual students, control 2: a nonworking DACA-cohort row + # (an EAD leaves it inside Pew's universe, so spilling it closes + # nothing), a nonworking non-cohort row, and the only worker. The + # corrective draw must take the nonworker: the worker pass cannot + # restore a worker the student pass removed. + person = _person_table( + [ + _noncitizen(A_HSCOL=2, PEINUSYR=17, A_AGE=30), + _noncitizen(A_HSCOL=2), + _noncitizen(A_HSCOL=2, A_LFSR=1), + ] + ) + output = _run(person, students=2.0, workers=1.0, seed=seed) + status = output["immigration_status_str"] + in_pew_universe = status.isin(["UNDOCUMENTED", "DACA"]) + in_labor_force = output["A_LFSR"].isin([1, 2, 3, 4]) + assert int(in_pew_universe.sum()) == 2 + assert int((in_pew_universe & in_labor_force).sum()) == 1 + assert status.iloc[2] == "UNDOCUMENTED" + def test_below_control_counts_spill_nothing(self) -> None: person = _person_table( [_noncitizen(A_LFSR=1, WSAL_VAL=10_000.0), _noncitizen(A_HSCOL=2)] From 2a9a8d811c5d64c9ef5fb8d5e1a2a4a7a6c5a958 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 19 Sep 2026 08:33:53 -0400 Subject: [PATCH 22/49] Keep attested schema-9 pools authenticable: freeze their sealed schedule Main's legacy scoring path rebuilt the historical schema-16 schedule and v11 authority from the *current* registry, so this branch's contract changes (the immigration stage reads A_LFSR instead of WSAL_VAL/SEMP_VAL, its transfer group gains seven evidence requirements, and the execution contract binds the constrained-immigration reconciliation) silently redefined history: reconstructions hashed to 2f61b55e/e024d438 instead of the 02e618cc/e660a8ce that real schema-9 pools sealed, so every such pool would have been refused. The schedule is now a frozen, digest-verified resource; the authority follows because its eight other components are byte-identical. Legacy pools also validate against the literal 11-step operator order they sealed, are not asked for the schema-10 canonical terminal gate set, and have their immigration targets replayed as the ordinary transfer targets they were. Found by the Astra agreement gate on #779. Refs #767 Co-Authored-By: Claude Fable 5.1 --- ...egacy_schema16_late_producer_schedule.json | 1 + .../src/microcosm/build/us_runtime/h5_io.py | 30 ++++++++++- .../build/us_runtime/stacked_spine.py | 14 +++-- .../us_runtime/us_late_producer_registry.py | 54 +++++++++++++------ 4 files changed, 78 insertions(+), 21 deletions(-) create mode 100644 packages/microcosm-build/src/microcosm/build/us/legacy_schema16_late_producer_schedule.json diff --git a/packages/microcosm-build/src/microcosm/build/us/legacy_schema16_late_producer_schedule.json b/packages/microcosm-build/src/microcosm/build/us/legacy_schema16_late_producer_schedule.json new file mode 100644 index 0000000000..8b1ee6c78c --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us/legacy_schema16_late_producer_schedule.json @@ -0,0 +1 @@ +{"acs_earnings_universe_input_inventory":{"operator":"acs_pums_earnings_universe","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"@acs_pums_earnings_universe_execution_config","entity":"person","value_kind":"non_null"}]],"label":"execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"mapped_earnings:employment_income_before_lsr","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"mapped_earnings:self_employment_income_before_lsr","optional":true,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"non_null"}]],"label":"person_tax_unit_link","optional":false,"required_scope":null},{"alternatives":[[{"column":"SEMP","entity":"person","value_kind":"column_present"}]],"label":"raw_source:SEMP","optional":false,"required_scope":null},{"alternatives":[[{"column":"WAGP","entity":"person","value_kind":"column_present"}]],"label":"raw_source:WAGP","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"non_null"}],[{"column":"person_source_id","entity":"person","value_kind":"non_null"}]],"label":"stable_person_lineage","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"support_clone_index","optional":false,"required_scope":null}]},"edges":[["acs_pums_earnings_universe","primary_puf_qrf"],["primary_puf_qrf","source:impute_us_housing_assistance_to_puf_support"],["primary_puf_qrf","source:with_us_adult_care_inputs"],["primary_puf_qrf","source:with_us_child_support_inputs"],["primary_puf_qrf","source:with_us_childcare_inputs"],["primary_puf_qrf","source:with_us_disability_benefits"],["primary_puf_qrf","source:with_us_education_inputs"],["primary_puf_qrf","source:with_us_energy_subsidy_input"],["primary_puf_qrf","source:with_us_immigration_inputs"],["primary_puf_qrf","source:with_us_medicare_take_up_input"],["primary_puf_qrf","source:with_us_pregnancy_inputs"],["primary_puf_qrf","source:with_us_prior_year_income_inputs"],["primary_puf_qrf","source:with_us_retirement_contribution_inputs"],["primary_puf_qrf","source:with_us_retirement_distribution_inputs"],["primary_puf_qrf","source:with_us_weeks_unemployed"],["primary_puf_qrf","source:with_us_wic_claim_input"],["primary_puf_qrf","source:with_us_workers_compensation"],["primary_puf_qrf","transfer:person/adult_care"],["primary_puf_qrf","transfer:person/model_required_boolean"],["primary_puf_qrf","transfer:person/puf_tax_itemization__batch_1"],["primary_puf_qrf","transfer:person/puf_tax_itemization__batch_2"],["primary_puf_qrf","transfer:person/puf_tax_itemization__batch_3"],["primary_puf_qrf","transfer:person/puf_tax_itemization__batch_4"],["primary_puf_qrf","transfer:person/puf_tax_itemization__batch_5"],["primary_puf_qrf","transfer:person/source_operator_child_support"],["primary_puf_qrf","transfer:person/source_operator_disability_benefits"],["primary_puf_qrf","transfer:person/source_operator_education_inputs"],["primary_puf_qrf","transfer:person/source_operator_immigration"],["primary_puf_qrf","transfer:person/source_operator_medicare_take_up"],["primary_puf_qrf","transfer:person/source_operator_retirement_contributions"],["primary_puf_qrf","transfer:person/source_operator_retirement_distributions"],["primary_puf_qrf","transfer:person/source_operator_weeks_unemployed"],["primary_puf_qrf","transfer:person/source_operator_wic_claim"],["primary_puf_qrf","transfer:person/source_operator_workers_compensation"],["primary_puf_qrf","transfer:spm_unit/source_operator_energy_subsidy"],["primary_puf_qrf","transfer:tax_unit/puf_tax_itemization"],["source:impute_us_housing_assistance_to_puf_support","source_finalizer"],["source:with_us_adult_care_inputs","source_finalizer"],["source:with_us_adult_care_inputs","transfer:person/adult_care"],["source:with_us_child_support_inputs","source_finalizer"],["source:with_us_child_support_inputs","transfer:person/source_operator_child_support"],["source:with_us_childcare_inputs","source:with_us_adult_care_inputs"],["source:with_us_childcare_inputs","source_finalizer"],["source:with_us_disability_benefits","source_finalizer"],["source:with_us_disability_benefits","transfer:person/source_operator_disability_benefits"],["source:with_us_education_inputs","source_finalizer"],["source:with_us_education_inputs","transfer:person/source_operator_education_inputs"],["source:with_us_energy_subsidy_input","source_finalizer"],["source:with_us_energy_subsidy_input","transfer:spm_unit/source_operator_energy_subsidy"],["source:with_us_immigration_inputs","source_finalizer"],["source:with_us_immigration_inputs","transfer:person/source_operator_immigration"],["source:with_us_medicare_take_up_input","source_finalizer"],["source:with_us_medicare_take_up_input","transfer:person/source_operator_medicare_take_up"],["source:with_us_pregnancy_inputs","source:with_us_wic_claim_input"],["source:with_us_pregnancy_inputs","source_finalizer"],["source:with_us_pregnancy_inputs","transfer:person/model_required_boolean"],["source:with_us_prior_year_income_inputs","source_finalizer"],["source:with_us_retirement_contribution_inputs","source_finalizer"],["source:with_us_retirement_contribution_inputs","transfer:person/puf_tax_itemization__batch_2"],["source:with_us_retirement_contribution_inputs","transfer:person/puf_tax_itemization__batch_3"],["source:with_us_retirement_contribution_inputs","transfer:person/source_operator_retirement_contributions"],["source:with_us_retirement_distribution_inputs","source_finalizer"],["source:with_us_retirement_distribution_inputs","transfer:person/source_operator_retirement_distributions"],["source:with_us_weeks_unemployed","source_finalizer"],["source:with_us_weeks_unemployed","transfer:person/source_operator_weeks_unemployed"],["source:with_us_wic_claim_input","source_finalizer"],["source:with_us_wic_claim_input","transfer:person/source_operator_wic_claim"],["source:with_us_workers_compensation","source_finalizer"],["source:with_us_workers_compensation","transfer:person/source_operator_workers_compensation"],["transfer:person/puf_tax_itemization__batch_2","source:with_us_education_inputs"],["transfer:person/puf_tax_itemization__batch_5","source:with_us_adult_care_inputs"]],"execution_receipt_contract":{"row_binding":"declared_globally_reconciled_input_and_scope_exact_output_source_and_primary_callback_resource_receipt_and_previous_execution_sha256","top_binding":"entry_and_output_frame_sha256_execution_chain_source_completion_and_nineteen_transfer_groups","transition_authority":{"authority_id":"us_stacked_late_producer_transition","independent_digest_required":true,"metadata_key":"us_late_producer_transition_authority","version":1},"version":3,"virtual_resource_binding":"exact_kind_specific_semantic_payload_and_sha256"},"external_stages":["post_clone_input_surface"],"order":["acs_pums_earnings_universe","primary_puf_qrf","source:impute_us_housing_assistance_to_puf_support","source:with_us_child_support_inputs","source:with_us_childcare_inputs","source:with_us_disability_benefits","source:with_us_energy_subsidy_input","source:with_us_immigration_inputs","source:with_us_medicare_take_up_input","source:with_us_pregnancy_inputs","source:with_us_prior_year_income_inputs","source:with_us_retirement_contribution_inputs","source:with_us_retirement_distribution_inputs","source:with_us_weeks_unemployed","source:with_us_workers_compensation","transfer:person/puf_tax_itemization__batch_1","transfer:person/puf_tax_itemization__batch_4","transfer:person/puf_tax_itemization__batch_5","transfer:tax_unit/puf_tax_itemization","source:with_us_adult_care_inputs","source:with_us_wic_claim_input","transfer:person/model_required_boolean","transfer:person/puf_tax_itemization__batch_2","transfer:person/puf_tax_itemization__batch_3","transfer:person/source_operator_child_support","transfer:person/source_operator_disability_benefits","transfer:person/source_operator_immigration","transfer:person/source_operator_medicare_take_up","transfer:person/source_operator_retirement_contributions","transfer:person/source_operator_retirement_distributions","transfer:person/source_operator_weeks_unemployed","transfer:person/source_operator_workers_compensation","transfer:spm_unit/source_operator_energy_subsidy","source:with_us_education_inputs","transfer:person/adult_care","transfer:person/source_operator_wic_claim","source_finalizer","transfer:person/source_operator_education_inputs"],"overlap_ownership":{"artifact_kind":"microcosm_us_late_overlap_ownership","doctrine":{"clone_2_policy":"inherit_or_mirror_clone_1_final_owner_bytes","non_owner_write_policy":"masked_or_verified_byte_exact_noop","owner_cardinality":"exactly_one_per_target_origin_clone_role","retirement_legacy_order":"primary_then_postclone_source_owner_then_late_transfer","tail_preservation_guard":"unchanged_fail_closed"},"ownership":[{"clone_index":0,"entity":"person","final_owner":"transfer:person/puf_tax_itemization__batch_2","finalization":"late_transfer_owner_last","origin":"asec","producer_actions":[{"action":"scope_masked_noop","owns_final":false,"producer":"primary_puf_qrf"},{"action":"consume_only_byte_exact_noop","owns_final":false,"producer":"source:with_us_education_inputs"},{"action":"final_write","owns_final":true,"producer":"transfer:person/puf_tax_itemization__batch_2"}],"target":"qualified_tuition_expenses"},{"clone_index":1,"entity":"person","final_owner":"primary_puf_qrf","finalization":"primary_write","origin":"asec","producer_actions":[{"action":"primary_write","owns_final":true,"producer":"primary_puf_qrf"},{"action":"consume_only_byte_exact_noop","owns_final":false,"producer":"source:with_us_education_inputs"},{"action":"producer_masked_byte_exact_noop","owns_final":false,"producer":"transfer:person/puf_tax_itemization__batch_2"}],"target":"qualified_tuition_expenses"},{"clone_index":2,"entity":"person","final_owner":"primary_puf_qrf","finalization":"byte_exact_clone_1_inheritance","origin":"asec","producer_actions":[{"action":"byte_exact_clone_1_inheritance","owns_final":true,"producer":"primary_puf_qrf"},{"action":"consume_only_byte_exact_noop","owns_final":false,"producer":"source:with_us_education_inputs"},{"action":"producer_masked_byte_exact_noop","owns_final":false,"producer":"transfer:person/puf_tax_itemization__batch_2"}],"target":"qualified_tuition_expenses"},{"clone_index":0,"entity":"person","final_owner":"transfer:person/puf_tax_itemization__batch_2","finalization":"late_transfer_owner_last","origin":"acs","producer_actions":[{"action":"scope_masked_noop","owns_final":false,"producer":"primary_puf_qrf"},{"action":"origin_projection_masked_noop","owns_final":false,"producer":"source:with_us_education_inputs"},{"action":"final_write","owns_final":true,"producer":"transfer:person/puf_tax_itemization__batch_2"}],"target":"qualified_tuition_expenses"},{"clone_index":1,"entity":"person","final_owner":"primary_puf_qrf","finalization":"primary_write","origin":"acs","producer_actions":[{"action":"primary_write","owns_final":true,"producer":"primary_puf_qrf"},{"action":"origin_projection_masked_noop","owns_final":false,"producer":"source:with_us_education_inputs"},{"action":"producer_masked_byte_exact_noop","owns_final":false,"producer":"transfer:person/puf_tax_itemization__batch_2"}],"target":"qualified_tuition_expenses"},{"clone_index":2,"entity":"person","final_owner":"primary_puf_qrf","finalization":"byte_exact_clone_1_inheritance","origin":"acs","producer_actions":[{"action":"byte_exact_clone_1_inheritance","owns_final":true,"producer":"primary_puf_qrf"},{"action":"origin_projection_masked_noop","owns_final":false,"producer":"source:with_us_education_inputs"},{"action":"producer_masked_byte_exact_noop","owns_final":false,"producer":"transfer:person/puf_tax_itemization__batch_2"}],"target":"qualified_tuition_expenses"},{"clone_index":0,"entity":"person","final_owner":"source:with_us_retirement_contribution_inputs","finalization":"source_direct_split","origin":"asec","producer_actions":[{"action":"scope_masked_noop","owns_final":false,"producer":"primary_puf_qrf"},{"action":"final_write","owns_final":true,"producer":"source:with_us_retirement_contribution_inputs"},{"action":"producer_masked_byte_exact_noop","owns_final":false,"producer":"transfer:person/puf_tax_itemization__batch_2"}],"target":"traditional_ira_contributions_desired"},{"clone_index":1,"entity":"person","final_owner":"source:with_us_retirement_contribution_inputs","finalization":"source_owner_last_overwrite","origin":"asec","producer_actions":[{"action":"interim_write_overwritten_by_owner_last","owns_final":false,"producer":"primary_puf_qrf"},{"action":"final_write","owns_final":true,"producer":"source:with_us_retirement_contribution_inputs"},{"action":"producer_masked_byte_exact_noop","owns_final":false,"producer":"transfer:person/puf_tax_itemization__batch_2"}],"target":"traditional_ira_contributions_desired"},{"clone_index":2,"entity":"person","final_owner":"source:with_us_retirement_contribution_inputs","finalization":"byte_exact_clone_1_mirror","origin":"asec","producer_actions":[{"action":"interim_clone_1_inheritance_overwritten_by_owner_last","owns_final":false,"producer":"primary_puf_qrf"},{"action":"byte_exact_clone_1_mirror","owns_final":true,"producer":"source:with_us_retirement_contribution_inputs"},{"action":"producer_masked_byte_exact_noop","owns_final":false,"producer":"transfer:person/puf_tax_itemization__batch_2"}],"target":"traditional_ira_contributions_desired"},{"clone_index":0,"entity":"person","final_owner":"transfer:person/puf_tax_itemization__batch_2","finalization":"late_transfer_owner_last","origin":"acs","producer_actions":[{"action":"scope_masked_noop","owns_final":false,"producer":"primary_puf_qrf"},{"action":"origin_projection_masked_noop","owns_final":false,"producer":"source:with_us_retirement_contribution_inputs"},{"action":"final_write","owns_final":true,"producer":"transfer:person/puf_tax_itemization__batch_2"}],"target":"traditional_ira_contributions_desired"},{"clone_index":1,"entity":"person","final_owner":"primary_puf_qrf","finalization":"primary_write","origin":"acs","producer_actions":[{"action":"primary_write","owns_final":true,"producer":"primary_puf_qrf"},{"action":"origin_projection_masked_noop","owns_final":false,"producer":"source:with_us_retirement_contribution_inputs"},{"action":"producer_masked_byte_exact_noop","owns_final":false,"producer":"transfer:person/puf_tax_itemization__batch_2"}],"target":"traditional_ira_contributions_desired"},{"clone_index":2,"entity":"person","final_owner":"primary_puf_qrf","finalization":"byte_exact_clone_1_inheritance","origin":"acs","producer_actions":[{"action":"byte_exact_clone_1_inheritance","owns_final":true,"producer":"primary_puf_qrf"},{"action":"origin_projection_masked_noop","owns_final":false,"producer":"source:with_us_retirement_contribution_inputs"},{"action":"producer_masked_byte_exact_noop","owns_final":false,"producer":"transfer:person/puf_tax_itemization__batch_2"}],"target":"traditional_ira_contributions_desired"},{"clone_index":0,"entity":"person","final_owner":"source:with_us_retirement_contribution_inputs","finalization":"source_direct_split","origin":"asec","producer_actions":[{"action":"scope_masked_noop","owns_final":false,"producer":"primary_puf_qrf"},{"action":"final_write","owns_final":true,"producer":"source:with_us_retirement_contribution_inputs"},{"action":"producer_masked_byte_exact_noop","owns_final":false,"producer":"transfer:person/puf_tax_itemization__batch_3"}],"target":"self_employed_pension_contributions_desired"},{"clone_index":1,"entity":"person","final_owner":"source:with_us_retirement_contribution_inputs","finalization":"source_owner_last_overwrite","origin":"asec","producer_actions":[{"action":"interim_write_overwritten_by_owner_last","owns_final":false,"producer":"primary_puf_qrf"},{"action":"final_write","owns_final":true,"producer":"source:with_us_retirement_contribution_inputs"},{"action":"producer_masked_byte_exact_noop","owns_final":false,"producer":"transfer:person/puf_tax_itemization__batch_3"}],"target":"self_employed_pension_contributions_desired"},{"clone_index":2,"entity":"person","final_owner":"source:with_us_retirement_contribution_inputs","finalization":"byte_exact_clone_1_mirror","origin":"asec","producer_actions":[{"action":"interim_clone_1_inheritance_overwritten_by_owner_last","owns_final":false,"producer":"primary_puf_qrf"},{"action":"byte_exact_clone_1_mirror","owns_final":true,"producer":"source:with_us_retirement_contribution_inputs"},{"action":"producer_masked_byte_exact_noop","owns_final":false,"producer":"transfer:person/puf_tax_itemization__batch_3"}],"target":"self_employed_pension_contributions_desired"},{"clone_index":0,"entity":"person","final_owner":"transfer:person/puf_tax_itemization__batch_3","finalization":"late_transfer_owner_last","origin":"acs","producer_actions":[{"action":"scope_masked_noop","owns_final":false,"producer":"primary_puf_qrf"},{"action":"origin_projection_masked_noop","owns_final":false,"producer":"source:with_us_retirement_contribution_inputs"},{"action":"final_write","owns_final":true,"producer":"transfer:person/puf_tax_itemization__batch_3"}],"target":"self_employed_pension_contributions_desired"},{"clone_index":1,"entity":"person","final_owner":"primary_puf_qrf","finalization":"primary_write","origin":"acs","producer_actions":[{"action":"primary_write","owns_final":true,"producer":"primary_puf_qrf"},{"action":"origin_projection_masked_noop","owns_final":false,"producer":"source:with_us_retirement_contribution_inputs"},{"action":"producer_masked_byte_exact_noop","owns_final":false,"producer":"transfer:person/puf_tax_itemization__batch_3"}],"target":"self_employed_pension_contributions_desired"},{"clone_index":2,"entity":"person","final_owner":"primary_puf_qrf","finalization":"byte_exact_clone_1_inheritance","origin":"acs","producer_actions":[{"action":"byte_exact_clone_1_inheritance","owns_final":true,"producer":"primary_puf_qrf"},{"action":"origin_projection_masked_noop","owns_final":false,"producer":"source:with_us_retirement_contribution_inputs"},{"action":"producer_masked_byte_exact_noop","owns_final":false,"producer":"transfer:person/puf_tax_itemization__batch_3"}],"target":"self_employed_pension_contributions_desired"}],"schema_version":1,"sha256":"5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356","targets":[{"entity":"person","source_producer":"source:with_us_education_inputs","source_touch":"consume_only_byte_exact_noop","target":"qualified_tuition_expenses","transfer_producer":"transfer:person/puf_tax_itemization__batch_2"},{"entity":"person","source_producer":"source:with_us_retirement_contribution_inputs","source_touch":"persisted_owner_last_write","target":"traditional_ira_contributions_desired","transfer_producer":"transfer:person/puf_tax_itemization__batch_2"},{"entity":"person","source_producer":"source:with_us_retirement_contribution_inputs","source_touch":"persisted_owner_last_write","target":"self_employed_pension_contributions_desired","transfer_producer":"transfer:person/puf_tax_itemization__batch_3"}]},"payload_sha256":"02e618cc656eb39990ed99dca2b30a52794e01e2b06a3c2df87ca4a7d85ab086","primary_puf_input_inventory":{"operator":"primary_puf_qrf","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"dividend_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"dividend_income","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"dividend_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"filing_status_input","entity":"tax_unit","value_kind":"non_null"}]],"label":"filing_status","optional":false,"required_scope":null},{"alternatives":[[{"column":"long_term_capital_gains","entity":"person","value_kind":"finite_numeric"}],[{"column":"long_term_capital_gains_before_response","entity":"person","value_kind":"finite_numeric"}],[{"column":"long_term_capital_gains","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"long_term_capital_gains","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"non_null"}]],"label":"person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"alimony_expense","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:alimony_expense","optional":true,"required_scope":null},{"alternatives":[[{"column":"alimony_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:alimony_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"business_is_sstb","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:business_is_sstb","optional":true,"required_scope":null},{"alternatives":[[{"column":"casualty_loss","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:casualty_loss","optional":true,"required_scope":null},{"alternatives":[[{"column":"charitable_cash_donations","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:charitable_cash_donations","optional":true,"required_scope":null},{"alternatives":[[{"column":"charitable_non_cash_donations","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:charitable_non_cash_donations","optional":true,"required_scope":null},{"alternatives":[[{"column":"educator_expense","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:educator_expense","optional":true,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:employment_income_before_lsr","optional":true,"required_scope":null},{"alternatives":[[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:estate_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"estate_income_would_be_qualified","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:estate_income_would_be_qualified","optional":true,"required_scope":null},{"alternatives":[[{"column":"farm_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:farm_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"farm_operations_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:farm_operations_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"farm_operations_income_would_be_qualified","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:farm_operations_income_would_be_qualified","optional":true,"required_scope":null},{"alternatives":[[{"column":"farm_rent_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:farm_rent_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"farm_rent_income_would_be_qualified","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:farm_rent_income_would_be_qualified","optional":true,"required_scope":null},{"alternatives":[[{"column":"home_mortgage_interest","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:home_mortgage_interest","optional":true,"required_scope":null},{"alternatives":[[{"column":"investment_income_elected_form_4952","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:investment_income_elected_form_4952","optional":true,"required_scope":null},{"alternatives":[[{"column":"investment_interest_expense","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:investment_interest_expense","optional":true,"required_scope":null},{"alternatives":[[{"column":"long_term_capital_gains_before_response","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:long_term_capital_gains_before_response","optional":true,"required_scope":null},{"alternatives":[[{"column":"long_term_capital_gains_on_collectibles","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:long_term_capital_gains_on_collectibles","optional":true,"required_scope":null},{"alternatives":[[{"column":"miscellaneous_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:miscellaneous_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:non_qualified_dividend_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"non_sch_d_capital_gains","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:non_sch_d_capital_gains","optional":true,"required_scope":null},{"alternatives":[[{"column":"partnership_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:partnership_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"partnership_s_corp_income_would_be_qualified","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:partnership_s_corp_income_would_be_qualified","optional":true,"required_scope":null},{"alternatives":[[{"column":"partnership_self_employment_net_earnings","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:partnership_self_employment_net_earnings","optional":true,"required_scope":null},{"alternatives":[[{"column":"qualified_bdc_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:qualified_bdc_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:qualified_dividend_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"qualified_reit_and_ptp_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:qualified_reit_and_ptp_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"qualified_tuition_expenses","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:qualified_tuition_expenses","optional":true,"required_scope":null},{"alternatives":[[{"column":"real_estate_taxes","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:real_estate_taxes","optional":true,"required_scope":null},{"alternatives":[[{"column":"rental_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:rental_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"rental_income_would_be_qualified","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:rental_income_would_be_qualified","optional":true,"required_scope":null},{"alternatives":[[{"column":"s_corp_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:s_corp_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"salt_refund_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:salt_refund_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employed_pension_contributions_desired","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:self_employed_pension_contributions_desired","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:self_employment_income_before_lsr","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_would_be_qualified","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:self_employment_income_would_be_qualified","optional":true,"required_scope":null},{"alternatives":[[{"column":"short_term_capital_gains","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:short_term_capital_gains","optional":true,"required_scope":null},{"alternatives":[[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:social_security_dependents","optional":true,"required_scope":null},{"alternatives":[[{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:social_security_disability","optional":true,"required_scope":null},{"alternatives":[[{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:social_security_retirement","optional":true,"required_scope":null},{"alternatives":[[{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:social_security_survivors","optional":true,"required_scope":null},{"alternatives":[[{"column":"sstb_self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:sstb_self_employment_income_before_lsr","optional":true,"required_scope":null},{"alternatives":[[{"column":"sstb_self_employment_income_would_be_qualified","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:sstb_self_employment_income_would_be_qualified","optional":true,"required_scope":null},{"alternatives":[[{"column":"sstb_unadjusted_basis_qualified_property","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:sstb_unadjusted_basis_qualified_property","optional":true,"required_scope":null},{"alternatives":[[{"column":"sstb_w2_wages_from_qualified_business","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:sstb_w2_wages_from_qualified_business","optional":true,"required_scope":null},{"alternatives":[[{"column":"student_loan_interest","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:student_loan_interest","optional":true,"required_scope":null},{"alternatives":[[{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:tax_exempt_interest_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:taxable_interest_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:taxable_ira_distributions","optional":true,"required_scope":null},{"alternatives":[[{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:taxable_private_pension_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"traditional_ira_contributions_desired","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:traditional_ira_contributions_desired","optional":true,"required_scope":null},{"alternatives":[[{"column":"unadjusted_basis_qualified_property","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:unadjusted_basis_qualified_property","optional":true,"required_scope":null},{"alternatives":[[{"column":"unreimbursed_business_employee_expenses","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:unreimbursed_business_employee_expenses","optional":true,"required_scope":null},{"alternatives":[[{"column":"w2_wages_from_qualified_business","entity":"person","value_kind":"finite_numeric"}]],"label":"person_output_allocation_basis:w2_wages_from_qualified_business","optional":true,"required_scope":null},{"alternatives":[[{"column":"@primary_puf_execution_config","entity":"tax_unit","value_kind":"non_null"}]],"label":"primary_puf_execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@primary_qrf_checkpoint","entity":"tax_unit","value_kind":"non_null"}]],"label":"primary_qrf_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"@puf_donor_tax_units","entity":"tax_unit","value_kind":"non_null"}]],"label":"puf_donor","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_full_time_college_student","entity":"person","value_kind":"finite_numeric"}]],"label":"qualified_tuition_allocation_fallback","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"tax_unit","value_kind":"non_null"}]],"label":"resolved_tax_unit_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"self_employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"short_term_capital_gains","entity":"person","value_kind":"finite_numeric"}],[{"column":"short_term_capital_gains","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"short_term_capital_gains","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"non_null"}]],"label":"support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"non_null"}]],"label":"tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"domestic_production_ald","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_output_passthrough:domestic_production_ald","optional":true,"required_scope":null},{"alternatives":[[{"column":"first_home_mortgage_balance","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_output_passthrough:first_home_mortgage_balance","optional":true,"required_scope":null},{"alternatives":[[{"column":"first_home_mortgage_interest","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_output_passthrough:first_home_mortgage_interest","optional":true,"required_scope":null},{"alternatives":[[{"column":"first_home_mortgage_origination_year","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_output_passthrough:first_home_mortgage_origination_year","optional":true,"required_scope":null},{"alternatives":[[{"column":"health_savings_account_ald","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_output_passthrough:health_savings_account_ald","optional":true,"required_scope":null},{"alternatives":[[{"column":"second_home_mortgage_balance","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_output_passthrough:second_home_mortgage_balance","optional":true,"required_scope":null},{"alternatives":[[{"column":"second_home_mortgage_interest","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_output_passthrough:second_home_mortgage_interest","optional":true,"required_scope":null},{"alternatives":[[{"column":"second_home_mortgage_origination_year","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_output_passthrough:second_home_mortgage_origination_year","optional":true,"required_scope":null},{"alternatives":[[{"column":"unrecaptured_section_1250_gain","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_output_passthrough:unrecaptured_section_1250_gain","optional":true,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"non_null"},{"column":"tax_unit_id","entity":"tax_unit","value_kind":"non_null"}]],"label":"tax_unit_person_count","optional":false,"required_scope":null},{"alternatives":[[{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"taxable_interest_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},"producer_count":38,"schedule_sha256":"e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f","schema_version":16,"source_input_inventories":[{"operator":"impute_us_housing_assistance_to_puf_support","requirements":[{"alternatives":[[{"column":"A_AGE","entity":"person","value_kind":"finite_numeric"}],[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"has_esi","entity":"person","value_kind":"finite_numeric"}]],"label":"employer_health_coverage","optional":false,"required_scope":null},{"alternatives":[[{"column":"WSAL_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"filing_status","entity":"tax_unit","value_kind":"non_null"}],[{"column":"filing_status_input","entity":"tax_unit","value_kind":"non_null"}]],"label":"filing_status","optional":false,"required_scope":null},{"alternatives":[[{"column":"receives_housing_assistance","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"housing_assistance_receipt","optional":false,"required_scope":null},{"alternatives":[[{"column":"takes_up_housing_assistance_if_eligible","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"housing_assistance_takeup","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"non_null"}]],"label":"person_spm_unit_link","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_tax_unit_link","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"SEMP_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"self_employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"A_SEX","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_male","entity":"person","value_kind":"finite_numeric"}]],"label":"sex","optional":false,"required_scope":null},{"alternatives":[[{"column":"SS_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"social_security_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"source_wrapper:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@post_clone_source_execution_config","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"source_wrapper:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"source_wrapper:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"source_wrapper:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:source_evidence","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"spm_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"non_null"}]],"label":"spm_support_clone_index","optional":true,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"non_null"}]],"label":"spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"support_clone_index","optional":true,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_role_input","entity":"person","value_kind":"non_null"}]],"label":"tax_unit_role","optional":false,"required_scope":null}]},{"operator":"with_us_adult_care_inputs","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_pre_subsidy_childcare_expenses","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"childcare_expenses","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_spm_unit_link","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_tax_unit_link","optional":false,"required_scope":null},{"alternatives":[[{"column":"PEDISDRS","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:PEDISDRS","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_full_time_college_student","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:is_full_time_college_student","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"spm_unit","value_kind":"non_null"}]],"label":"resolved_spm_unit_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"tax_unit","value_kind":"non_null"}]],"label":"resolved_tax_unit_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"self_employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"source_wrapper:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@post_clone_source_execution_config","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"source_wrapper:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"source_wrapper:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"source_wrapper:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:source_evidence","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"sstb_self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"sstb_earned_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"},{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"support_role","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_role_input","entity":"person","value_kind":"non_null"}]],"label":"tax_unit_role","optional":false,"required_scope":null}]},{"operator":"with_us_child_support_inputs","requirements":[{"alternatives":[[{"column":"A_AGE","entity":"person","value_kind":"finite_numeric"}],[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"has_esi","entity":"person","value_kind":"finite_numeric"}]],"label":"employer_health_coverage","optional":false,"required_scope":null},{"alternatives":[[{"column":"WSAL_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"filing_status","entity":"tax_unit","value_kind":"non_null"}],[{"column":"filing_status_input","entity":"tax_unit","value_kind":"non_null"}]],"label":"filing_status","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_tax_unit_link","optional":false,"required_scope":null},{"alternatives":[[{"column":"CHSP_VAL","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:CHSP_VAL","optional":false,"required_scope":null},{"alternatives":[[{"column":"CSP_VAL","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:CSP_VAL","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"SEMP_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"self_employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"A_SEX","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_male","entity":"person","value_kind":"finite_numeric"}]],"label":"sex","optional":false,"required_scope":null},{"alternatives":[[{"column":"SS_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"social_security_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"source_wrapper:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@post_clone_source_execution_config","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"source_wrapper:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"source_wrapper:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"source_wrapper:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:source_evidence","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"support_clone_index","optional":true,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_role_input","entity":"person","value_kind":"non_null"}]],"label":"tax_unit_role","optional":false,"required_scope":null}]},{"operator":"with_us_childcare_inputs","requirements":[{"alternatives":[[{"column":"A_AGE","entity":"person","value_kind":"finite_numeric"}],[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"has_esi","entity":"person","value_kind":"finite_numeric"}]],"label":"employer_health_coverage","optional":false,"required_scope":null},{"alternatives":[[{"column":"WSAL_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"filing_status","entity":"tax_unit","value_kind":"non_null"}],[{"column":"filing_status_input","entity":"tax_unit","value_kind":"non_null"}]],"label":"filing_status","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_tax_unit_link","optional":false,"required_scope":null},{"alternatives":[[{"column":"SPM_CHILDCAREXPNS","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:SPM_CHILDCAREXPNS","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"SEMP_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"self_employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"A_SEX","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_male","entity":"person","value_kind":"finite_numeric"}]],"label":"sex","optional":false,"required_scope":null},{"alternatives":[[{"column":"SS_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"social_security_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"source_wrapper:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@post_clone_source_execution_config","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"source_wrapper:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"source_wrapper:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"source_wrapper:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:source_evidence","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"non_null"}]],"label":"spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"support_clone_index","optional":true,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_role_input","entity":"person","value_kind":"non_null"}]],"label":"tax_unit_role","optional":false,"required_scope":null}]},{"operator":"with_us_disability_benefits","requirements":[{"alternatives":[[{"column":"A_AGE","entity":"person","value_kind":"finite_numeric"}],[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"has_esi","entity":"person","value_kind":"finite_numeric"}]],"label":"employer_health_coverage","optional":false,"required_scope":null},{"alternatives":[[{"column":"WSAL_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"filing_status","entity":"tax_unit","value_kind":"non_null"}],[{"column":"filing_status_input","entity":"tax_unit","value_kind":"non_null"}]],"label":"filing_status","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_tax_unit_link","optional":false,"required_scope":null},{"alternatives":[[{"column":"DIS_SC1","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:DIS_SC1","optional":false,"required_scope":null},{"alternatives":[[{"column":"DIS_SC2","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:DIS_SC2","optional":false,"required_scope":null},{"alternatives":[[{"column":"DIS_VAL1","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:DIS_VAL1","optional":false,"required_scope":null},{"alternatives":[[{"column":"DIS_VAL2","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:DIS_VAL2","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"SEMP_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"self_employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"A_SEX","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_male","entity":"person","value_kind":"finite_numeric"}]],"label":"sex","optional":false,"required_scope":null},{"alternatives":[[{"column":"SS_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"social_security_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"source_wrapper:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@post_clone_source_execution_config","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"source_wrapper:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"source_wrapper:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"source_wrapper:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:source_evidence","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"support_clone_index","optional":true,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_role_input","entity":"person","value_kind":"non_null"}]],"label":"tax_unit_role","optional":false,"required_scope":null}]},{"operator":"with_us_education_inputs","requirements":[{"alternatives":[[{"column":"ED_VAL","entity":"person","value_kind":"finite_numeric"}]],"label":"education_source","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"non_null"}]],"label":"person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"qualified_tuition_expenses","entity":"person","value_kind":"finite_numeric"}]],"label":"qualified_tuition","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"source_wrapper:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@post_clone_source_execution_config","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"source_wrapper:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"source_wrapper:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"source_wrapper:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:source_evidence","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:tax_unit_id","optional":false,"required_scope":null}]},{"operator":"with_us_energy_subsidy_input","requirements":[{"alternatives":[[{"column":"A_AGE","entity":"person","value_kind":"finite_numeric"}],[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"has_esi","entity":"person","value_kind":"finite_numeric"}]],"label":"employer_health_coverage","optional":false,"required_scope":null},{"alternatives":[[{"column":"WSAL_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"filing_status","entity":"tax_unit","value_kind":"non_null"}],[{"column":"filing_status_input","entity":"tax_unit","value_kind":"non_null"}]],"label":"filing_status","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_tax_unit_link","optional":false,"required_scope":null},{"alternatives":[[{"column":"SPM_ENGVAL","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:SPM_ENGVAL","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"SEMP_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"self_employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"A_SEX","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_male","entity":"person","value_kind":"finite_numeric"}]],"label":"sex","optional":false,"required_scope":null},{"alternatives":[[{"column":"SS_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"social_security_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"source_wrapper:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@post_clone_source_execution_config","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"source_wrapper:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"source_wrapper:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"source_wrapper:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:source_evidence","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"non_null"}]],"label":"spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"support_clone_index","optional":true,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_role_input","entity":"person","value_kind":"non_null"}]],"label":"tax_unit_role","optional":false,"required_scope":null}]},{"operator":"with_us_immigration_inputs","requirements":[{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"non_null"}]],"label":"person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"A_AGE","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:A_AGE","optional":false,"required_scope":null},{"alternatives":[[{"column":"A_HSCOL","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:A_HSCOL","optional":false,"required_scope":null},{"alternatives":[[{"column":"A_MARITL","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:A_MARITL","optional":false,"required_scope":null},{"alternatives":[[{"column":"A_MJOCC","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:A_MJOCC","optional":false,"required_scope":null},{"alternatives":[[{"column":"A_SPOUSE","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:A_SPOUSE","optional":false,"required_scope":null},{"alternatives":[[{"column":"CAID","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:CAID","optional":false,"required_scope":null},{"alternatives":[[{"column":"CHAMPVA","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:CHAMPVA","optional":false,"required_scope":null},{"alternatives":[[{"column":"IHSFLG","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:IHSFLG","optional":false,"required_scope":null},{"alternatives":[[{"column":"MCARE","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:MCARE","optional":false,"required_scope":null},{"alternatives":[[{"column":"MIL","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:MIL","optional":false,"required_scope":null},{"alternatives":[[{"column":"PEAFEVER","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:PEAFEVER","optional":false,"required_scope":null},{"alternatives":[[{"column":"PEINUSYR","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:PEINUSYR","optional":false,"required_scope":null},{"alternatives":[[{"column":"PEIO1COW","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:PEIO1COW","optional":false,"required_scope":null},{"alternatives":[[{"column":"PENATVTY","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:PENATVTY","optional":false,"required_scope":null},{"alternatives":[[{"column":"PEN_SC1","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:PEN_SC1","optional":false,"required_scope":null},{"alternatives":[[{"column":"PEN_SC2","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:PEN_SC2","optional":false,"required_scope":null},{"alternatives":[[{"column":"PRCITSHP","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:PRCITSHP","optional":false,"required_scope":null},{"alternatives":[[{"column":"RESNSS1","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:RESNSS1","optional":false,"required_scope":null},{"alternatives":[[{"column":"RESNSS2","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:RESNSS2","optional":false,"required_scope":null},{"alternatives":[[{"column":"SEMP_VAL","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:SEMP_VAL","optional":false,"required_scope":null},{"alternatives":[[{"column":"SPM_CAPHOUSESUB","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:SPM_CAPHOUSESUB","optional":false,"required_scope":null},{"alternatives":[[{"column":"SSI_YN","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:SSI_YN","optional":false,"required_scope":null},{"alternatives":[[{"column":"SS_YN","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:SS_YN","optional":false,"required_scope":null},{"alternatives":[[{"column":"WSAL_VAL","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:WSAL_VAL","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"source_wrapper:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@post_clone_source_execution_config","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"source_wrapper:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"source_wrapper:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"source_wrapper:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:source_evidence","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"non_null"}],[{"column":"source_person_id","entity":"person","value_kind":"non_null"},{"column":"source_year","entity":"person","value_kind":"non_null"}]],"label":"stable_source_identity","optional":false,"required_scope":null}]},{"operator":"with_us_medicare_take_up_input","requirements":[{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"non_null"}]],"label":"person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"MCARE","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:MCARE","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"source_wrapper:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@post_clone_source_execution_config","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"source_wrapper:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"source_wrapper:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"source_wrapper:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:source_evidence","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:tax_unit_id","optional":false,"required_scope":null}]},{"operator":"with_us_pregnancy_inputs","requirements":[{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"non_null"}]],"label":"person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"pregnancy_structural_source_person_id","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_AGE","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:A_AGE","optional":false,"required_scope":null},{"alternatives":[[{"column":"A_SEX","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:A_SEX","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"source_wrapper:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@post_clone_source_execution_config","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"source_wrapper:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"source_wrapper:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"source_wrapper:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:source_evidence","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"non_null"}],[{"column":"source_household_id","entity":"person","value_kind":"non_null"},{"column":"source_person_id","entity":"person","value_kind":"non_null"},{"column":"source_year","entity":"person","value_kind":"non_null"}]],"label":"stable_source_identity","optional":false,"required_scope":null}]},{"operator":"with_us_prior_year_income_inputs","requirements":[{"alternatives":[[{"column":"A_AGE","entity":"person","value_kind":"finite_numeric"}],[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"has_esi","entity":"person","value_kind":"finite_numeric"}]],"label":"employer_health_coverage","optional":false,"required_scope":null},{"alternatives":[[{"column":"WSAL_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_last_year","entity":"person","value_kind":"finite_numeric"}]],"label":"employment_income_last_year","optional":false,"required_scope":null},{"alternatives":[[{"column":"filing_status","entity":"tax_unit","value_kind":"non_null"}],[{"column":"filing_status_input","entity":"tax_unit","value_kind":"non_null"}]],"label":"filing_status","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_tax_unit_link","optional":false,"required_scope":null},{"alternatives":[[{"column":"I_ERNVAL","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:I_ERNVAL","optional":false,"required_scope":null},{"alternatives":[[{"column":"I_SEVAL","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:I_SEVAL","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:PERIDNUM","optional":false,"required_scope":null},{"alternatives":[[{"column":"SEMP_VAL","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:SEMP_VAL","optional":false,"required_scope":null},{"alternatives":[[{"column":"WSAL_VAL","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:WSAL_VAL","optional":false,"required_scope":null},{"alternatives":[[{"column":"source_year","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:source_year","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"SEMP_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"self_employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_last_year","entity":"person","value_kind":"finite_numeric"}]],"label":"self_employment_income_last_year","optional":false,"required_scope":null},{"alternatives":[[{"column":"A_SEX","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_male","entity":"person","value_kind":"finite_numeric"}]],"label":"sex","optional":false,"required_scope":null},{"alternatives":[[{"column":"SS_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"social_security_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"source_wrapper:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@post_clone_source_execution_config","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"source_wrapper:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"source_wrapper:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"source_wrapper:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:source_evidence","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"support_clone_index","optional":true,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_role_input","entity":"person","value_kind":"non_null"}]],"label":"tax_unit_role","optional":false,"required_scope":null}]},{"operator":"with_us_retirement_contribution_inputs","requirements":[{"alternatives":[[{"column":"A_AGE","entity":"person","value_kind":"finite_numeric"}],[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"has_esi","entity":"person","value_kind":"finite_numeric"}]],"label":"employer_health_coverage","optional":false,"required_scope":null},{"alternatives":[[{"column":"WSAL_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"filing_status","entity":"tax_unit","value_kind":"non_null"}],[{"column":"filing_status_input","entity":"tax_unit","value_kind":"non_null"}]],"label":"filing_status","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_tax_unit_link","optional":false,"required_scope":null},{"alternatives":[[{"column":"RETCB_VAL","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:RETCB_VAL","optional":false,"required_scope":null},{"alternatives":[[{"column":"SEMP_VAL","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:SEMP_VAL","optional":false,"required_scope":null},{"alternatives":[[{"column":"WSAL_VAL","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:WSAL_VAL","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"SEMP_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"self_employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"A_SEX","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_male","entity":"person","value_kind":"finite_numeric"}]],"label":"sex","optional":false,"required_scope":null},{"alternatives":[[{"column":"SS_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"social_security_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"source_wrapper:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@post_clone_source_execution_config","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"source_wrapper:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"source_wrapper:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"source_wrapper:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:source_evidence","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"support_clone_index","optional":true,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_role_input","entity":"person","value_kind":"non_null"}]],"label":"tax_unit_role","optional":false,"required_scope":null}]},{"operator":"with_us_retirement_distribution_inputs","requirements":[{"alternatives":[[{"column":"A_AGE","entity":"person","value_kind":"finite_numeric"}],[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"has_esi","entity":"person","value_kind":"finite_numeric"}]],"label":"employer_health_coverage","optional":false,"required_scope":null},{"alternatives":[[{"column":"WSAL_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"filing_status","entity":"tax_unit","value_kind":"non_null"}],[{"column":"filing_status_input","entity":"tax_unit","value_kind":"non_null"}]],"label":"filing_status","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_tax_unit_link","optional":false,"required_scope":null},{"alternatives":[[{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"}]],"label":"puf_taxable_ira_distribution","optional":false,"required_scope":null},{"alternatives":[[{"column":"DST_SC1","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:DST_SC1","optional":false,"required_scope":null},{"alternatives":[[{"column":"DST_SC1_YNG","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:DST_SC1_YNG","optional":false,"required_scope":null},{"alternatives":[[{"column":"DST_SC2","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:DST_SC2","optional":false,"required_scope":null},{"alternatives":[[{"column":"DST_SC2_YNG","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:DST_SC2_YNG","optional":false,"required_scope":null},{"alternatives":[[{"column":"DST_VAL1","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:DST_VAL1","optional":false,"required_scope":null},{"alternatives":[[{"column":"DST_VAL1_YNG","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:DST_VAL1_YNG","optional":false,"required_scope":null},{"alternatives":[[{"column":"DST_VAL2","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:DST_VAL2","optional":false,"required_scope":null},{"alternatives":[[{"column":"DST_VAL2_YNG","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:DST_VAL2_YNG","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"SEMP_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"self_employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"A_SEX","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_male","entity":"person","value_kind":"finite_numeric"}]],"label":"sex","optional":false,"required_scope":null},{"alternatives":[[{"column":"SS_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"social_security_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"source_wrapper:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@post_clone_source_execution_config","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"source_wrapper:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"source_wrapper:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"source_wrapper:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:source_evidence","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"support_clone_index","optional":true,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_role_input","entity":"person","value_kind":"non_null"}]],"label":"tax_unit_role","optional":false,"required_scope":null}]},{"operator":"with_us_weeks_unemployed","requirements":[{"alternatives":[[{"column":"A_AGE","entity":"person","value_kind":"finite_numeric"}],[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_tax_unit_dependent","entity":"person","value_kind":"finite_numeric"},{"column":"is_tax_unit_head","entity":"person","value_kind":"finite_numeric"},{"column":"is_tax_unit_spouse","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_unit_role_input","entity":"person","value_kind":"non_null"}]],"label":"explicit_tax_unit_roles","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"non_null"},{"column":"filing_status","entity":"tax_unit","value_kind":"non_null"},{"column":"tax_unit_id","entity":"tax_unit","value_kind":"non_null"}],[{"column":"person_tax_unit_id","entity":"person","value_kind":"non_null"},{"column":"filing_status_input","entity":"tax_unit","value_kind":"non_null"},{"column":"tax_unit_id","entity":"tax_unit","value_kind":"non_null"}],[{"column":"tax_unit_is_joint","entity":"person","value_kind":"finite_numeric"}]],"label":"joint_filing_status","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"A_SEX","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_male","entity":"person","value_kind":"finite_numeric"}]],"label":"sex","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"finite_numeric"}]],"label":"source_identity","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"source_wrapper:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@post_clone_source_execution_config","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"source_wrapper:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"source_wrapper:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"source_wrapper:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:source_evidence","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"source_year","entity":"person","value_kind":"finite_numeric"}]],"label":"source_year","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"UC_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"unemployment_compensation","entity":"person","value_kind":"finite_numeric"}]],"label":"unemployment_compensation_predictor","optional":true,"required_scope":null},{"alternatives":[[{"column":"LKWEEKS","entity":"person","value_kind":"finite_numeric"}]],"label":"weeks_source","optional":false,"required_scope":null}]},{"operator":"with_us_wic_claim_input","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_pregnant","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:is_pregnant","optional":false,"required_scope":null},{"alternatives":[[{"column":"own_children_in_household","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:own_children_in_household","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"source_wrapper:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@post_clone_source_execution_config","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"source_wrapper:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"source_wrapper:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"source_wrapper:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:source_evidence","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"non_null"}],[{"column":"person_support_source_id","entity":"person","value_kind":"non_null"}],[{"column":"source_household_id","entity":"person","value_kind":"non_null"},{"column":"source_person_id","entity":"person","value_kind":"non_null"},{"column":"source_year","entity":"person","value_kind":"non_null"}]],"label":"stable_source_identity","optional":false,"required_scope":null}]},{"operator":"with_us_workers_compensation","requirements":[{"alternatives":[[{"column":"A_AGE","entity":"person","value_kind":"finite_numeric"}],[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"has_esi","entity":"person","value_kind":"finite_numeric"}]],"label":"employer_health_coverage","optional":false,"required_scope":null},{"alternatives":[[{"column":"WSAL_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"filing_status","entity":"tax_unit","value_kind":"non_null"}],[{"column":"filing_status_input","entity":"tax_unit","value_kind":"non_null"}]],"label":"filing_status","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"person_tax_unit_link","optional":false,"required_scope":null},{"alternatives":[[{"column":"WC_VAL","entity":"person","value_kind":"finite_numeric"}]],"label":"raw_person:WC_VAL","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"SEMP_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"self_employment_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"A_SEX","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_male","entity":"person","value_kind":"finite_numeric"}]],"label":"sex","optional":false,"required_scope":null},{"alternatives":[[{"column":"SS_VAL","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"social_security_income","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"source_wrapper:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@post_clone_source_execution_config","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:execution_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"source_wrapper:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"source_wrapper:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"source_wrapper:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"source_wrapper:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"PERIDNUM","entity":"person","value_kind":"non_null"}]],"label":"source_wrapper:source_evidence","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"source_wrapper:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"support_clone_index","optional":true,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_role_input","entity":"person","value_kind":"non_null"}]],"label":"tax_unit_role","optional":false,"required_scope":null}]}],"source_producer_count":16,"status":"derived_and_import_validated","transfer_group_count":19,"transfer_groups":[{"entity":"person","family":"adult_care","name":"transfer:person/adult_care","targets":["is_incapable_of_self_care","pre_subsidy_care_expenses"]},{"entity":"person","family":"model_required_boolean","name":"transfer:person/model_required_boolean","targets":["is_pregnant"]},{"entity":"person","family":"puf_tax_itemization__batch_1","name":"transfer:person/puf_tax_itemization__batch_1","targets":["tax_exempt_interest_income","long_term_capital_gains_on_collectibles","non_sch_d_capital_gains","alimony_expense","salt_refund_income","charitable_cash_donations","charitable_non_cash_donations","home_mortgage_interest"]},{"entity":"person","family":"puf_tax_itemization__batch_2","name":"transfer:person/puf_tax_itemization__batch_2","targets":["investment_interest_expense","investment_income_elected_form_4952","student_loan_interest","educator_expense","qualified_tuition_expenses","casualty_loss","unreimbursed_business_employee_expenses","traditional_ira_contributions_desired"]},{"entity":"person","family":"puf_tax_itemization__batch_3","name":"transfer:person/puf_tax_itemization__batch_3","targets":["self_employed_pension_contributions_desired","estate_income","farm_income","farm_rent_income","partnership_income","partnership_self_employment_net_earnings","estate_income_would_be_qualified","farm_operations_income_would_be_qualified"]},{"entity":"person","family":"puf_tax_itemization__batch_4","name":"transfer:person/puf_tax_itemization__batch_4","targets":["farm_rent_income_would_be_qualified","partnership_s_corp_income_would_be_qualified","rental_income_would_be_qualified","self_employment_income_would_be_qualified","sstb_self_employment_income_would_be_qualified","business_is_sstb","qualified_bdc_income","qualified_reit_and_ptp_income"]},{"entity":"person","family":"puf_tax_itemization__batch_5","name":"transfer:person/puf_tax_itemization__batch_5","targets":["sstb_self_employment_income_before_lsr","sstb_unadjusted_basis_qualified_property","sstb_w2_wages_from_qualified_business","unadjusted_basis_qualified_property","w2_wages_from_qualified_business"]},{"entity":"person","family":"source_operator_child_support","name":"transfer:person/source_operator_child_support","targets":["child_support_expense","child_support_received"]},{"entity":"person","family":"source_operator_disability_benefits","name":"transfer:person/source_operator_disability_benefits","targets":["disability_benefits"]},{"entity":"person","family":"source_operator_education_inputs","name":"transfer:person/source_operator_education_inputs","targets":["attends_eligible_educational_institution_for_american_opportunity_credit","educational_assistance","has_american_opportunity_credit_1098_t_or_exception","has_american_opportunity_credit_institution_ein","is_enrolled_at_least_half_time_for_american_opportunity_credit","is_pursuing_credential_for_american_opportunity_credit"]},{"entity":"person","family":"source_operator_immigration","name":"transfer:person/source_operator_immigration","targets":["ssn_card_type","immigration_status_str"]},{"entity":"person","family":"source_operator_medicare_take_up","name":"transfer:person/source_operator_medicare_take_up","targets":["takes_up_medicare_if_eligible"]},{"entity":"person","family":"source_operator_retirement_contributions","name":"transfer:person/source_operator_retirement_contributions","targets":["roth_401k_contributions_desired","roth_ira_contributions_desired","traditional_401k_contributions_desired"]},{"entity":"person","family":"source_operator_retirement_distributions","name":"transfer:person/source_operator_retirement_distributions","targets":["keogh_distributions","tax_exempt_ira_distributions","taxable_401k_distributions","taxable_403b_distributions","taxable_sep_distributions"]},{"entity":"person","family":"source_operator_weeks_unemployed","name":"transfer:person/source_operator_weeks_unemployed","targets":["weeks_unemployed"]},{"entity":"person","family":"source_operator_wic_claim","name":"transfer:person/source_operator_wic_claim","targets":["takes_up_wic_if_eligible"]},{"entity":"person","family":"source_operator_workers_compensation","name":"transfer:person/source_operator_workers_compensation","targets":["workers_compensation"]},{"entity":"tax_unit","family":"puf_tax_itemization","name":"transfer:tax_unit/puf_tax_itemization","targets":["domestic_production_ald","unrecaptured_section_1250_gain","first_home_mortgage_balance","first_home_mortgage_interest","first_home_mortgage_origination_year","health_savings_account_ald"]},{"entity":"spm_unit","family":"source_operator_energy_subsidy","name":"transfer:spm_unit/source_operator_energy_subsidy","targets":["spm_unit_energy_subsidy"]}],"transfer_input_inventories":[{"operator":"transfer:person/adult_care","requirements":[{"alternatives":[[{"column":"tax_unit_role_input","entity":"person","value_kind":"non_null"}]],"label":"adult_care_tax_unit_role","optional":false,"required_scope":null},{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:person/model_required_boolean","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:person/puf_tax_itemization__batch_1","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:person/puf_tax_itemization__batch_2","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:person/puf_tax_itemization__batch_3","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:person/puf_tax_itemization__batch_4","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:person/puf_tax_itemization__batch_5","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:person/source_operator_child_support","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:person/source_operator_disability_benefits","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:person/source_operator_education_inputs","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:person/source_operator_immigration","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:person/source_operator_medicare_take_up","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:person/source_operator_retirement_contributions","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:person/source_operator_retirement_distributions","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:person/source_operator_weeks_unemployed","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:person/source_operator_wic_claim","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:person/source_operator_workers_compensation","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"person","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:spm_unit/source_operator_energy_subsidy","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"spm_unit","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"spm_unit","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"spm_unit","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]},{"operator":"transfer:tax_unit/puf_tax_itemization","requirements":[{"alternatives":[[{"column":"age","entity":"person","value_kind":"finite_numeric"}]],"label":"age","optional":false,"required_scope":null},{"alternatives":[[{"column":"is_female","entity":"person","value_kind":"finite_numeric"}]],"label":"is_female","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_model_config","entity":"tax_unit","value_kind":"non_null"}]],"label":"late_transfer_model_config","optional":false,"required_scope":null},{"alternatives":[[{"column":"@late_transfer_target_bank","entity":"tax_unit","value_kind":"non_null"}]],"label":"late_transfer_target_bank","optional":false,"required_scope":null},{"alternatives":[[{"column":"employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"A_EXPRRP","entity":"person","value_kind":"finite_numeric"}],[{"column":"A_LINENO","entity":"person","value_kind":"finite_numeric"}],[{"column":"is_household_head","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_household_head","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_interest_dividend_rental_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"estate_income","entity":"person","value_kind":"finite_numeric"},{"column":"non_qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"qualified_dividend_income","entity":"person","value_kind":"finite_numeric"},{"column":"rental_income","entity":"person","value_kind":"finite_numeric"},{"column":"tax_exempt_interest_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_interest_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_investment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_retirement_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"tax_exempt_private_pension_income","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_ira_distributions","entity":"person","value_kind":"finite_numeric"},{"column":"taxable_private_pension_income","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_retirement_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"self_employment_income_before_lsr","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_self_employment_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"acs_social_security_income","entity":"person","value_kind":"finite_numeric"}],[{"column":"social_security_dependents","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_disability","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_retirement","entity":"person","value_kind":"finite_numeric"},{"column":"social_security_survivors","entity":"person","value_kind":"finite_numeric"}]],"label":"optional_social_security_income","optional":true,"required_scope":null},{"alternatives":[[{"column":"tenure_type","entity":"person","value_kind":"non_null"}],[{"column":"spm_unit_tenure_type","entity":"spm_unit","value_kind":"non_null"}]],"label":"optional_tenure","optional":true,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"person","value_kind":"non_null"}]],"label":"resolved_person_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"tax_unit","value_kind":"non_null"}]],"label":"resolved_target_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"},{"column":"state_fips","entity":"household","value_kind":"finite_numeric"},{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}],[{"column":"state_fips","entity":"person","value_kind":"finite_numeric"}]],"label":"state_fips","optional":false,"required_scope":null},{"alternatives":[[{"column":"TYPEHUGQ","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:TYPEHUGQ","optional":false,"required_scope":"acs_source"},{"alternatives":[[{"column":"@us_spine_assembly_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:assembly_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_id","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_channel","entity":"family","value_kind":"non_null"}]],"label":"validated_structure:family_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"family_support_clone_index","entity":"family","value_kind":"finite_numeric"}]],"label":"validated_structure:family_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_spine_source_id","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_channel","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:household_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"household_support_clone_index","entity":"household","value_kind":"finite_numeric"}]],"label":"validated_structure:household_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_id","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_channel","entity":"marital_unit","value_kind":"non_null"}]],"label":"validated_structure:marital_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"marital_unit_support_clone_index","entity":"marital_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:marital_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_family_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_family_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_household_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_household_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_marital_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_marital_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spine_source_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spine_source_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_spm_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_channel","entity":"person","value_kind":"non_null"}]],"label":"validated_structure:person_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_support_clone_index","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"person_tax_unit_id","entity":"person","value_kind":"finite_numeric"}]],"label":"validated_structure:person_tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_puf_clone_attachment_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:puf_attachment_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"@resolved_weight","entity":"household","value_kind":"non_null"}]],"label":"validated_structure:resolved_household_weight","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_id","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_channel","entity":"spm_unit","value_kind":"non_null"}]],"label":"validated_structure:spm_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"spm_unit_support_clone_index","entity":"spm_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:spm_unit_support_clone_index","optional":false,"required_scope":null},{"alternatives":[[{"column":"@us_stacked_spine_manifest","entity":"frame","value_kind":"non_null"}]],"label":"validated_structure:stacked_manifest","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_id","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_id","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_channel","entity":"tax_unit","value_kind":"non_null"}]],"label":"validated_structure:tax_unit_support_channel","optional":false,"required_scope":null},{"alternatives":[[{"column":"tax_unit_support_clone_index","entity":"tax_unit","value_kind":"finite_numeric"}]],"label":"validated_structure:tax_unit_support_clone_index","optional":false,"required_scope":null}]}],"transfer_target_count":70,"waves":[["acs_pums_earnings_universe"],["primary_puf_qrf"],["source:impute_us_housing_assistance_to_puf_support","source:with_us_child_support_inputs","source:with_us_childcare_inputs","source:with_us_disability_benefits","source:with_us_energy_subsidy_input","source:with_us_immigration_inputs","source:with_us_medicare_take_up_input","source:with_us_pregnancy_inputs","source:with_us_prior_year_income_inputs","source:with_us_retirement_contribution_inputs","source:with_us_retirement_distribution_inputs","source:with_us_weeks_unemployed","source:with_us_workers_compensation","transfer:person/puf_tax_itemization__batch_1","transfer:person/puf_tax_itemization__batch_4","transfer:person/puf_tax_itemization__batch_5","transfer:tax_unit/puf_tax_itemization"],["source:with_us_adult_care_inputs","source:with_us_wic_claim_input","transfer:person/model_required_boolean","transfer:person/puf_tax_itemization__batch_2","transfer:person/puf_tax_itemization__batch_3","transfer:person/source_operator_child_support","transfer:person/source_operator_disability_benefits","transfer:person/source_operator_immigration","transfer:person/source_operator_medicare_take_up","transfer:person/source_operator_retirement_contributions","transfer:person/source_operator_retirement_distributions","transfer:person/source_operator_weeks_unemployed","transfer:person/source_operator_workers_compensation","transfer:spm_unit/source_operator_energy_subsidy"],["source:with_us_education_inputs","transfer:person/adult_care","transfer:person/source_operator_wic_claim"],["source_finalizer","transfer:person/source_operator_education_inputs"]]} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py b/packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py index 14baddbaa9..0c743e1cb6 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py @@ -145,6 +145,22 @@ "immigration_composition", } ) +#: The stacked operator order attested schema-9 pools sealed, before the +#: humanitarian-immigration composition gate joined it. A literal, not a +#: function of the current order: history must not move when the present does. +_SCHEMA9_STACKED_POOL_OPERATOR_ORDER = ( + "assemble_stacked_spine", + "assign_us_puma_ladder", + "prepare_multispine_source_inputs_for_clone", + "gap_fill_stacked_spine", + "run_stacked_late_producer_dag", + "prepare_stacked_tail_derivation", + "derive_multispine_pool_inputs", + "seed_multispine_pool_inputs", + "materialize_multispine_agreement_outputs", + "stacked_completeness_gate", + "by_origin_battery", +) _LEGACY_POOL_OPERATOR_ORDER = ( "assemble", "clone", @@ -1393,6 +1409,7 @@ def _load_authenticated_us_multispine_pool_manifest( diagnostics_path=diagnostics_path, manifest_agreement_gate=manifest_agreement_gate, diagnostics_agreement_gate=diagnostics_agreement_gate, + require_canonical_gate_set=legacy_worker_authentication is None, ) if diagnostics_agreement_gate != manifest_agreement_gate: raise ValueError( @@ -1437,7 +1454,12 @@ def _validate_stacked_late_dag_manifest_binding( if manifest.get("pipeline") != "us-stacked-pool": return - if manifest.get("operator_order") != list(US_STACKED_POOL_OPERATOR_ORDER): + expected_operator_order = ( + US_STACKED_POOL_OPERATOR_ORDER + if legacy_worker_authentication is None + else _SCHEMA9_STACKED_POOL_OPERATOR_ORDER + ) + if manifest.get("operator_order") != list(expected_operator_order): raise ValueError( f"US stacked pool manifest {manifest_path} does not bind the " "canonical late-DAG operator order." @@ -2679,6 +2701,7 @@ def _require_matching_terminal_gate_aliases( diagnostics_path: Path, manifest_agreement_gate: Mapping[str, object], diagnostics_agreement_gate: Mapping[str, object], + require_canonical_gate_set: bool = True, ) -> None: """Bind stacked terminal gates to the legacy compatibility aliases. @@ -2729,6 +2752,11 @@ def _require_matching_terminal_gate_aliases( diagnostics_terminal_gates.get("gates"), label=(f"US stacked pool diagnostics {diagnostics_path}.terminal_gates.gates"), ) + # The canonical gate set arrived with manifest schema 10. Attested + # schema-9 pools sealed whatever terminal gates their build ran, so only + # the alias equalities above apply to them. + if not require_canonical_gate_set: + return for label, gates in ( (f"US stacked pool manifest {manifest_path}", manifest_gates), (f"US stacked pool diagnostics {diagnostics_path}", diagnostics_gates), diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py b/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py index 0a0eb61df6..a9d844592e 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py @@ -5236,10 +5236,16 @@ def validate_stacked_post_puf_transfer_receipt( owner_receipt = target_receipt.get("post_transfer_calibration") spec = expected_calibrations.get(target_key) target = target_key.rsplit("/", 1)[1] - is_immigration = _is_immigration_transfer_target( - entity=group.entity, - family=group.family, - target=target, + # Attested schema-9 pools sealed their immigration targets as + # ordinary transfer targets: the paired reconciliation and QRF + # evidence did not exist, so requiring it would reject history. + is_immigration = ( + legacy_worker_authentication is None + and _is_immigration_transfer_target( + entity=group.entity, + family=group.family, + target=target, + ) ) if is_immigration: group_immigration_reconciliations.append( diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_producer_registry.py b/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_producer_registry.py index 6847c895c7..b83dcd8cfc 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_producer_registry.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_producer_registry.py @@ -21,6 +21,8 @@ from collections import defaultdict from collections.abc import Mapping, Sequence from dataclasses import dataclass +from functools import cache +from importlib.resources import files from types import MappingProxyType from microcosm.build.us_runtime.acs_income_universe import ( @@ -2215,26 +2217,38 @@ def us_late_producer_schedule_receipt() -> Mapping[str, object]: ) +#: Content hash of the schema-16 schedule that attested schema-9 pools sealed. +#: History is data: the receipt is loaded from a frozen resource and checked +#: against this digest, never rebuilt from the current registry, so a later +#: contract change (microcosm #767 changed the immigration stage's inputs and +#: transfer evidence) cannot silently redefine what a historical pool sealed. +LEGACY_SCHEMA16_LATE_PRODUCER_SCHEDULE_PAYLOAD_SHA256 = ( + "02e618cc656eb39990ed99dca2b30a52794e01e2b06a3c2df87ca4a7d85ab086" +) +_LEGACY_SCHEDULE_DERIVED_KEYS = frozenset( + { + "payload_sha256", + "producer_count", + "source_producer_count", + "transfer_group_count", + "transfer_target_count", + "status", + } +) + + +@cache def legacy_us_late_producer_schedule_receipt() -> Mapping[str, object]: - """Reconstruct the exact schema-16 schedule for attested legacy scoring.""" + """Load the frozen schema-16 schedule for attested legacy scoring.""" - receipt = json.loads(json.dumps(dict(us_late_producer_schedule_receipt()))) - receipt["schema_version"] = 16 - contract = receipt["execution_receipt_contract"] - contract["version"] = 3 - contract["transition_authority"]["version"] = 1 + resource = files("microcosm.build.us").joinpath( + "legacy_schema16_late_producer_schedule.json" + ) + receipt = json.loads(resource.read_text(encoding="utf-8")) payload = { key: value for key, value in receipt.items() - if key - not in { - "payload_sha256", - "producer_count", - "source_producer_count", - "transfer_group_count", - "transfer_target_count", - "status", - } + if key not in _LEGACY_SCHEDULE_DERIVED_KEYS } canonical = json.dumps( payload, @@ -2242,5 +2256,13 @@ def legacy_us_late_producer_schedule_receipt() -> Mapping[str, object]: separators=(",", ":"), allow_nan=False, ).encode("utf-8") - receipt["payload_sha256"] = hashlib.sha256(canonical).hexdigest() + observed = hashlib.sha256(canonical).hexdigest() + if ( + observed != LEGACY_SCHEMA16_LATE_PRODUCER_SCHEDULE_PAYLOAD_SHA256 + or receipt.get("payload_sha256") != observed + ): + raise ValueError( + "frozen schema-16 late-producer schedule does not match its " + "sealed content hash." + ) return MappingProxyType(receipt) From 0dcf98c9456417480cecdcd2d507006830f1c326 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 19 Sep 2026 08:35:16 -0400 Subject: [PATCH 23/49] Pin the contract attested schema-9 pools sealed Six engine-free tests: the legacy schedule and v11 authority hash to the values real pools carry (computed on main at the PR base), the payload hash is a genuine content hash, history demonstrably no longer follows the current registry, a tampered frozen resource is refused, and the schema-9 operator order is the literal 11-step order. Refs #767 Co-Authored-By: Claude Fable 5.1 --- .../tests/test_us_legacy_schema9_contract.py | 117 ++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 packages/microcosm-build/tests/test_us_legacy_schema9_contract.py diff --git a/packages/microcosm-build/tests/test_us_legacy_schema9_contract.py b/packages/microcosm-build/tests/test_us_legacy_schema9_contract.py new file mode 100644 index 0000000000..d74915d04b --- /dev/null +++ b/packages/microcosm-build/tests/test_us_legacy_schema9_contract.py @@ -0,0 +1,117 @@ +"""The contract attested schema-9 stacked pools sealed stays frozen (#767). + +Authenticated legacy scoring compares a historical pool's sealed schedule and +authority against reconstructions. Those reconstructions used to be derived +from the *current* late-producer registry, so any contract change silently +redefined history and every real schema-9 pool would be refused — while the +positive tests, which downgrade a freshly built artifact, kept passing. These +pins are the regression: the historical content is data, and its hashes are +the ones real pools carry. +""" + +from __future__ import annotations + +import hashlib +import json + +import pytest + +from microcosm.build.us_runtime import h5_io, stacked_spine +from microcosm.build.us_runtime import us_late_producer_registry as registry + +#: What pools built before manifest schema 10 actually sealed, computed on +#: main at 7c4a5ac0 (the base of microcosm #779) and reproduced here. +HISTORICAL_SCHEDULE_PAYLOAD_SHA256 = ( + "02e618cc656eb39990ed99dca2b30a52794e01e2b06a3c2df87ca4a7d85ab086" +) +HISTORICAL_V11_AUTHORITY_SHA256 = ( + "e660a8ce42b69a39d29c5f0ec37264bc69d61b03f27adc386336ec8889531bb2" +) + + +def test_legacy_schedule_is_the_sealed_historical_content() -> None: + receipt = registry.legacy_us_late_producer_schedule_receipt() + assert receipt["payload_sha256"] == HISTORICAL_SCHEDULE_PAYLOAD_SHA256 + assert ( + registry.LEGACY_SCHEMA16_LATE_PRODUCER_SCHEDULE_PAYLOAD_SHA256 + == HISTORICAL_SCHEDULE_PAYLOAD_SHA256 + ) + assert receipt["schema_version"] == 16 + contract = receipt["execution_receipt_contract"] + assert contract["version"] == 3 + assert contract["transition_authority"]["version"] == 1 + + +def test_legacy_schedule_payload_hash_is_a_real_content_hash() -> None: + receipt = dict(registry.legacy_us_late_producer_schedule_receipt()) + payload = { + key: value + for key, value in receipt.items() + if key not in registry._LEGACY_SCHEDULE_DERIVED_KEYS + } + canonical = json.dumps( + payload, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode("utf-8") + assert hashlib.sha256(canonical).hexdigest() == HISTORICAL_SCHEDULE_PAYLOAD_SHA256 + + +def test_legacy_schedule_does_not_follow_the_current_registry() -> None: + # The current contract differs from history exactly where #767 changed + # it; if these ever coincide the frozen resource has stopped mattering. + current = registry.us_late_producer_schedule_receipt() + legacy = registry.legacy_us_late_producer_schedule_receipt() + assert current["payload_sha256"] != legacy["payload_sha256"] + + def immigration_inputs(receipt) -> set[str]: + inventory = next( + row + for row in receipt["source_input_inventories"] + if row["operator"] == "with_us_immigration_inputs" + ) + return {requirement["label"] for requirement in inventory["requirements"]} + + assert {"raw_person:WSAL_VAL", "raw_person:SEMP_VAL"} <= immigration_inputs(legacy) + assert "raw_person:A_LFSR" not in immigration_inputs(legacy) + assert "raw_person:A_LFSR" in immigration_inputs(current) + assert legacy["execution_receipt_contract"]["top_binding"] == ( + "entry_and_output_frame_sha256_execution_chain_source_completion_" + "and_nineteen_transfer_groups" + ) + + +def test_legacy_authority_is_the_sealed_v11_authority() -> None: + receipt = stacked_spine._legacy_stacked_authority_receipt() + assert receipt["sha256"] == HISTORICAL_V11_AUTHORITY_SHA256 + + +def test_tampered_frozen_schedule_is_refused(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + registry, + "LEGACY_SCHEMA16_LATE_PRODUCER_SCHEDULE_PAYLOAD_SHA256", + "0" * 64, + ) + registry.legacy_us_late_producer_schedule_receipt.cache_clear() + try: + with pytest.raises(ValueError, match="sealed content hash"): + registry.legacy_us_late_producer_schedule_receipt() + finally: + registry.legacy_us_late_producer_schedule_receipt.cache_clear() + + +def test_schema9_operator_order_is_the_literal_historical_order() -> None: + historical = h5_io._SCHEMA9_STACKED_POOL_OPERATOR_ORDER + assert historical == ( + "assemble_stacked_spine", + "assign_us_puma_ladder", + "prepare_multispine_source_inputs_for_clone", + "gap_fill_stacked_spine", + "run_stacked_late_producer_dag", + "prepare_stacked_tail_derivation", + "derive_multispine_pool_inputs", + "seed_multispine_pool_inputs", + "materialize_multispine_agreement_outputs", + "stacked_completeness_gate", + "by_origin_battery", + ) + assert "us_immigration_composition_gate" not in historical + assert h5_io.US_STACKED_POOL_OPERATOR_ORDER[-1] == "us_immigration_composition_gate" From 21f0c38b32e642041263ee0004b70f99ba425eba Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 19 Sep 2026 08:40:44 -0400 Subject: [PATCH 24/49] Declare the frozen schedule in the US package closure; historical test fixture load_bundle enforces a closed file set on the US package, so the frozen schema-16 schedule joins LEGACY_RESOURCE_PATHS and country_package.json. The legacy-pool test helper now installs that historical schedule, the 11-step operator order, and the ordinary immigration transfer receipts a schema-9 pool sealed, instead of version-downgrading today's contract. Refs #767 Co-Authored-By: Claude Fable 5.1 --- .../microcosm/build/us/country_package.json | 5 ++ .../tests/test_us_multispine_pool_h5_io.py | 61 +++++++++++++------ tools/generate_us_bundle_from_constants.py | 1 + 3 files changed, 49 insertions(+), 18 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/us/country_package.json b/packages/microcosm-build/src/microcosm/build/us/country_package.json index 34e30f6dca..9fe476f1e9 100644 --- a/packages/microcosm-build/src/microcosm/build/us/country_package.json +++ b/packages/microcosm-build/src/microcosm/build/us/country_package.json @@ -82,6 +82,11 @@ "kind": "legacy_json", "schema_id": "legacy_json" }, + { + "path": "legacy_schema16_late_producer_schedule.json", + "kind": "legacy_json", + "schema_id": "legacy_json" + }, { "path": "obbba_reforms.json", "kind": "legacy_json", diff --git a/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py b/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py index 97bf32f163..413972d202 100644 --- a/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py +++ b/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py @@ -17,6 +17,7 @@ import microcosm.build.us_runtime.immigration as immigration_runtime import microcosm.build.us_runtime.post_transfer_calibration as post_transfer_calibration_runtime import microcosm.build.us_runtime.stacked_spine as stacked_spine_module +import microcosm.build.us_runtime.us_late_producer_registry as late_producer_registry_module import microcosm.build.us_runtime.worker_identity as worker_identity_module from microcosm.build.frame_checkpoint import ( load_frame_checkpoint, @@ -1574,6 +1575,34 @@ def _canonical_stacked_late_dag_receipt( return receipt +def _strip_immigration_transfer_evidence(transfer: dict[str, object]) -> None: + """Restore the pre-schema-10 shape of the immigration transfer targets.""" + + branch_only = ( + "qrf_pattern_evidence", + "post_transfer_reconciliation", + "producer_roles", + "producer_rows", + ) + + def historical(target_key: str, receipt: dict[str, object]) -> None: + entity, family, target = target_key.split("/", 2) + if not stacked_spine_module._is_immigration_transfer_target( + entity=entity, family=family, target=target + ): + return + for key in branch_only: + receipt.pop(key, None) + receipt["authorized_null_rows"] = 0 + receipt["imputed_rows"] = 0 + + for group in transfer["groups"].values(): + for target_key, receipt in group["targets"].items(): + historical(target_key, receipt) + for target_key, receipt in transfer["targets"].items(): + historical(target_key, receipt) + + def _rewrite_as_legacy_relocated_worker_pool( manifest_path: Path, ) -> tuple[dict[str, object], dict[str, object]]: @@ -1659,25 +1688,21 @@ def _rewrite_as_legacy_relocated_worker_pool( available ) - schedule = dag["producer_schedule"] - schedule["schema_version"] = 16 - schedule["execution_receipt_contract"]["version"] = 3 - schedule["execution_receipt_contract"]["transition_authority"]["version"] = 1 - schedule_payload = { - key: value - for key, value in schedule.items() - if key - not in { - "payload_sha256", - "producer_count", - "source_producer_count", - "transfer_group_count", - "transfer_target_count", - "status", - } - } - schedule["payload_sha256"] = _json_sha256(schedule_payload) + # A schema-9 pool sealed the historical schedule, not a version-downgraded + # copy of today's: install the frozen content real pools carry, and give + # the immigration targets the ordinary transfer receipts they had before + # the paired reconciliation and QRF evidence existed. + schedule = json.loads( + json.dumps( + dict( + late_producer_registry_module.legacy_us_late_producer_schedule_receipt() + ) + ) + ) + dag["producer_schedule"] = schedule dag["post_puf_transfer"]["producer_schedule"] = json.loads(json.dumps(schedule)) + _strip_immigration_transfer_evidence(dag["post_puf_transfer"]) + manifest["operator_order"] = list(h5_io._SCHEMA9_STACKED_POOL_OPERATOR_ORDER) dag["post_puf_transfer"]["authority"] = ( stacked_spine_module._legacy_stacked_authority_receipt() ) diff --git a/tools/generate_us_bundle_from_constants.py b/tools/generate_us_bundle_from_constants.py index 3b08ef80c1..3579f7fd3c 100644 --- a/tools/generate_us_bundle_from_constants.py +++ b/tools/generate_us_bundle_from_constants.py @@ -140,6 +140,7 @@ def _domain_builders() -> dict[str, Callable[[], object]]: "ecps_parity_reference.json", "federal_eitc_by_state.json", "fiscal_target_references.json", + "legacy_schema16_late_producer_schedule.json", "obbba_reforms.json", "puf_aggregate_record_disaggregation.json", "release_input_coverage_manifest.json", From 8a78caa5c7cba906477672b16c75dd1b7268ff17 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 19 Sep 2026 08:46:13 -0400 Subject: [PATCH 25/49] Carry the historical immigration transfer receipts into the execution rows The legacy fixture strips the schema-10 immigration evidence from the aggregate transfer proof; the matching execution rows must seal the same historical group receipts before the chain is re-signed. Co-Authored-By: Claude Fable 5.1 --- .../microcosm-build/tests/test_us_multispine_pool_h5_io.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py b/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py index 413972d202..34ea02cc61 100644 --- a/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py +++ b/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py @@ -1702,6 +1702,10 @@ def _rewrite_as_legacy_relocated_worker_pool( dag["producer_schedule"] = schedule dag["post_puf_transfer"]["producer_schedule"] = json.loads(json.dumps(schedule)) _strip_immigration_transfer_evidence(dag["post_puf_transfer"]) + for row in dag["execution"]: + group_receipt = dag["post_puf_transfer"]["groups"].get(row["producer"]) + if group_receipt is not None: + row["producer_receipt"] = json.loads(json.dumps(group_receipt)) manifest["operator_order"] = list(h5_io._SCHEMA9_STACKED_POOL_OPERATOR_ORDER) dag["post_puf_transfer"]["authority"] = ( stacked_spine_module._legacy_stacked_authority_receipt() From 5a589bd8ae4793edee9234302b1e44aa5e64979f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 19 Sep 2026 08:53:22 -0400 Subject: [PATCH 26/49] Refuse a version-downgraded current contract as a schema-9 pool Negative pair for the historical fixture: today's schedule, operator order, and immigration evidence under lowered version numbers is refused at the operator order, and still refused at the sealed schedule once the historical order is borrowed. Co-Authored-By: Claude Fable 5.1 --- .../tests/test_us_multispine_pool_h5_io.py | 100 +++++++++++++++--- 1 file changed, 87 insertions(+), 13 deletions(-) diff --git a/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py b/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py index 34ea02cc61..bd90cd28f8 100644 --- a/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py +++ b/packages/microcosm-build/tests/test_us_multispine_pool_h5_io.py @@ -1605,8 +1605,15 @@ def historical(target_key: str, receipt: dict[str, object]) -> None: def _rewrite_as_legacy_relocated_worker_pool( manifest_path: Path, + *, + historical_contract: bool = True, ) -> tuple[dict[str, object], dict[str, object]]: - """Re-sign the tiny fixture with the frozen schema-9 worker binding.""" + """Re-sign the tiny fixture with the frozen schema-9 worker binding. + + ``historical_contract=False`` keeps today's schedule, operator order, and + immigration evidence and only lowers the version numbers: the forgery a + real schema-9 pool can never be, which the loader must refuse. + """ manifest = json.loads(manifest_path.read_text(encoding="utf-8")) dag = manifest["stage_receipts"]["impute"]["stacked_late_producer_dag"] @@ -1692,21 +1699,42 @@ def _rewrite_as_legacy_relocated_worker_pool( # copy of today's: install the frozen content real pools carry, and give # the immigration targets the ordinary transfer receipts they had before # the paired reconciliation and QRF evidence existed. - schedule = json.loads( - json.dumps( - dict( - late_producer_registry_module.legacy_us_late_producer_schedule_receipt() + if historical_contract: + schedule = json.loads( + json.dumps( + dict( + late_producer_registry_module.legacy_us_late_producer_schedule_receipt() + ) ) ) - ) - dag["producer_schedule"] = schedule + dag["producer_schedule"] = schedule + _strip_immigration_transfer_evidence(dag["post_puf_transfer"]) + for row in dag["execution"]: + group_receipt = dag["post_puf_transfer"]["groups"].get(row["producer"]) + if group_receipt is not None: + row["producer_receipt"] = json.loads(json.dumps(group_receipt)) + manifest["operator_order"] = list(h5_io._SCHEMA9_STACKED_POOL_OPERATOR_ORDER) + else: + schedule = dag["producer_schedule"] + schedule["schema_version"] = 16 + schedule["execution_receipt_contract"]["version"] = 3 + schedule["execution_receipt_contract"]["transition_authority"]["version"] = 1 + schedule["payload_sha256"] = _json_sha256( + { + key: value + for key, value in schedule.items() + if key + not in { + "payload_sha256", + "producer_count", + "source_producer_count", + "transfer_group_count", + "transfer_target_count", + "status", + } + } + ) dag["post_puf_transfer"]["producer_schedule"] = json.loads(json.dumps(schedule)) - _strip_immigration_transfer_evidence(dag["post_puf_transfer"]) - for row in dag["execution"]: - group_receipt = dag["post_puf_transfer"]["groups"].get(row["producer"]) - if group_receipt is not None: - row["producer_receipt"] = json.loads(json.dumps(group_receipt)) - manifest["operator_order"] = list(h5_io._SCHEMA9_STACKED_POOL_OPERATOR_ORDER) dag["post_puf_transfer"]["authority"] = ( stacked_spine_module._legacy_stacked_authority_receipt() ) @@ -2621,6 +2649,52 @@ def test_scoring_loader_accepts_legacy_worker_alias_relocation_only( ) +def test_scoring_loader_refuses_version_downgraded_current_contract( + tmp_path: Path, +) -> None: + """Lowered version numbers do not make today's contract a schema-9 pool.""" + + pytest.importorskip("tables") + manifest_path = _write_gate_failed_pool(tmp_path) + recorded_worker, semantic_identity = _rewrite_as_legacy_relocated_worker_pool( + manifest_path, + historical_contract=False, + ) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + dag = manifest["stage_receipts"]["impute"]["stacked_late_producer_dag"] + assert dag["producer_schedule"]["payload_sha256"] != ( + late_producer_registry_module.LEGACY_SCHEMA16_LATE_PRODUCER_SCHEDULE_PAYLOAD_SHA256 + ) + attestation_path = _write_legacy_worker_attestation( + manifest_path, + recorded_worker=recorded_worker, + semantic_identity=semantic_identity, + ) + + with pytest.raises(ValueError, match="canonical late-DAG operator order"): + load_authenticated_us_multispine_pool_for_scoring( + manifest_path, + expected_manifest_sha256=_sha256(manifest_path), + worker_identity_attestation=attestation_path, + ) + + # Borrowing the historical operator order does not rescue it: the sealed + # schedule is still today's content under yesterday's version numbers. + manifest["operator_order"] = list(h5_io._SCHEMA9_STACKED_POOL_OPERATOR_ORDER) + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + attestation_path = _write_legacy_worker_attestation( + manifest_path, + recorded_worker=recorded_worker, + semantic_identity=semantic_identity, + ) + with pytest.raises(ValueError, match="DAG schedule is not canonical"): + load_authenticated_us_multispine_pool_for_scoring( + manifest_path, + expected_manifest_sha256=_sha256(manifest_path), + worker_identity_attestation=attestation_path, + ) + + @pytest.mark.usefixtures("live_worker_identity") def test_scoring_loader_requires_complete_schema_nine_stacked_envelope( tmp_path: Path, From ae8364a636d3b46e4fa128d0d000bdc18d4983c8 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 19 Sep 2026 09:06:35 -0400 Subject: [PATCH 27/49] Re-pin the field-usage ledger for the frozen schedule resource The country manifest gains one resource entry (three fields): the configuration universe moves 42,540 -> 42,543 and the country_manifest pointer claim 98 -> 101. Co-Authored-By: Claude Fable 5.1 --- .../src/microcosm/build/spec_engine/field_usage.py | 8 ++++---- .../tests/test_spec_engine_coverage_tool.py | 10 +++++----- .../tests/test_spec_engine_field_usage.py | 10 +++++----- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py index 499e4d8403..c998e2d97e 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py @@ -26,9 +26,9 @@ ) from .schemas import load_schema_registry -EXPECTED_AUTHORED_FIELD_COUNT = 32_521 +EXPECTED_AUTHORED_FIELD_COUNT = 32_524 EXPECTED_RESOLVED_BINDING_FIELD_COUNT = 10_019 -EXPECTED_CONFIGURATION_FIELD_COUNT = 42_540 +EXPECTED_CONFIGURATION_FIELD_COUNT = 42_543 class FieldUsageError(AssertionError): @@ -373,8 +373,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "64182b6be1ea6d95bff345b30a2aa046b6fa7e8ee61a282b225dfa49c28fbfdc", ), "country_manifest": ( - 98, - "cbbda6d2d245f04325c0b5a7b986cb71d24d6e3c81a7a3af1544de7f75be2a1f", + 101, + "be7cb568758bd3374612d2bcc37ea96a8269ffeac53fe0d10dc62195471b0d2e", ), "generated_authorities": ( 8_606, diff --git a/packages/microcosm-build/tests/test_spec_engine_coverage_tool.py b/packages/microcosm-build/tests/test_spec_engine_coverage_tool.py index 932636266a..d7062f65b0 100644 --- a/packages/microcosm-build/tests/test_spec_engine_coverage_tool.py +++ b/packages/microcosm-build/tests/test_spec_engine_coverage_tool.py @@ -88,22 +88,22 @@ def test_us_coverage_is_exact_complete_and_honest( assert_coverage_complete(coverage_report) assert coverage_report["status"] == "pass" fields = coverage_report["field_usage"] - assert fields["configuration_field_count"] == 42_540 - assert fields["authored_normative_field_count"] == 32_521 + assert fields["configuration_field_count"] == 42_543 + assert fields["authored_normative_field_count"] == 32_524 assert fields["resolved_binding_field_count"] == 10_019 - assert fields["consumed_field_count"] == 42_540 + assert fields["consumed_field_count"] == 42_543 assert fields["unused_field_count"] == 0 assert fields["multiple_primary_use_field_count"] == 0 assert fields["claim_count"] == 49 assert fields["mode_counts"] == { "legacy_behavior": 14_020, "compiler_semantic": 28_069, - "front_end_validation": 348, + "front_end_validation": 351, "identity_only": 103, } assert fields["generation0_effect_counts"] == { "legacy_behavior": 38_574, - "no_generation0_effect": 3_966, + "no_generation0_effect": 3_969, } inventory = coverage_report["inventory_coverage"] diff --git a/packages/microcosm-build/tests/test_spec_engine_field_usage.py b/packages/microcosm-build/tests/test_spec_engine_field_usage.py index 752704688e..002dc37a65 100644 --- a/packages/microcosm-build/tests/test_spec_engine_field_usage.py +++ b/packages/microcosm-build/tests/test_spec_engine_field_usage.py @@ -92,22 +92,22 @@ def _mutated_bundle( def test_exact_complete_ledger_has_one_primary_mode_per_pointer(field_ledger) -> None: - assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_540 + assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_543 assert field_ledger.source_counts == { - "authored": 32_521, + "authored": 32_524, "resolved_bindings": 10_019, } assert field_ledger.mode_counts == { "legacy_behavior": 14_020, "compiler_semantic": 28_069, - "front_end_validation": 348, + "front_end_validation": 351, "identity_only": 103, } assert field_ledger.generation0_effect_counts == { "legacy_behavior": 38_574, - "no_generation0_effect": 3_966, + "no_generation0_effect": 3_969, } - assert len({field.pointer for field in field_ledger.fields}) == 42_540 + assert len({field.pointer for field in field_ledger.fields}) == 42_543 def test_eligibility_concepts_are_validation_not_generation0_behavior( From ced08f2da55f03ecfcdaf0651e40c5bbd057f072 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 19 Sep 2026 09:16:05 -0400 Subject: [PATCH 28/49] Re-pin seed protocol digests and regenerate coverage evidence Co-Authored-By: Claude Fable 5.1 --- docs/evidence/spec-engine/us-f0-coverage.json | 32 +++++++++---------- .../build/spec_engine/inventory_coverage.py | 4 +-- tools/spec_engine_coverage.py | 2 +- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index a1edaeb857..713b8c6fa6 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -6,7 +6,7 @@ "country": "us", "documentation_sha256": "5beb9497d74b7407fbb3fff888007d089423122db504171ce5e4b337a79e3dcd", "field_usage": { - "authored_normative_field_count": 32521, + "authored_normative_field_count": 32524, "claim_count": 49, "claims": [ { @@ -16,8 +16,8 @@ "legacy_sinks": [], "mode": "front_end_validation", "pointer_class": "all", - "pointer_count": 98, - "pointer_sha256": "cbbda6d2d245f04325c0b5a7b986cb71d24d6e3c81a7a3af1544de7f75be2a1f", + "pointer_count": 101, + "pointer_sha256": "be7cb568758bd3374612d2bcc37ea96a8269ffeac53fe0d10dc62195471b0d2e", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/country_package.json", @@ -772,20 +772,20 @@ "verifier": "vintages" } ], - "configuration_field_count": 42540, - "consumed_field_count": 42540, + "configuration_field_count": 42543, + "consumed_field_count": 42543, "generation0_effect_counts": { "legacy_behavior": 38574, - "no_generation0_effect": 3966 + "no_generation0_effect": 3969 }, "mode_counts": { "compiler_semantic": 28069, - "front_end_validation": 348, + "front_end_validation": 351, "identity_only": 103, "legacy_behavior": 14020 }, "multiple_primary_use_field_count": 0, - "pointer_inventory_sha256": "370e1958448fed48a635701439905d2145c3ae5bf3878adc1406a56a381a303b", + "pointer_inventory_sha256": "b5fb54a54e5bb150ad8d6a980daa0e23fa294ef8e9c81f5f6fcde7bb68a6af63", "resolved_binding_field_count": 10019, "unused_field_count": 0 }, @@ -1669,13 +1669,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "674f1b222d31c925a9914791bb23fcfcfb6039c56261ae47a13cf735b29de393", - "protocol_sha256": "7883cc956c11fe43fb0dedbe0bedb02b9ad804696f5551ee5d2c7a250b0abdbe" + "map_sha256": "ac74a667d89e4cab9b9253b9a7817ae57fbda749eee903065c147c42c3b44967", + "protocol_sha256": "b4ea35f32e38570896aa08e25515a6f3c50e13fd8db18146dd5322aab18629cb" }, "failures": [], "observed": { - "map_sha256": "674f1b222d31c925a9914791bb23fcfcfb6039c56261ae47a13cf735b29de393", - "protocol_sha256": "7883cc956c11fe43fb0dedbe0bedb02b9ad804696f5551ee5d2c7a250b0abdbe" + "map_sha256": "ac74a667d89e4cab9b9253b9a7817ae57fbda749eee903065c147c42c3b44967", + "protocol_sha256": "b4ea35f32e38570896aa08e25515a6f3c50e13fd8db18146dd5322aab18629cb" }, "status": "covered" }, @@ -1690,7 +1690,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "7883cc956c11fe43fb0dedbe0bedb02b9ad804696f5551ee5d2c7a250b0abdbe", + "implementation_sha256": "b4ea35f32e38570896aa08e25515a6f3c50e13fd8db18146dd5322aab18629cb", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1711,7 +1711,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "7883cc956c11fe43fb0dedbe0bedb02b9ad804696f5551ee5d2c7a250b0abdbe", + "implementation_sha256": "b4ea35f32e38570896aa08e25515a6f3c50e13fd8db18146dd5322aab18629cb", "protocol": "legacy-v1", "streams": [ "build_model", @@ -2612,7 +2612,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "7f1d3106448c57baf7205ee4c024cab986de0ed7faef7f98f296d7c79c792587" + "spec_sha256": "9f4f0af295277cc546e164fbb0d50dcc67e55b4a5dabe0e7271dd252843dd258" } }, "report_schema_version": 3, @@ -2622,7 +2622,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "7f1d3106448c57baf7205ee4c024cab986de0ed7faef7f98f296d7c79c792587" + "spec_sha256": "9f4f0af295277cc546e164fbb0d50dcc67e55b4a5dabe0e7271dd252843dd258" }, "status": "pass" } diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 764dcba78f..ab91b9c750 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -372,8 +372,8 @@ "late_schedule": "88bc9243a3518982ae951c3de21bd55877e296ce4fcb183b9bee420d3a684b10", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "987b501c695e31f45521c4a178528f75ab3df22c09bc407b182213b2de99ee57", - "seed_map": "674f1b222d31c925a9914791bb23fcfcfb6039c56261ae47a13cf735b29de393", - "seed_protocol": "7883cc956c11fe43fb0dedbe0bedb02b9ad804696f5551ee5d2c7a250b0abdbe", + "seed_map": "ac74a667d89e4cab9b9253b9a7817ae57fbda749eee903065c147c42c3b44967", + "seed_protocol": "b4ea35f32e38570896aa08e25515a6f3c50e13fd8db18146dd5322aab18629cb", "source_manifest": "9df42433ba92c5cad75f1df2394f8b64fb25ed027b5a5e6ad383874345112afa", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", diff --git a/tools/spec_engine_coverage.py b/tools/spec_engine_coverage.py index ab17a17a8e..dc9090ab97 100644 --- a/tools/spec_engine_coverage.py +++ b/tools/spec_engine_coverage.py @@ -41,7 +41,7 @@ REPORT_SCHEMA_VERSION = 3 EXPECTED_POINTER_INVENTORY_SHA256 = ( - "370e1958448fed48a635701439905d2145c3ae5bf3878adc1406a56a381a303b" + "b5fb54a54e5bb150ad8d6a980daa0e23fa294ef8e9c81f5f6fcde7bb68a6af63" ) DEFAULT_REPORT_PATH = ( Path(__file__).resolve().parents[1] From 2357500388d87d5daa54c501879bf61cca36347f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 19 Sep 2026 09:19:14 -0400 Subject: [PATCH 29/49] Re-pin the loader golden vector and the US spec sha mirror Co-Authored-By: Claude Fable 5.1 --- packages/microcosm-build/tests/test_spec_engine_loader.py | 2 +- packages/microcosm-build/tests/test_us_multispine_pool_tool.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/microcosm-build/tests/test_spec_engine_loader.py b/packages/microcosm-build/tests/test_spec_engine_loader.py index bd4d9b3196..35975b26c0 100644 --- a/packages/microcosm-build/tests/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/test_spec_engine_loader.py @@ -236,7 +236,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # Pin the domain separator, normalization rules, schema-set receipt, and # exact normative projection as one reviewable golden vector. assert first.spec_sha256 == ( - "cbc3321df143a3fab6b100eab503190e9b84ade88100486a087e1292f94beed6" + "47608ddd982bae51af7dd87e4d5ec49e60fb341674e976e1e00d1c857c112ff4" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") diff --git a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py index 9b825fc9d3..c2ee8ac6bc 100644 --- a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py @@ -2933,7 +2933,7 @@ def capture_equality(expected: object, actual: object) -> None: "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "7f1d3106448c57baf7205ee4c024cab986de0ed7faef7f98f296d7c79c792587", + "spec_sha256": "9f4f0af295277cc546e164fbb0d50dcc67e55b4a5dabe0e7271dd252843dd258", }, } From 185f792c697bff2ca3684eaf875756ef5342b1bc Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 19 Sep 2026 09:20:13 -0400 Subject: [PATCH 30/49] Document the frozen contract attested schema-9 pools are held to Co-Authored-By: Claude Fable 5.1 --- ...s-humanitarian-immigration-statuses-767.added.md | 2 +- docs/us-multispine-operator-ordering.md | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/changelog.d/us-humanitarian-immigration-statuses-767.added.md b/changelog.d/us-humanitarian-immigration-statuses-767.added.md index 86176b426a..2765828feb 100644 --- a/changelog.d/us-humanitarian-immigration-statuses-767.added.md +++ b/changelog.d/us-humanitarian-immigration-statuses-767.added.md @@ -1 +1 @@ -Impute humanitarian and temporary-protection immigration statuses (REFUGEE, ASYLEE, PAROLED_ONE_YEAR per program origin, TPS per country) in the US immigration stage, drawn to cited DHS/OHSS/CBP/CRS stocks, so the enacted H.R.1 §71109/§71301/§71302 and SNAP §10108 eligibility channels stop computing $0; gate each category against its target and add four release-blocking reform-coverage probes. +Impute humanitarian and temporary-protection immigration statuses (REFUGEE, ASYLEE, PAROLED_ONE_YEAR per program origin, TPS per country) in the US immigration stage, drawn to cited DHS/OHSS/CBP/CRS stocks, so the enacted H.R.1 §71109/§71301/§71302 and SNAP §10108 eligibility channels stop computing $0; gate each category against its target and add four release-blocking reform-coverage probes. Attested schema-9 pools keep authenticating for scoring against the frozen contract they sealed. diff --git a/docs/us-multispine-operator-ordering.md b/docs/us-multispine-operator-ordering.md index 810086769c..5fae10429c 100644 --- a/docs/us-multispine-operator-ordering.md +++ b/docs/us-multispine-operator-ordering.md @@ -594,6 +594,19 @@ authorization tuple (`gate`, `plan_sha256`, `prompt_sha256`, `checklist_sha256`, and `evidence_sha256`) checked as data; this boundary does not claim public-key or cryptographic signature verification. +An attested schema-9 pool is validated against the contract it sealed, not a +reconstruction from today's registry. The late-producer schedule it must carry +is the frozen resource `us/legacy_schema16_late_producer_schedule.json`, loaded +only if its recomputed payload SHA-256 equals +`02e618cc656eb39990ed99dca2b30a52794e01e2b06a3c2df87ca4a7d85ab086`; its operator +order is the literal eleven-step order ending at `by_origin_battery`, before +`us_immigration_composition_gate` existed; and its immigration transfer targets +are replayed as ordinary transfer receipts, without the paired reconciliation +and QRF pattern evidence current pools must carry. None of these relaxations +applies without `legacy_worker_authentication`, and a current-contract artifact +whose version numbers were merely lowered is refused (at the operator order, +and at the sealed schedule if the historical order is borrowed). + Current manifests, diagnostics, `release_manifest.json`, and scoring receipts surface `worker_execution_authentication`: manifest, execution-config, and worker schema versions, `semantic_identity_sha256`, and audit aliases. From 7ee36ac1bea125218912996b1030b77a208db963 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 19 Sep 2026 09:37:36 -0400 Subject: [PATCH 31/49] Cache only the verified frozen schedule text, not a shared parse Each caller of the legacy schedule receipt now gets its own parse, so no caller can alter nested content another validates against. Co-Authored-By: Claude Fable 5.1 --- .../us_runtime/us_late_producer_registry.py | 19 +++++++++++++++---- .../tests/test_us_legacy_schema9_contract.py | 12 ++++++++++-- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_producer_registry.py b/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_producer_registry.py index b83dcd8cfc..bc19bb05f8 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_producer_registry.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_producer_registry.py @@ -2238,13 +2238,14 @@ def us_late_producer_schedule_receipt() -> Mapping[str, object]: @cache -def legacy_us_late_producer_schedule_receipt() -> Mapping[str, object]: - """Load the frozen schema-16 schedule for attested legacy scoring.""" +def _verified_legacy_schedule_json() -> str: + """Return the frozen schema-16 schedule text once its hash is verified.""" resource = files("microcosm.build.us").joinpath( "legacy_schema16_late_producer_schedule.json" ) - receipt = json.loads(resource.read_text(encoding="utf-8")) + text = resource.read_text(encoding="utf-8") + receipt = json.loads(text) payload = { key: value for key, value in receipt.items() @@ -2265,4 +2266,14 @@ def legacy_us_late_producer_schedule_receipt() -> Mapping[str, object]: "frozen schema-16 late-producer schedule does not match its " "sealed content hash." ) - return MappingProxyType(receipt) + return text + + +def legacy_us_late_producer_schedule_receipt() -> Mapping[str, object]: + """Load the frozen schema-16 schedule for attested legacy scoring. + + Only the verified text is cached; every caller gets its own parse, so no + caller can alter the nested content another one validates against. + """ + + return MappingProxyType(json.loads(_verified_legacy_schedule_json())) diff --git a/packages/microcosm-build/tests/test_us_legacy_schema9_contract.py b/packages/microcosm-build/tests/test_us_legacy_schema9_contract.py index d74915d04b..5467193b4d 100644 --- a/packages/microcosm-build/tests/test_us_legacy_schema9_contract.py +++ b/packages/microcosm-build/tests/test_us_legacy_schema9_contract.py @@ -90,12 +90,12 @@ def test_tampered_frozen_schedule_is_refused(monkeypatch: pytest.MonkeyPatch) -> "LEGACY_SCHEMA16_LATE_PRODUCER_SCHEDULE_PAYLOAD_SHA256", "0" * 64, ) - registry.legacy_us_late_producer_schedule_receipt.cache_clear() + registry._verified_legacy_schedule_json.cache_clear() try: with pytest.raises(ValueError, match="sealed content hash"): registry.legacy_us_late_producer_schedule_receipt() finally: - registry.legacy_us_late_producer_schedule_receipt.cache_clear() + registry._verified_legacy_schedule_json.cache_clear() def test_schema9_operator_order_is_the_literal_historical_order() -> None: @@ -115,3 +115,11 @@ def test_schema9_operator_order_is_the_literal_historical_order() -> None: ) assert "us_immigration_composition_gate" not in historical assert h5_io.US_STACKED_POOL_OPERATOR_ORDER[-1] == "us_immigration_composition_gate" + + +def test_each_caller_gets_its_own_frozen_schedule() -> None: + first = registry.legacy_us_late_producer_schedule_receipt() + first["execution_receipt_contract"]["version"] = 99 + + second = registry.legacy_us_late_producer_schedule_receipt() + assert second["execution_receipt_contract"]["version"] == 3 From d4835a5e44ec97643754dee17dd7308b65bded26 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 22 Sep 2026 17:44:13 -0400 Subject: [PATCH 32/49] Re-pin the merged tree's attested identities for the fifth main sync Main's two new US package resources and this branch's frozen schedule together move the configuration universe to 42,549 fields; the seed protocol/map digests, pointer inventory, coverage evidence, loader golden vector, and US spec sha mirror are recomputed on the merge. Co-Authored-By: Claude Opus 5.5 --- docs/evidence/spec-engine/us-f0-coverage.json | 32 +++++++++---------- .../build/spec_engine/field_usage.py | 8 ++--- .../build/spec_engine/inventory_coverage.py | 4 +-- .../tests/test_spec_engine_coverage_tool.py | 10 +++--- .../tests/test_spec_engine_field_usage.py | 10 +++--- .../tests/test_spec_engine_loader.py | 2 +- .../tests/test_us_multispine_pool_tool.py | 2 +- tools/spec_engine_coverage.py | 2 +- 8 files changed, 35 insertions(+), 35 deletions(-) diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index 713b8c6fa6..4f420521f7 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -6,7 +6,7 @@ "country": "us", "documentation_sha256": "5beb9497d74b7407fbb3fff888007d089423122db504171ce5e4b337a79e3dcd", "field_usage": { - "authored_normative_field_count": 32524, + "authored_normative_field_count": 32530, "claim_count": 49, "claims": [ { @@ -16,8 +16,8 @@ "legacy_sinks": [], "mode": "front_end_validation", "pointer_class": "all", - "pointer_count": 101, - "pointer_sha256": "be7cb568758bd3374612d2bcc37ea96a8269ffeac53fe0d10dc62195471b0d2e", + "pointer_count": 107, + "pointer_sha256": "2c2099d3d5ef3d664f74a2e732870391022a852cf49d5cf96810c9a984d2b20b", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/country_package.json", @@ -772,20 +772,20 @@ "verifier": "vintages" } ], - "configuration_field_count": 42543, - "consumed_field_count": 42543, + "configuration_field_count": 42549, + "consumed_field_count": 42549, "generation0_effect_counts": { "legacy_behavior": 38574, - "no_generation0_effect": 3969 + "no_generation0_effect": 3975 }, "mode_counts": { "compiler_semantic": 28069, - "front_end_validation": 351, + "front_end_validation": 357, "identity_only": 103, "legacy_behavior": 14020 }, "multiple_primary_use_field_count": 0, - "pointer_inventory_sha256": "b5fb54a54e5bb150ad8d6a980daa0e23fa294ef8e9c81f5f6fcde7bb68a6af63", + "pointer_inventory_sha256": "d526813a4458082f2bd6d05bef66d08a05e97d0c2028508a9352d4e3605f02cb", "resolved_binding_field_count": 10019, "unused_field_count": 0 }, @@ -1669,13 +1669,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "ac74a667d89e4cab9b9253b9a7817ae57fbda749eee903065c147c42c3b44967", - "protocol_sha256": "b4ea35f32e38570896aa08e25515a6f3c50e13fd8db18146dd5322aab18629cb" + "map_sha256": "0210ebae670bd725e0ae782f14f2fc157f0d549ed0ffefa19d3e98ad5c005115", + "protocol_sha256": "d1122ced29be708009763d47350a2b39a601e83271b3696ead425f66de373d1a" }, "failures": [], "observed": { - "map_sha256": "ac74a667d89e4cab9b9253b9a7817ae57fbda749eee903065c147c42c3b44967", - "protocol_sha256": "b4ea35f32e38570896aa08e25515a6f3c50e13fd8db18146dd5322aab18629cb" + "map_sha256": "0210ebae670bd725e0ae782f14f2fc157f0d549ed0ffefa19d3e98ad5c005115", + "protocol_sha256": "d1122ced29be708009763d47350a2b39a601e83271b3696ead425f66de373d1a" }, "status": "covered" }, @@ -1690,7 +1690,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "b4ea35f32e38570896aa08e25515a6f3c50e13fd8db18146dd5322aab18629cb", + "implementation_sha256": "d1122ced29be708009763d47350a2b39a601e83271b3696ead425f66de373d1a", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1711,7 +1711,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "b4ea35f32e38570896aa08e25515a6f3c50e13fd8db18146dd5322aab18629cb", + "implementation_sha256": "d1122ced29be708009763d47350a2b39a601e83271b3696ead425f66de373d1a", "protocol": "legacy-v1", "streams": [ "build_model", @@ -2612,7 +2612,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "9f4f0af295277cc546e164fbb0d50dcc67e55b4a5dabe0e7271dd252843dd258" + "spec_sha256": "0b01f6dca76176670cb29bb108f797b65ff402f56a8153aec9680ce2dc1bb8b5" } }, "report_schema_version": 3, @@ -2622,7 +2622,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "9f4f0af295277cc546e164fbb0d50dcc67e55b4a5dabe0e7271dd252843dd258" + "spec_sha256": "0b01f6dca76176670cb29bb108f797b65ff402f56a8153aec9680ce2dc1bb8b5" }, "status": "pass" } diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py index c998e2d97e..47616922dd 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py @@ -26,9 +26,9 @@ ) from .schemas import load_schema_registry -EXPECTED_AUTHORED_FIELD_COUNT = 32_524 +EXPECTED_AUTHORED_FIELD_COUNT = 32_530 EXPECTED_RESOLVED_BINDING_FIELD_COUNT = 10_019 -EXPECTED_CONFIGURATION_FIELD_COUNT = 42_543 +EXPECTED_CONFIGURATION_FIELD_COUNT = 42_549 class FieldUsageError(AssertionError): @@ -373,8 +373,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "64182b6be1ea6d95bff345b30a2aa046b6fa7e8ee61a282b225dfa49c28fbfdc", ), "country_manifest": ( - 101, - "be7cb568758bd3374612d2bcc37ea96a8269ffeac53fe0d10dc62195471b0d2e", + 107, + "2c2099d3d5ef3d664f74a2e732870391022a852cf49d5cf96810c9a984d2b20b", ), "generated_authorities": ( 8_606, diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index ab91b9c750..f7a2be442b 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -372,8 +372,8 @@ "late_schedule": "88bc9243a3518982ae951c3de21bd55877e296ce4fcb183b9bee420d3a684b10", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "987b501c695e31f45521c4a178528f75ab3df22c09bc407b182213b2de99ee57", - "seed_map": "ac74a667d89e4cab9b9253b9a7817ae57fbda749eee903065c147c42c3b44967", - "seed_protocol": "b4ea35f32e38570896aa08e25515a6f3c50e13fd8db18146dd5322aab18629cb", + "seed_map": "0210ebae670bd725e0ae782f14f2fc157f0d549ed0ffefa19d3e98ad5c005115", + "seed_protocol": "d1122ced29be708009763d47350a2b39a601e83271b3696ead425f66de373d1a", "source_manifest": "9df42433ba92c5cad75f1df2394f8b64fb25ed027b5a5e6ad383874345112afa", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", diff --git a/packages/microcosm-build/tests/test_spec_engine_coverage_tool.py b/packages/microcosm-build/tests/test_spec_engine_coverage_tool.py index d7062f65b0..b2ba8f96d6 100644 --- a/packages/microcosm-build/tests/test_spec_engine_coverage_tool.py +++ b/packages/microcosm-build/tests/test_spec_engine_coverage_tool.py @@ -88,22 +88,22 @@ def test_us_coverage_is_exact_complete_and_honest( assert_coverage_complete(coverage_report) assert coverage_report["status"] == "pass" fields = coverage_report["field_usage"] - assert fields["configuration_field_count"] == 42_543 - assert fields["authored_normative_field_count"] == 32_524 + assert fields["configuration_field_count"] == 42_549 + assert fields["authored_normative_field_count"] == 32_530 assert fields["resolved_binding_field_count"] == 10_019 - assert fields["consumed_field_count"] == 42_543 + assert fields["consumed_field_count"] == 42_549 assert fields["unused_field_count"] == 0 assert fields["multiple_primary_use_field_count"] == 0 assert fields["claim_count"] == 49 assert fields["mode_counts"] == { "legacy_behavior": 14_020, "compiler_semantic": 28_069, - "front_end_validation": 351, + "front_end_validation": 357, "identity_only": 103, } assert fields["generation0_effect_counts"] == { "legacy_behavior": 38_574, - "no_generation0_effect": 3_969, + "no_generation0_effect": 3_975, } inventory = coverage_report["inventory_coverage"] diff --git a/packages/microcosm-build/tests/test_spec_engine_field_usage.py b/packages/microcosm-build/tests/test_spec_engine_field_usage.py index 002dc37a65..72a281f34d 100644 --- a/packages/microcosm-build/tests/test_spec_engine_field_usage.py +++ b/packages/microcosm-build/tests/test_spec_engine_field_usage.py @@ -92,22 +92,22 @@ def _mutated_bundle( def test_exact_complete_ledger_has_one_primary_mode_per_pointer(field_ledger) -> None: - assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_543 + assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_549 assert field_ledger.source_counts == { - "authored": 32_524, + "authored": 32_530, "resolved_bindings": 10_019, } assert field_ledger.mode_counts == { "legacy_behavior": 14_020, "compiler_semantic": 28_069, - "front_end_validation": 351, + "front_end_validation": 357, "identity_only": 103, } assert field_ledger.generation0_effect_counts == { "legacy_behavior": 38_574, - "no_generation0_effect": 3_969, + "no_generation0_effect": 3_975, } - assert len({field.pointer for field in field_ledger.fields}) == 42_543 + assert len({field.pointer for field in field_ledger.fields}) == 42_549 def test_eligibility_concepts_are_validation_not_generation0_behavior( diff --git a/packages/microcosm-build/tests/test_spec_engine_loader.py b/packages/microcosm-build/tests/test_spec_engine_loader.py index 35975b26c0..bb0593d4de 100644 --- a/packages/microcosm-build/tests/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/test_spec_engine_loader.py @@ -236,7 +236,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # Pin the domain separator, normalization rules, schema-set receipt, and # exact normative projection as one reviewable golden vector. assert first.spec_sha256 == ( - "47608ddd982bae51af7dd87e4d5ec49e60fb341674e976e1e00d1c857c112ff4" + "83fd64e835518917d670a36e6e49a9cacadc4a875b2f9ecfab389baba82724f3" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") diff --git a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py index c2ee8ac6bc..d5ef5905d7 100644 --- a/packages/microcosm-build/tests/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/test_us_multispine_pool_tool.py @@ -2933,7 +2933,7 @@ def capture_equality(expected: object, actual: object) -> None: "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "9f4f0af295277cc546e164fbb0d50dcc67e55b4a5dabe0e7271dd252843dd258", + "spec_sha256": "0b01f6dca76176670cb29bb108f797b65ff402f56a8153aec9680ce2dc1bb8b5", }, } diff --git a/tools/spec_engine_coverage.py b/tools/spec_engine_coverage.py index dc9090ab97..c65a8c02fd 100644 --- a/tools/spec_engine_coverage.py +++ b/tools/spec_engine_coverage.py @@ -41,7 +41,7 @@ REPORT_SCHEMA_VERSION = 3 EXPECTED_POINTER_INVENTORY_SHA256 = ( - "b5fb54a54e5bb150ad8d6a980daa0e23fa294ef8e9c81f5f6fcde7bb68a6af63" + "d526813a4458082f2bd6d05bef66d08a05e97d0c2028508a9352d4e3605f02cb" ) DEFAULT_REPORT_PATH = ( Path(__file__).resolve().parents[1] From 48a371bb354cf56a903f2014e8117b2484815f3a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 25 Sep 2026 15:17:24 -0400 Subject: [PATCH 33/49] Validate attested schema-9 execution rows against the contracts they sealed Round-2 review: legacy scoring accepted the frozen schema-16 schedule but still checked every execution row against today's registry, so a real schema-9 pool's immigration source row (57 inputs) and transfer row (93) were refused against today's 56 and 100. The registry builder now takes its source/transfer inventories as arguments; legacy_us_late_producer_contracts rebuilds the historical contracts from the frozen schedule's own inventories and refuses unless that derivation reproduces both the frozen schedule_sha256 (which hashes every contract's full inputs and outputs) and the sealed payload hash. Co-Authored-By: Claude Opus 5.5 --- .../build/us_runtime/stacked_spine.py | 14 +- .../us_runtime/us_late_producer_registry.py | 203 ++++++++++++++---- .../tests/test_us_legacy_schema9_contract.py | 75 +++++++ 3 files changed, 254 insertions(+), 38 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py b/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py index a9d844592e..e2ad030bc6 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py @@ -221,6 +221,7 @@ US_LATE_SOURCE_FINALIZER_STAGE, US_LATE_TRANSFER_MODEL_CONFIG_INPUT, US_LATE_TRANSFER_TARGET_BANK_INPUT, + legacy_us_late_producer_contracts, legacy_us_late_producer_schedule_receipt, us_late_producer_schedule_receipt, ) @@ -8693,7 +8694,16 @@ def validate_stacked_late_producer_receipt( receipt.get(digest_field), boundary=f"{boundary} {digest_field}" ) execution = receipt.get("execution") - expected_order = CANONICAL_US_LATE_PRODUCER_SCHEDULE.order + # An attested schema-9 pool sealed its rows against the schema-16 + # contracts, rebuilt (and hash-verified) from the frozen schedule; today's + # registry would reject its historical input surfaces. + if legacy_worker_authentication is not None: + legacy_contracts = legacy_us_late_producer_contracts() + expected_registry = legacy_contracts.registry + expected_order = legacy_contracts.schedule.order + else: + expected_registry = CANONICAL_US_LATE_PRODUCER_REGISTRY + expected_order = CANONICAL_US_LATE_PRODUCER_SCHEDULE.order if not isinstance(execution, list) or len(execution) != len(expected_order): raise ValueError( f"{boundary}: stacked late-producer DAG must carry exactly " @@ -8709,7 +8719,7 @@ def validate_stacked_late_producer_receipt( raw_row = execution[index] previous_sha256 = _validate_late_execution_row( raw_row, - contract=CANONICAL_US_LATE_PRODUCER_REGISTRY[producer_name], + contract=expected_registry[producer_name], execution_index=index, expected_previous_sha256=previous_sha256, boundary=boundary, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_producer_registry.py b/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_producer_registry.py index bc19bb05f8..7476544d04 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_producer_registry.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_producer_registry.py @@ -97,6 +97,8 @@ "US_LATE_TRANSFER_TARGET_BANK_INPUT", "source_producer_name", "transfer_producer_name", + "LegacyLateProducerContracts", + "legacy_us_late_producer_contracts", "legacy_us_late_producer_schedule_receipt", "us_late_producer_schedule_payload", "us_late_producer_schedule_receipt", @@ -1682,7 +1684,17 @@ def _inventory_contract_inputs( return tuple(inputs) -def _build_registry() -> dict[str, ProducerContract]: +def _build_registry( + *, + source_inventories: Mapping[str, SourceInputInventory], + transfer_inventories: Mapping[str, SourceInputInventory], +) -> dict[str, ProducerContract]: + """Derive every late producer contract from its effective input inventories. + + The inventories are arguments so attested legacy scoring can rebuild the + contracts a historical schedule sealed from that schedule's own frozen + inventories; everything else is derived exactly as for the live registry. + """ late_surface = pool_post_puf_transfer_target_families() late_keys = _target_key_rows(late_surface) puf_keys = _target_key_rows(pool_post_puf_puf_producer_target_families()) @@ -1906,7 +1918,7 @@ def _build_registry() -> dict[str, ProducerContract]: direct_dependency_keys = { (item.entity, item.column) for item in direct_dependencies } - for requirement in US_LATE_SOURCE_INPUT_INVENTORIES[operator].requirements: + for requirement in source_inventories[operator].requirements: for alternative in requirement.alternatives: for item in alternative: key = (item.entity, item.column) @@ -1928,7 +1940,7 @@ def _build_registry() -> dict[str, ProducerContract]: *direct_dependencies, *_inventory_contract_inputs( name, - US_LATE_SOURCE_INPUT_INVENTORIES[operator], + source_inventories[operator], required_scope=_ASEC_SOURCE_SCOPE, ), } @@ -1978,7 +1990,7 @@ def _build_registry() -> dict[str, ProducerContract]: inputs: list[ProducerInput] = list( _inventory_contract_inputs( group.name, - US_LATE_TRANSFER_INPUT_INVENTORIES[group.name], + transfer_inventories[group.name], required_scope=_WHOLE_POOL_SCOPE, ) ) @@ -2021,7 +2033,7 @@ def _build_registry() -> dict[str, ProducerContract]: for item in inputs if not item.column.startswith("@effective:") } - for requirement in US_LATE_TRANSFER_INPUT_INVENTORIES[group.name].requirements: + for requirement in transfer_inventories[group.name].requirements: for alternative in requirement.alternatives: for item in alternative: key = (item.entity, item.column) @@ -2099,7 +2111,10 @@ def _build_registry() -> dict[str, ProducerContract]: CANONICAL_US_LATE_PRODUCER_REGISTRY: Mapping[str, ProducerContract] = MappingProxyType( - _build_registry() + _build_registry( + source_inventories=US_LATE_SOURCE_INPUT_INVENTORIES, + transfer_inventories=US_LATE_TRANSFER_INPUT_INVENTORIES, + ) ) CANONICAL_US_LATE_PRODUCER_SCHEDULE: ProducerSchedule = derive_producer_schedule( CANONICAL_US_LATE_PRODUCER_REGISTRY, @@ -2132,35 +2147,43 @@ def _inventory_payload(inventory: SourceInputInventory) -> dict[str, object]: } -def us_late_producer_schedule_payload() -> dict[str, object]: - """Return the complete JSON-safe declaration bound into checkpoint identity.""" +def _live_execution_receipt_contract() -> dict[str, object]: + return { + "version": US_LATE_PRODUCER_RECEIPT_SCHEMA_VERSION, + "row_binding": ( + "declared_globally_reconciled_input_and_scope_exact_output_" + "source_and_primary_callback_resource_receipt_and_previous_" + "execution_sha256" + ), + "virtual_resource_binding": ("exact_kind_specific_semantic_payload_and_sha256"), + "top_binding": ( + "entry_and_output_frame_sha256_execution_chain_source_" + "completion_nineteen_transfer_groups_and_constrained_" + "immigration_reconciliation" + ), + "transition_authority": { + "authority_id": US_LATE_PRODUCER_TRANSITION_AUTHORITY_ID, + "metadata_key": US_LATE_PRODUCER_TRANSITION_AUTHORITY_KEY, + "version": US_LATE_PRODUCER_TRANSITION_AUTHORITY_VERSION, + "independent_digest_required": True, + }, + } - schedule = CANONICAL_US_LATE_PRODUCER_SCHEDULE + +def _schedule_payload( + *, + schema_version: int, + execution_receipt_contract: Mapping[str, object], + schedule: ProducerSchedule, + source_inventories: Mapping[str, SourceInputInventory], + transfer_inventories: Mapping[str, SourceInputInventory], +) -> dict[str, object]: return { - "schema_version": US_LATE_PRODUCER_REGISTRY_SCHEMA_VERSION, + "schema_version": schema_version, "overlap_ownership": dict(us_late_overlap_ownership_receipt()), - "execution_receipt_contract": { - "version": US_LATE_PRODUCER_RECEIPT_SCHEMA_VERSION, - "row_binding": ( - "declared_globally_reconciled_input_and_scope_exact_output_" - "source_and_primary_callback_resource_receipt_and_previous_" - "execution_sha256" - ), - "virtual_resource_binding": ( - "exact_kind_specific_semantic_payload_and_sha256" - ), - "top_binding": ( - "entry_and_output_frame_sha256_execution_chain_source_" - "completion_nineteen_transfer_groups_and_constrained_" - "immigration_reconciliation" - ), - "transition_authority": { - "authority_id": US_LATE_PRODUCER_TRANSITION_AUTHORITY_ID, - "metadata_key": US_LATE_PRODUCER_TRANSITION_AUTHORITY_KEY, - "version": US_LATE_PRODUCER_TRANSITION_AUTHORITY_VERSION, - "independent_digest_required": True, - }, - }, + "execution_receipt_contract": json.loads( + json.dumps(dict(execution_receipt_contract)) + ), "schedule_sha256": schedule.sha256, "external_stages": list(US_LATE_EXTERNAL_STAGES), "order": list(schedule.order), @@ -2176,8 +2199,8 @@ def us_late_producer_schedule_payload() -> dict[str, object]: for group in CANONICAL_US_LATE_TRANSFER_GROUPS ], "source_input_inventories": [ - _inventory_payload(US_LATE_SOURCE_INPUT_INVENTORIES[operator]) - for operator in sorted(US_LATE_SOURCE_INPUT_INVENTORIES) + _inventory_payload(source_inventories[operator]) + for operator in sorted(source_inventories) ], "primary_puf_input_inventory": _inventory_payload( US_LATE_PRIMARY_PUF_INPUT_INVENTORY @@ -2186,12 +2209,24 @@ def us_late_producer_schedule_payload() -> dict[str, object]: US_LATE_ACS_EARNINGS_UNIVERSE_INPUT_INVENTORY ), "transfer_input_inventories": [ - _inventory_payload(US_LATE_TRANSFER_INPUT_INVENTORIES[name]) - for name in sorted(US_LATE_TRANSFER_INPUT_INVENTORIES) + _inventory_payload(transfer_inventories[name]) + for name in sorted(transfer_inventories) ], } +def us_late_producer_schedule_payload() -> dict[str, object]: + """Return the complete JSON-safe declaration bound into checkpoint identity.""" + + return _schedule_payload( + schema_version=US_LATE_PRODUCER_REGISTRY_SCHEMA_VERSION, + execution_receipt_contract=_live_execution_receipt_contract(), + schedule=CANONICAL_US_LATE_PRODUCER_SCHEDULE, + source_inventories=US_LATE_SOURCE_INPUT_INVENTORIES, + transfer_inventories=US_LATE_TRANSFER_INPUT_INVENTORIES, + ) + + def us_late_producer_schedule_receipt() -> Mapping[str, object]: """Return the byte-stable production schedule identity and useful counts.""" @@ -2277,3 +2312,99 @@ def legacy_us_late_producer_schedule_receipt() -> Mapping[str, object]: """ return MappingProxyType(json.loads(_verified_legacy_schedule_json())) + + +def _inventory_from_payload(payload: Mapping[str, object]) -> SourceInputInventory: + """Invert :func:`_inventory_payload` exactly.""" + + requirements = payload["requirements"] + assert isinstance(requirements, Sequence) + return SourceInputInventory( + operator=str(payload["operator"]), + requirements=tuple( + EffectiveInputRequirement( + label=row["label"], + optional=row["optional"], + required_scope=row["required_scope"], + alternatives=tuple( + tuple( + ScopedInput(item["entity"], item["column"], item["value_kind"]) + for item in alternative + ) + for alternative in row["alternatives"] + ), + ) + for row in requirements + ), + ) + + +@dataclass(frozen=True) +class LegacyLateProducerContracts: + """The late producer contracts an attested schema-16 schedule sealed.""" + + registry: Mapping[str, ProducerContract] + schedule: ProducerSchedule + + +@cache +def legacy_us_late_producer_contracts() -> LegacyLateProducerContracts: + """Rebuild the contracts attested schema-9 pools sealed, fail-closed. + + The frozen schedule serializes every source and transfer input inventory. + Feeding those historical inventories to the live contract derivation must + reproduce both the frozen ``schedule_sha256`` -- which hashes every + contract's full inputs and outputs -- and the sealed payload hash. Any + other drift in the derivation since schema 16 therefore refuses here, + instead of validating a historical pool's execution rows against today's + contracts. + """ + + frozen = json.loads(_verified_legacy_schedule_json()) + source_inventories = { + inventory.operator: inventory + for inventory in map( + _inventory_from_payload, frozen["source_input_inventories"] + ) + } + transfer_inventories = { + inventory.operator: inventory + for inventory in map( + _inventory_from_payload, frozen["transfer_input_inventories"] + ) + } + registry = _build_registry( + source_inventories=source_inventories, + transfer_inventories=transfer_inventories, + ) + schedule = derive_producer_schedule( + registry, + external_stages=US_LATE_EXTERNAL_STAGES, + ) + payload = _schedule_payload( + schema_version=frozen["schema_version"], + execution_receipt_contract=frozen["execution_receipt_contract"], + schedule=schedule, + source_inventories=source_inventories, + transfer_inventories=transfer_inventories, + ) + canonical = json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + if ( + schedule.sha256 != frozen["schedule_sha256"] + or hashlib.sha256(canonical).hexdigest() + != LEGACY_SCHEMA16_LATE_PRODUCER_SCHEDULE_PAYLOAD_SHA256 + ): + raise ValueError( + "the live late-producer contract derivation no longer reproduces " + "the frozen schema-16 schedule; attested legacy scoring cannot " + "rebuild the contracts those pools sealed." + ) + return LegacyLateProducerContracts( + registry=MappingProxyType(registry), + schedule=schedule, + ) diff --git a/packages/microcosm-build/tests/test_us_legacy_schema9_contract.py b/packages/microcosm-build/tests/test_us_legacy_schema9_contract.py index 5467193b4d..094a55aa83 100644 --- a/packages/microcosm-build/tests/test_us_legacy_schema9_contract.py +++ b/packages/microcosm-build/tests/test_us_legacy_schema9_contract.py @@ -123,3 +123,78 @@ def test_each_caller_gets_its_own_frozen_schedule() -> None: second = registry.legacy_us_late_producer_schedule_receipt() assert second["execution_receipt_contract"]["version"] == 3 + + +# --------------------------------------------------------------------------- +# The execution contracts schema-9 pools sealed (#767, second review round) +# --------------------------------------------------------------------------- + +IMMIGRATION_SOURCE = "source:with_us_immigration_inputs" +IMMIGRATION_TRANSFER = "transfer:person/source_operator_immigration" + + +def test_inventory_payload_round_trips_exactly() -> None: + for inventories in ( + registry.US_LATE_SOURCE_INPUT_INVENTORIES, + registry.US_LATE_TRANSFER_INPUT_INVENTORIES, + ): + for name, inventory in inventories.items(): + assert inventory.operator == name + parsed = registry._inventory_from_payload( + registry._inventory_payload(inventory) + ) + assert parsed == inventory + + +def test_legacy_contracts_reproduce_the_frozen_schedule() -> None: + legacy = registry.legacy_us_late_producer_contracts() + frozen = registry.legacy_us_late_producer_schedule_receipt() + assert legacy.schedule.sha256 == frozen["schedule_sha256"] + assert list(legacy.schedule.order) == frozen["order"] + assert set(legacy.registry) == set(registry.CANONICAL_US_LATE_PRODUCER_REGISTRY) + + +def test_legacy_contracts_differ_from_live_only_where_767_changed_them() -> None: + legacy = registry.legacy_us_late_producer_contracts().registry + live = registry.CANONICAL_US_LATE_PRODUCER_REGISTRY + changed = {name for name in live if legacy[name] != live[name]} + assert changed == {IMMIGRATION_SOURCE, IMMIGRATION_TRANSFER} + # The historical input surfaces schema-9 pools sealed, which today's + # contracts would refuse as "not the exact N-input readiness surface". + assert len(legacy[IMMIGRATION_SOURCE].inputs) == 57 + assert len(live[IMMIGRATION_SOURCE].inputs) == 56 + assert len(legacy[IMMIGRATION_TRANSFER].inputs) == 93 + assert len(live[IMMIGRATION_TRANSFER].inputs) == 100 + for name in (IMMIGRATION_SOURCE, IMMIGRATION_TRANSFER): + assert legacy[name].outputs == live[name].outputs + assert legacy[name].kind == live[name].kind + + +@pytest.mark.parametrize("drift", ["overlap_ownership", "contract_inputs"]) +def test_legacy_contracts_refuse_a_drifted_derivation( + monkeypatch: pytest.MonkeyPatch, drift: str +) -> None: + if drift == "overlap_ownership": + live = registry.us_late_overlap_ownership_receipt + + def drifted_ownership(): + receipt = dict(live()) + receipt["schema_version"] = -1 + return receipt + + monkeypatch.setattr( + registry, "us_late_overlap_ownership_receipt", drifted_ownership + ) + else: + live_inputs = registry._inventory_contract_inputs + + def drifted_inputs(node_name, inventory, *, required_scope): + return live_inputs(node_name, inventory, required_scope=required_scope)[1:] + + monkeypatch.setattr(registry, "_inventory_contract_inputs", drifted_inputs) + registry.legacy_us_late_producer_contracts.cache_clear() + try: + with pytest.raises(ValueError, match="no longer reproduces the frozen"): + registry.legacy_us_late_producer_contracts() + finally: + registry.legacy_us_late_producer_contracts.cache_clear() From 9f81cb88902be9c87d5c5e6bc965b8b1d41958f3 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 27 Sep 2026 15:42:00 -0400 Subject: [PATCH 34/49] Re-pin field-usage and inventory identities for the sixth main sync (partial) Observed on the merged tree: 42,567 configuration fields, 38 source stages, and the source-stages, generated-authorities, predictor-tuple, resource semantics, stacked checkpoint base identity, and seed digests. The ledger mode counts, pointer inventory, coverage evidence, loader golden and spec sha mirror follow. Co-Authored-By: Claude Opus 5.5 --- .../microcosm/build/spec_engine/field_usage.py | 14 +++++++------- .../build/spec_engine/inventory_coverage.py | 16 ++++++++-------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py index 47616922dd..b621e6bef8 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py @@ -26,9 +26,9 @@ ) from .schemas import load_schema_registry -EXPECTED_AUTHORED_FIELD_COUNT = 32_530 -EXPECTED_RESOLVED_BINDING_FIELD_COUNT = 10_019 -EXPECTED_CONFIGURATION_FIELD_COUNT = 42_549 +EXPECTED_AUTHORED_FIELD_COUNT = 32_540 +EXPECTED_RESOLVED_BINDING_FIELD_COUNT = 10_027 +EXPECTED_CONFIGURATION_FIELD_COUNT = 42_567 class FieldUsageError(AssertionError): @@ -377,8 +377,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "2c2099d3d5ef3d664f74a2e732870391022a852cf49d5cf96810c9a984d2b20b", ), "generated_authorities": ( - 8_606, - "3f20975597d93f7313583a944eeb9d6437651c4ff20e67628bf6bf4c5aa9f004", + 8_614, + "66eec1216f404e43f7605fba0016d1cd666e71ebe3e35278d14a944d8b48b619", ), "geography_assignment": ( 28, @@ -485,8 +485,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "d6782c5de5bbed1bdc6bf653c4a6d4aadcad4ccc72d35e1092e130fcb04680a3", ), "source_stages": ( - 1_715, - "78227f34c3f4ef946c06693dec9177dd906069f98016854bfd4d39c4cd4c51fa", + 1_725, + "84e499b0b557642a21470dd7bbd2877eb7d096de1b54928b978db11c52945098", ), "spine_assembly_household_mass_shares": ( 2, diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index f7a2be442b..95b9abff32 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -363,18 +363,18 @@ "acs_person_predictors": "878c788a6f037d7aca12b3586ea034eff04f3034ffa11935a736493042551f25", "authority": "5730bd1c7410a59baa5a85ced86d9dcf0f901b3fa1c56617953199ad1a180eaa", "early_families": "4aa9f736fd76e83955477ad1667e58f48f264783f05bdc7f0102cd32d61323bd", - "full_checkpoint": "9c2af34c0779efe4cb7fc7d25c37e5b318e7d886dda4cedcadc5acda0cf9b010", + "full_checkpoint": "9ff42fbfc412dce17589e8c6cd4e2628647956c5086962f2ff93ba33a572a155", "gap_fill_schedule": "1c31f9868f7884347cc19cf1ff65da43f950b9114941a715bab168246db414a7", "graph_nodes": "2c71f35d54330f13c96783e0252ed9354ff3b4c0edd7e30205886c87ab7ec078", "geography_assignment": "f49425ca8734ac559c73cf44f6458d86d3162a48956b98a27e6e758959361585", "late_families": "d91f9ff0eb52f43e7b6eed3d5c58c37abe1620c3a11021da15dae9c10e16d382", - "late_resource_semantics": "fda05279f38b371b38faa44b87d3a64aa34f1a4cc7f93bec7e059ca6418176ff", + "late_resource_semantics": "8312b706436a0a28d623eff0d0267bc44aed3f8391febf9f3f810243f2c29c96", "late_schedule": "88bc9243a3518982ae951c3de21bd55877e296ce4fcb183b9bee420d3a684b10", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", - "primary_tuples": "987b501c695e31f45521c4a178528f75ab3df22c09bc407b182213b2de99ee57", - "seed_map": "0210ebae670bd725e0ae782f14f2fc157f0d549ed0ffefa19d3e98ad5c005115", - "seed_protocol": "d1122ced29be708009763d47350a2b39a601e83271b3696ead425f66de373d1a", - "source_manifest": "9df42433ba92c5cad75f1df2394f8b64fb25ed027b5a5e6ad383874345112afa", + "primary_tuples": "fdf23da429f8c501e198a2f5b719763d523a41565c2153fdf17b0344f21bcd7c", + "seed_map": "437faed245a63b109cc25e76bf5e86fd0cd9730be19a302c5c30fac308586c9f", + "seed_protocol": "36fd4a7f90bc0b75ae9c183cbafb0006829072ce702e6bfef245fcc00040c46c", + "source_manifest": "2fe81ec636df4125ce56725ef13c81ffbe942d9e8fd04474969905dbeac4edd3", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", } @@ -1201,7 +1201,7 @@ def add( "source manifest digest differs": sha256_json(source_manifest) == EXPECTED_HASHES["source_manifest"], "stage count differs": len(_array(sources["stages"], "source stages")) - == 37, + == 38, }, homes=("/sources/stage_manifest", "/sources/stages"), consumers=( @@ -1212,7 +1212,7 @@ def add( "stages": len(_array(sources["stages"], "source stages")), "sha256": sha256_json(source_manifest), }, - expected={"stages": 37, "sha256": EXPECTED_HASHES["source_manifest"]}, + expected={"stages": 38, "sha256": EXPECTED_HASHES["source_manifest"]}, ) pipeline = _mapping(spine["pipeline_contract"], "spine pipeline contract") operator_order = tuple( From 300aa3c17134cbe1b9cf831717fc3c819356ea0d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 28 Sep 2026 17:08:02 -0400 Subject: [PATCH 35/49] Carry main's SPM cofailure line and the H.R.1 baseline keys in two merged tests The immigration-drift case now expects the SPM composition gate's degraded-mode line main's cofailure contract appends on the fake export frame; main's prepared-pool SPM case gets the immigration statuses every post-#767 pool carries; the shipped reform-coverage baseline plans count the three keys the four H.R.1 probes add (medicaid 2027, aca_ptc 2027, snap 2026). Co-Authored-By: Claude Opus 5.5 --- .../engine/us/test_us_post_export_scoring.py | 5 ++++- .../us/test_us_fiscal_refresh_builder.py | 22 +++++++++++++++++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/packages/microcosm-build/tests/engine/us/test_us_post_export_scoring.py b/packages/microcosm-build/tests/engine/us/test_us_post_export_scoring.py index 45f7c2c958..9a2867730f 100644 --- a/packages/microcosm-build/tests/engine/us/test_us_post_export_scoring.py +++ b/packages/microcosm-build/tests/engine/us/test_us_post_export_scoring.py @@ -276,8 +276,11 @@ def test_shipped_baseline_plans_match_whole_file_policyengine_us( sweep = _load_sweep_module() path = _write_engine_h5(builder, tmp_path, _GUARD_HOUSEHOLDS) plans, _, counts = sweep._record_requests(builder) + # The four H.R.1 humanitarian-eligibility probes (#767) add three new + # baseline keys: medicaid 2027, aca_ptc 2027 and snap 2026 (aca_ptc 2026 + # was already read by another probe). assert counts["baseline_keys"] == { - "reform_coverage_smoke": 18, + "reform_coverage_smoke": 21, "reform_validation": 75, "demographics": 1, } diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py index 3505d35215..33221e3265 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py @@ -6428,8 +6428,19 @@ def n(self, entity): def table(self, entity): if terminal_mode == "spm_missing_pool" and entity == "person": + # A prepared pool carries its immigration statuses; only the + # SPM source role is missing. return pd.DataFrame( - {"person_id": [1, 2, 3, 4], "age": [30, 40, 16, 17]} + { + "person_id": [1, 2, 3, 4], + "age": [30, 40, 16, 17], + "immigration_status_str": [ + "REFUGEE", + "LEGAL_PERMANENT_RESIDENT", + "ASYLEE", + "LEGAL_PERMANENT_RESIDENT", + ], + } ) if entity not in self.schema.entities: raise ValueError( @@ -8515,10 +8526,17 @@ def fake_release_gate_failures(*args, **kwargs): ) assert "SSI take-up delivery failed:" not in message elif terminal_mode == "immigration_drift": + # The drift leads; the only other line is the SPM composition + # gate's degraded-mode refusal on the household/person-only fake + # export frame, which this cofailure contract expects. assert message == ( "Release gates failed: Immigration composition failed: weighted " "humanitarian status share drifted from 0.500000 to 0.255319 " - "after calibration [final-immigration-sentinel]" + "after calibration [final-immigration-sentinel]; SPM measurement " + "composition failed (export frame): the export frame cannot be " + "classified, so the rule the engine applies to it cannot be " + "checked: Unknown entity 'spm_unit'; schema declares " + "['person', 'household']." ) elif clean_run: # The register mismatch is the run's only terminal failure, yet From 5a89fed1ea52d2030dbf0740dfbb0095bd688621 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 28 Sep 2026 17:10:48 -0400 Subject: [PATCH 36/49] Gate immigration composition at the build's period; close spill tiers within tolerance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two defects the invariant tests surfaced (#767): - us_immigration_composition_gate re-derived the source-aware cohort evidence at a hard-coded 2024 while the stage derives it at the build's time_period, so a non-2024 build would be gated against a different DACA cohort than it assigned. The gate now takes a required time_period, and both builders pass their own period (PERIOD, POOL_TIME_PERIOD). - _spill_pew_unauthorized_excess compared the recomputed excess to zero exactly; a selection that closes the gap exactly can leave ~1e-15 of summation residue, which opened the next tier and spilled a row inside the other controlled margin — the ordering the round-1 fix promised to keep. Closure now allows a billionth of the in-scope weight. Co-Authored-By: Claude Opus 5.5 --- .../microcosm/build/us_runtime/immigration.py | 20 ++++++- .../us/test_us_fiscal_refresh_builder.py | 7 ++- .../engine_free/us/test_us_immigration.py | 54 +++++++++++++------ .../us_multispine_pool_tool.py | 3 +- tools/build_us_fiscal_refresh_release.py | 6 ++- tools/build_us_multispine_pool.py | 2 +- 6 files changed, 69 insertions(+), 23 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py b/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py index e7c8072e15..5625364e11 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py @@ -1161,6 +1161,9 @@ def _pew_unauthorized_projection_mask( ) +_SPILL_CLOSURE_RELATIVE_TOLERANCE = 1e-9 + + def _spill_pew_unauthorized_excess( person: pd.DataFrame, ssn_codes: np.ndarray, @@ -1188,6 +1191,14 @@ def _spill_pew_unauthorized_excess( """ draws = _stable_person_draws(person, seed=seed, salt=salt) + # A selection that closes the gap exactly can leave floating-point + # residue in the recomputed sum (it adds the weights in a different order + # than the selection's cumulative sum). Treat anything within a + # billionth of the in-scope weight as closed, so residue never opens the + # next tier and spills a row the ordering promised to keep. + closed_within = _SPILL_CLOSURE_RELATIVE_TOLERANCE * max( + 1.0, float(weights[scope].sum()) + ) def current_excess() -> float: included = _pew_unauthorized_projection_mask( @@ -1206,7 +1217,7 @@ def current_excess() -> float: # EAD and so remain inside Pew's estimate; only rows outside those # cohorts reduce the count, which the corrective draw supplies. for retained_allowed in (True, False): - if current_excess() <= 0: + if current_excess() <= closed_within: return candidates = (ssn_codes == 0) & noncitizens & scope & priority if not retained_allowed: @@ -2124,6 +2135,7 @@ def us_immigration_composition_summary(frame: Frame) -> dict[str, object]: def us_immigration_composition_gate( frame: Frame, *, + time_period: int, controls: ImmigrationControls | None = None, ) -> GateResult: """Release gate: the SSN/immigration surface exists and is plausible. @@ -2138,6 +2150,10 @@ def us_immigration_composition_gate( cited stock target (microcosm #767 — the H.R.1 §71109/§71301/§71302 and SNAP §10108 eligibility channels all bind through these categories; an explicit zero target must emit exactly zero). + + ``time_period`` must be the period the stage ran for: the source-aware + cohort evidence (the DACA arrival windows in particular) is re-derived at + that period, exactly as ``with_us_immigration_inputs`` derived it. """ if controls is None: @@ -2184,7 +2200,7 @@ def us_immigration_composition_gate( ssn = person["ssn_card_type"].astype(str) status = person["immigration_status_str"].astype(str) try: - evidence = _source_aware_immigration_profile(person, time_period=2024) + evidence = _source_aware_immigration_profile(person, time_period=time_period) except (SourceRuntimeError, ValueError) as exc: evidence = None failures.append(f"source-aware immigration evidence is invalid: {exc}") diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py index 33221e3265..bdf00a3deb 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py @@ -7265,7 +7265,8 @@ def fake_energy_subsidy_signal_gate(frame): lambda frame, *, seed, time_period: frame, ) - def fake_immigration_composition_gate(frame): + def fake_immigration_composition_gate(frame, *, time_period): + assert time_period == builder.PERIOD statuses = frame.table("person")["immigration_status_str"].astype(str) household_weights = frame.weights_for("household") weights = np.asarray(household_weights.values, dtype=np.float64) @@ -11845,6 +11846,10 @@ def test_main_rechecks_immigration_composition_on_export_for_terminal_gates() -> "us_immigration_composition_gate" ) assert ast.unparse(final_assignment.value.args[0]) == "export_frame" + assert [ + (keyword.arg, ast.unparse(keyword.value)) + for keyword in final_assignment.value.keywords + ] == [("time_period", "PERIOD")] terminal_calls = [ node diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_immigration.py b/packages/microcosm-build/tests/engine_free/us/test_us_immigration.py index 89e74a62df..d62735d421 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_immigration.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_immigration.py @@ -1400,13 +1400,17 @@ def _composition_frame( class TestCompositionGate: def test_passes_on_plausible_composition(self) -> None: gate = us_immigration_composition_gate( - _composition_frame(), controls=_plausible_controls() + time_period=TIME_PERIOD, + frame=_composition_frame(), + controls=_plausible_controls(), ) assert gate.passed, gate.failures def test_fails_when_columns_missing(self) -> None: gate = us_immigration_composition_gate( - _us_frame([{"PRCITSHP": 1}]), controls=_plausible_controls() + time_period=TIME_PERIOD, + frame=_us_frame([{"PRCITSHP": 1}]), + controls=_plausible_controls(), ) assert not gate.passed assert any("missing person column" in failure for failure in gate.failures) @@ -1421,7 +1425,9 @@ def test_fails_on_the_225_failure_mode_all_citizens(self) -> None: for _ in range(50) ] gate = us_immigration_composition_gate( - _us_frame(rows), controls=_plausible_controls() + time_period=TIME_PERIOD, + frame=_us_frame(rows), + controls=_plausible_controls(), ) assert not gate.passed assert any("constant" in failure for failure in gate.failures) @@ -1430,7 +1436,9 @@ def test_fails_on_values_outside_engine_enum_domain(self) -> None: frame = _composition_frame() person = frame.table("person") person.loc[0, "ssn_card_type"] = "5" - gate = us_immigration_composition_gate(frame, controls=_plausible_controls()) + gate = us_immigration_composition_gate( + time_period=TIME_PERIOD, frame=frame, controls=_plausible_controls() + ) assert not gate.passed assert any("enum domain" in failure for failure in gate.failures) @@ -1438,13 +1446,16 @@ def test_fails_when_columns_disagree_about_citizenship(self) -> None: frame = _composition_frame() person = frame.table("person") person.loc[0, "immigration_status_str"] = "LEGAL_PERMANENT_RESIDENT" - gate = us_immigration_composition_gate(frame, controls=_plausible_controls()) + gate = us_immigration_composition_gate( + time_period=TIME_PERIOD, frame=frame, controls=_plausible_controls() + ) assert not gate.passed assert any("citizenship" in failure for failure in gate.failures) def test_fails_when_undocumented_far_from_anchor(self) -> None: gate = us_immigration_composition_gate( - _composition_frame(citizens=930, other=48, ead=10, none=2), + time_period=TIME_PERIOD, + frame=_composition_frame(citizens=930, other=48, ead=10, none=2), controls=_plausible_controls(), ) assert not gate.passed @@ -1453,7 +1464,8 @@ def test_fails_when_undocumented_far_from_anchor(self) -> None: def test_fails_when_non_citizen_share_implausible(self) -> None: gate = us_immigration_composition_gate( - _composition_frame(citizens=40, other=20, ead=20, none=20), + time_period=TIME_PERIOD, + frame=_composition_frame(citizens=40, other=20, ead=20, none=20), controls=ImmigrationControls( undocumented=UndocumentedControls( workers=8.0, @@ -1471,7 +1483,9 @@ def test_gate_reads_packaged_controls_by_default(self) -> None: packaged = us_immigration_controls() assert packaged.undocumented.workers == 9_700_000 assert packaged.humanitarian_target("refugee") == 160_000 - gate = us_immigration_composition_gate(_us_frame([{"PRCITSHP": 1}])) + gate = us_immigration_composition_gate( + time_period=TIME_PERIOD, frame=_us_frame([{"PRCITSHP": 1}]) + ) assert not gate.passed controls = gate.details["controls"] assert controls["undocumented_workers"] == 9_700_000 @@ -1497,7 +1511,8 @@ def test_gate_reads_packaged_controls_by_default(self) -> None: def test_gate_passes_with_in_band_humanitarian_masses(self) -> None: gate = us_immigration_composition_gate( - _composition_frame(humanitarian={"REFUGEE": 4, "TPS": 6}), + time_period=TIME_PERIOD, + frame=_composition_frame(humanitarian={"REFUGEE": 4, "TPS": 6}), controls=_plausible_controls(refugee=4.0, tps=6.0), ) assert gate.passed, gate.failures @@ -1508,7 +1523,8 @@ def test_gate_passes_with_in_band_humanitarian_masses(self) -> None: def test_gate_counts_temporary_protections_in_pew_population(self) -> None: gate = us_immigration_composition_gate( - _composition_frame( + time_period=TIME_PERIOD, + frame=_composition_frame( none=10, humanitarian={"DACA": 5, "PAROLED_ONE_YEAR": 10, "TPS": 10}, ), @@ -1530,7 +1546,8 @@ def test_gate_counts_only_residual_ead_cuban_haitian_entrants(self) -> None: person.loc[row, "PEINUSYR"] = 24 gate = us_immigration_composition_gate( - frame, + time_period=TIME_PERIOD, + frame=frame, controls=_plausible_controls(), ) assert gate.passed, gate.failures @@ -1551,7 +1568,8 @@ def test_gate_excludes_refugees_and_asylees_from_pew_population(self) -> None: humanitarian=_humanitarian_draws(refugee=10.0, asylee=10.0), ) gate = us_immigration_composition_gate( - _composition_frame( + time_period=TIME_PERIOD, + frame=_composition_frame( none=10, humanitarian={"REFUGEE": 10, "ASYLEE": 10}, ), @@ -1562,7 +1580,8 @@ def test_gate_excludes_refugees_and_asylees_from_pew_population(self) -> None: def test_gate_fails_when_humanitarian_category_collapses(self) -> None: gate = us_immigration_composition_gate( - _composition_frame(), + time_period=TIME_PERIOD, + frame=_composition_frame(), controls=_plausible_controls(refugee=4.0), ) assert not gate.passed @@ -1573,7 +1592,8 @@ def test_gate_fails_when_humanitarian_category_collapses(self) -> None: def test_gate_fails_when_explicit_zero_category_emits(self) -> None: gate = us_immigration_composition_gate( - _composition_frame(humanitarian={"DEPORTATION_WITHHELD": 2}), + time_period=TIME_PERIOD, + frame=_composition_frame(humanitarian={"DEPORTATION_WITHHELD": 2}), controls=_plausible_controls(), ) assert not gate.passed @@ -1583,7 +1603,8 @@ def test_gate_accepts_saturated_draws_inside_band(self) -> None: # The ASEC undercovers 2022-24 arrivals; a draw that saturates at # roughly three-quarters of its admin target stays inside the band. gate = us_immigration_composition_gate( - _composition_frame(humanitarian={"PAROLED_ONE_YEAR": 3}), + time_period=TIME_PERIOD, + frame=_composition_frame(humanitarian={"PAROLED_ONE_YEAR": 3}), controls=_plausible_controls(paroled_one_year=4.0), ) assert gate.passed, gate.failures @@ -1612,7 +1633,8 @@ def test_gate_checks_per_origin_draws_not_only_category_total(self) -> None: # Both rows carry Ukraine evidence. The category total is exactly two, # but the per-origin targets are a zero/two redistribution. gate = us_immigration_composition_gate( - _composition_frame(humanitarian={"PAROLED_ONE_YEAR": 2}), + time_period=TIME_PERIOD, + frame=_composition_frame(humanitarian={"PAROLED_ONE_YEAR": 2}), controls=controls, ) assert not gate.passed diff --git a/test_support/microcosm_build/us_multispine_pool_tool.py b/test_support/microcosm_build/us_multispine_pool_tool.py index c1485c5f5f..a501a98cfa 100644 --- a/test_support/microcosm_build/us_multispine_pool_tool.py +++ b/test_support/microcosm_build/us_multispine_pool_tool.py @@ -2493,7 +2493,8 @@ def battery( monkeypatch.setattr(pool_tool, "stacked_completeness_gate", completeness) monkeypatch.setattr(pool_tool, "by_origin_battery", battery) - def immigration(_frame: Frame) -> GateResult: + def immigration(_frame: Frame, *, time_period: int) -> GateResult: + assert time_period == pool_tool.POOL_TIME_PERIOD order.append("immigration") if terminal == "immigration_red": return GateResult( diff --git a/tools/build_us_fiscal_refresh_release.py b/tools/build_us_fiscal_refresh_release.py index 327f859021..31b587e36e 100644 --- a/tools/build_us_fiscal_refresh_release.py +++ b/tools/build_us_fiscal_refresh_release.py @@ -12276,7 +12276,7 @@ def _main(argv: Sequence[str] | None = None) -> None: seed=args.seed, time_period=PERIOD, ) - immigration_gate = us_immigration_composition_gate(base_frame) + immigration_gate = us_immigration_composition_gate(base_frame, time_period=PERIOD) if not immigration_gate.passed: if telemetry is not None: telemetry.stage( @@ -13831,7 +13831,9 @@ def _main(argv: Sequence[str] | None = None) -> None: # labels remain structurally valid. Re-evaluate the composition contract # at delivered weights and use this final verdict in every terminal gate # and release artifact below. - final_immigration_gate = us_immigration_composition_gate(export_frame) + final_immigration_gate = us_immigration_composition_gate( + export_frame, time_period=PERIOD + ) result = _without_calibrated_frames( result, export_frame=export_frame if ladder_outcome is not None else None, diff --git a/tools/build_us_multispine_pool.py b/tools/build_us_multispine_pool.py index b7c9dea91d..c897f87e08 100644 --- a/tools/build_us_multispine_pool.py +++ b/tools/build_us_multispine_pool.py @@ -4063,7 +4063,7 @@ def primary_puf_producer(primary_input: Frame): simulation_frame, tail_manifest=tail_manifest, ) - immigration = us_immigration_composition_gate(current) + immigration = us_immigration_composition_gate(current, time_period=POOL_TIME_PERIOD) # Manifest conversion is itself the final canonical-authority check and # deliberately happens before publication or readiness is asserted. GateReport((completeness, battery, immigration)).to_manifest() From 246891c4a2f5ca5351d887fe34ab78c9c23124b5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 28 Sep 2026 17:33:15 -0400 Subject: [PATCH 37/49] Re-pin the seed digests, spec sha, loader golden and coverage evidence for the immigration fixes Co-Authored-By: Claude Opus 5.5 --- docs/evidence/spec-engine/us-f0-coverage.json | 16 ++++++++-------- .../build/spec_engine/inventory_coverage.py | 4 ++-- .../tests/engine/us/test_spec_engine_loader.py | 2 +- .../engine/us/test_us_multispine_pool_tool.py | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index 63665864e0..b4a7de77c6 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -1669,13 +1669,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "e3891d8e61931439e6019b5b4a5908cb38b53c1ba9b689b242443b22d02831a7", - "protocol_sha256": "eee95bf2c823c110390029e6b20676438f5db5ee88b2daf09491a348efaef6c3" + "map_sha256": "0effba9357bd0cfa1979c1e7a09ace0962928f58ac25376b4d99268deb731e9c", + "protocol_sha256": "e23d2e4a83bc1166d630674a955aa96232dd790fde8df17071e83ad6f5137c90" }, "failures": [], "observed": { - "map_sha256": "e3891d8e61931439e6019b5b4a5908cb38b53c1ba9b689b242443b22d02831a7", - "protocol_sha256": "eee95bf2c823c110390029e6b20676438f5db5ee88b2daf09491a348efaef6c3" + "map_sha256": "0effba9357bd0cfa1979c1e7a09ace0962928f58ac25376b4d99268deb731e9c", + "protocol_sha256": "e23d2e4a83bc1166d630674a955aa96232dd790fde8df17071e83ad6f5137c90" }, "status": "covered" }, @@ -1690,7 +1690,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "eee95bf2c823c110390029e6b20676438f5db5ee88b2daf09491a348efaef6c3", + "implementation_sha256": "e23d2e4a83bc1166d630674a955aa96232dd790fde8df17071e83ad6f5137c90", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1711,7 +1711,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "eee95bf2c823c110390029e6b20676438f5db5ee88b2daf09491a348efaef6c3", + "implementation_sha256": "e23d2e4a83bc1166d630674a955aa96232dd790fde8df17071e83ad6f5137c90", "protocol": "legacy-v1", "streams": [ "build_model", @@ -2612,7 +2612,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "1e3125e91306c98cd9f726f58e1f038dc292cb62e8a97c8646a19253e0448030" + "spec_sha256": "abb06736db7438f5004d0a8a3b5d91f97381bafecb6be1a45c09cf97b82662d5" } }, "report_schema_version": 3, @@ -2622,7 +2622,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "1e3125e91306c98cd9f726f58e1f038dc292cb62e8a97c8646a19253e0448030" + "spec_sha256": "abb06736db7438f5004d0a8a3b5d91f97381bafecb6be1a45c09cf97b82662d5" }, "status": "pass" } diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 97ab3902f8..92e774ee06 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -372,8 +372,8 @@ "late_schedule": "88bc9243a3518982ae951c3de21bd55877e296ce4fcb183b9bee420d3a684b10", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "fdf23da429f8c501e198a2f5b719763d523a41565c2153fdf17b0344f21bcd7c", - "seed_map": "e3891d8e61931439e6019b5b4a5908cb38b53c1ba9b689b242443b22d02831a7", - "seed_protocol": "eee95bf2c823c110390029e6b20676438f5db5ee88b2daf09491a348efaef6c3", + "seed_map": "0effba9357bd0cfa1979c1e7a09ace0962928f58ac25376b4d99268deb731e9c", + "seed_protocol": "e23d2e4a83bc1166d630674a955aa96232dd790fde8df17071e83ad6f5137c90", "source_manifest": "00825729a6a7271affb1b1c76781f1cc271e53f201a8de181567a6010c1e01d4", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", diff --git a/packages/microcosm-build/tests/engine/us/test_spec_engine_loader.py b/packages/microcosm-build/tests/engine/us/test_spec_engine_loader.py index 38b7c2504e..e6a088a7b4 100644 --- a/packages/microcosm-build/tests/engine/us/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/engine/us/test_spec_engine_loader.py @@ -13,7 +13,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # Pin the domain separator, normalization rules, schema-set receipt, and # exact normative projection as one reviewable golden vector. assert first.spec_sha256 == ( - "f0b761c8cadf43f1600127712d526941c552217dc9c7d57df4cae5a1910808d1" + "646929f66c8496d40b4afec6c37d74a1d69dd8085b0610e1252e39fbd774ba5f" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") diff --git a/packages/microcosm-build/tests/engine/us/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/engine/us/test_us_multispine_pool_tool.py index 6700de0dfb..a11cf6c618 100644 --- a/packages/microcosm-build/tests/engine/us/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/engine/us/test_us_multispine_pool_tool.py @@ -363,7 +363,7 @@ def capture_equality(expected: object, actual: object) -> None: "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "1e3125e91306c98cd9f726f58e1f038dc292cb62e8a97c8646a19253e0448030", + "spec_sha256": "abb06736db7438f5004d0a8a3b5d91f97381bafecb6be1a45c09cf97b82662d5", }, } From 6cca304b36d07066560621c5a43b59182e29b968 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 29 Sep 2026 20:40:19 -0400 Subject: [PATCH 38/49] Property-test the humanitarian imputation's promised invariants 28 Hypothesis properties over synthetic ASEC/ACS person tables (6-32 rows, a third humanitarian-profile or DACA-cohort rows, periods 2024/2025/2030) for every invariant immigration.py states: NONE <=> UNDOCUMENTED; mutually exclusive humanitarian draws that never select an excluded cohort; the weight-to-target selection contract; seed determinism and lineage keying; the tiered spill (non-increasing Pew count, closure within tolerance, never touching the other margin while an outside row remains); and a differential between the spill's Pew projection and the composition gate's, with the gate evaluated at the stage's own period. Two example regression tests pin the period and closure-tolerance fixes of 5a89fed1e; each fails with its fix reverted. Co-Authored-By: Claude Opus 5.5 --- .../engine_free/us/test_us_immigration.py | 95 + .../us/test_us_immigration_properties.py | 2054 +++++++++++++++++ 2 files changed, 2149 insertions(+) create mode 100644 packages/microcosm-build/tests/engine_free/us/test_us_immigration_properties.py diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_immigration.py b/packages/microcosm-build/tests/engine_free/us/test_us_immigration.py index d62735d421..47ffdb982a 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_immigration.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_immigration.py @@ -365,6 +365,40 @@ def test_student_spill_exhausts_nonworkers_before_touching_a_worker( assert int((in_pew_universe & in_labor_force).sum()) == 1 assert status.iloc[2] == "UNDOCUMENTED" + def test_exact_student_closure_residue_never_spills_a_worker( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + # Regression for 5a89fed1e. The student control 0.3 equals, in exact + # arithmetic, the two working students' 0.1 + 0.2, so spilling the + # one nonworking student (weight 1.0) closes the student excess + # exactly. In binary floating point the recomputed remainder is + # 0.1 + 0.2 - 0.3 = 5.6e-17 > 0; compared with zero, that residue + # opened the second tier and spilled a worker the worker control + # still needed. + person = _person_table( + [ + _noncitizen(A_HSCOL=2, person_weight=1.0), + _noncitizen(A_HSCOL=2, A_LFSR=1, person_weight=0.1), + _noncitizen(A_HSCOL=2, A_LFSR=1, person_weight=0.2), + ] + ) + assert float(np.asarray([0.1, 0.2]).sum()) - 0.3 > 0.0 + for seed in range(4): + output = _run(person, students=0.3, workers=100.0, seed=seed) + assert output["ssn_card_type"].tolist() == [ + "NON_CITIZEN_VALID_EAD", + "NONE", + "NONE", + ], seed + + # Sensitivity: without the closure tolerance the same case spills a + # worker, so the assertions above bind on the tolerance. + monkeypatch.setattr( + immigration_module, "_SPILL_CLOSURE_RELATIVE_TOLERANCE", 0.0 + ) + output = _run(person, students=0.3, workers=100.0, seed=0) + assert output.loc[1:, "ssn_card_type"].eq("NON_CITIZEN_VALID_EAD").any() + def test_below_control_counts_spill_nothing(self) -> None: person = _person_table( [_noncitizen(A_LFSR=1, WSAL_VAL=10_000.0), _noncitizen(A_HSCOL=2)] @@ -1406,6 +1440,67 @@ def test_passes_on_plausible_composition(self) -> None: ) assert gate.passed, gate.failures + def test_gate_rederives_the_daca_cohort_at_the_stage_period(self) -> None: + # Regression for 5a89fed1e: the gate re-derived cohort evidence at a + # hard-coded 2024. A worker who arrived in 2007 (PEINUSYR 20, the + # 2006-2007 bin) and is 33 entered at 16 when measured in 2024, + # outside the DACA cohort ("arrived by 2007 before age 16"), but at + # 15 when measured in 2025, inside it. The residual worker spill + # gives the row an EAD at either period. + rows = ( + [{"PRCITSHP": 1} for _ in range(90)] + + [_noncitizen() for _ in range(9)] + + [_noncitizen(PEINUSYR=20, A_AGE=33, A_LFSR=1)] + ) + controls = ImmigrationControls( + undocumented=UndocumentedControls( + workers=0.5, + students=100.0, + population_anchor=10.0, + sources=_plausible_controls().undocumented.sources, + ), + humanitarian=_humanitarian_draws(), + ) + + def stage_frame(period: int) -> Frame: + output = run_source_stage( + _stage_spec(workers=0.5, students=100.0, anchor=10.0), + tables={"person": _person_table(rows)}, + operation_handlers=_HANDLERS, + config=SourceRuntimeConfig(seed=0, target_year=period), + ) + return _us_frame(output.drop(columns=["person_weight"]).to_dict("records")) + + by_period = {period: stage_frame(period) for period in (2024, 2025)} + labels = { + period: frame.table("person").iloc[-1][list(US_IMMIGRATION_OUTPUT_COLUMNS)] + for period, frame in by_period.items() + } + assert labels[2024].tolist() == [ + "NON_CITIZEN_VALID_EAD", + "LEGAL_PERMANENT_RESIDENT", + ] + assert labels[2025].tolist() == ["NON_CITIZEN_VALID_EAD", "DACA"] + + cohort_failure = ( + "1 person(s) violate the source-aware Cuban/Haitian entrant or DACA " + "cohort contract." + ) + for stage_period, frame in by_period.items(): + for gate_period in (2024, 2025): + gate = us_immigration_composition_gate( + frame, time_period=gate_period, controls=controls + ) + invalid = gate.details["evidence_derived_status_compatibility"][ + "invalid_rows" + ] + if gate_period == stage_period: + assert gate.passed, (stage_period, gate.failures) + assert invalid == 0 + else: + assert gate.failures == (cohort_failure,) + assert invalid == 1 + def test_fails_when_columns_missing(self) -> None: gate = us_immigration_composition_gate( time_period=TIME_PERIOD, diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_immigration_properties.py b/packages/microcosm-build/tests/engine_free/us/test_us_immigration_properties.py new file mode 100644 index 0000000000..66fd4b0d4f --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/us/test_us_immigration_properties.py @@ -0,0 +1,2054 @@ +"""Property tests for the US immigration-status stage (microcosm #225, #767). + +Every property below quotes the docstring, comment or runtime check in +``microcosm.build.us_runtime.immigration`` that promises it. Properties marked +*deduced* combine two stated mechanisms rather than restating one sentence +(for example, draws keyed by person identity plus threshold selection imply +row-order equivariance). The example tests live in ``test_us_immigration.py``. + +Invariants checked, by function: + +- ``_select_weight_to_target``: the selection is the candidates at or below + one draw threshold, so tie groups are never split; a non-positive amount + selects nothing; a feasible selection reaches the amount and overshoots it + by less than its final tie group, and an infeasible one takes every + candidate; it equals an independent group-walk reference; it is row-order + equivariant (deduced). +- ``_stable_person_draws``: draws depend only on seed, salt and the most + complete stable lineage (equal keys draw equally, different keys draw + differently), reruns repeat and row order does not matter (deduced). +- ``_spill_pew_unauthorized_excess``: only in-scope residual non-citizens move, + and only to EAD; Pew-universe membership never grows; a margin within the + closure tolerance of its control spills nothing; every closable excess is + closed and an unclosable one exhausts the removable pool; the undershoot is + less than one flipped row; rows inside the other margin are touched only + after every removable row outside it, and only when removing those leaves + more than rounding residue (explicit example: the minimal exact-closure + case, where 0.1 + 0.2 - 0.3 > 0 in binary floating point). +- The stage (``derive_us_immigration_status_from_manifest``): ``NONE`` pairs + exactly with ``UNDOCUMENTED`` and ``CITIZEN`` with measured citizenship; + source columns pass through; the status column is the documented function of + SSN code, humanitarian mark and evidence; the indicator-documented pool is + exactly the non-citizens carrying a documented indicator; humanitarian + labels respect their exclusions and SSN pools, are mutually exclusive, reach + their targets or exhaust their pools, precede the EAD spill, and sit inside + the ASEC arrival windows the module's comments state; the output is + bit-reproducible, row-order equivariant (deduced) and consistent across + support clones; the Pew margins close whenever the residual pool can close + them. +- ``with_us_immigration_inputs``: labels every person exactly as the manifest + handler does, leaves weights and other tables alone, and is idempotent. +- Differential: stage output satisfies every structural check of + ``us_immigration_composition_gate`` at the stage's own period (explicit + example: a row whose DACA-cohort membership changes between 2024 and + 2025), and the spill's ``_pew_unauthorized_projection_mask`` selects + exactly the rows the gate counts as Pew-unauthorized. +- ``reconcile_us_immigration_humanitarian_transfer``: its output satisfies the + gate's structural contract; each receipt's achieved population is its + immutable plus selected population; and + ``us_immigration_humanitarian_transfer_selection_masks`` replays the + reconciliation's mutable-row selections exactly (differential). + +Independence limits, stated so no test claims more than it shows: + +- The candidate pools (origin x arrival window x SSN pool) of the draw-target + and transfer properties come from the module's own + ``_humanitarian_draw_candidates``; those properties check selection against + target, not the windows. Only + ``test_humanitarian_rows_sit_inside_the_asec_windows_the_comments_state`` + checks windows, against values transcribed from the module's comments. + That is a comment-versus-table consistency check, not external validation. +- Reconciliation raises ``RuntimeError`` before returning a receipt that + misses its tie-mass bounds or changes an immutable row, so asserting those + on returned receipts only confirms the receipt records what was checked. +- ``_facts`` reuses the module's arrival-midpoint and Cuba/Haiti code tables; + the DACA thresholds come from the module docstring. + +Hypothesis reaches the engine-free CI job through ``uv sync --all-packages`` +(microcosm-graph's dev group), so it is imported directly: a missing +dependency fails loudly rather than skipping these tests. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass, replace + +import numpy as np +import pandas as pd +import pytest +from hypothesis import event, example, given, settings +from hypothesis import strategies as st + +import microcosm.build.us_runtime.immigration as immigration +from microcosm.build.source_manifest import SourceOperationSpec +from microcosm.build.source_runtime import ( + SourceRuntimeConfig, + SourceRuntimeContext, + SourceRuntimeError, +) +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights + +#: Unit properties are cheap; stage properties run the full derivation per +#: example. ``database=None`` keeps runs from saving examples into the +#: checkout. +_UNIT = settings(max_examples=100, deadline=None, database=None) +_STAGE = settings(max_examples=50, deadline=None, database=None) +_TRANSFER = settings(max_examples=30, deadline=None, database=None) +_FRAME = settings(max_examples=20, deadline=None, database=None) + +_TIME_PERIOD = 2024 +#: Stage periods. The DACA cohort's age-at-entry test moves with the period, +#: so the stage and the composition gate must agree at each of them. +_PERIODS = (2024, 2025, 2030) + +_HUMANITARIAN_STATUSES = frozenset( + immigration._HUMANITARIAN_STATUS_BY_CATEGORY.values() +) +_INDICATOR_POOL_STATUSES = frozenset({"REFUGEE", "ASYLEE", "DEPORTATION_WITHHELD"}) + +#: Every origin in the humanitarian codebook tables. +_HUMANITARIAN_ORIGIN_CODES: tuple[int, ...] = tuple( + sorted( + { + *( + code + for codes in immigration._PAROLE_ORIGIN_CODES.values() + for code in codes + ), + *immigration._REFUGEE_ORIGIN_CODES, + *immigration._ASYLEE_ORIGIN_CODES, + *( + code + for codes, _ in immigration._TPS_ORIGIN_CODES.values() + for code in codes + ), + } + ) +) +#: Birth countries the stage distinguishes: the United States, a generic +#: origin, Cuba/Haiti, and the humanitarian origins. +_BIRTH_CODES: tuple[int, ...] = tuple( + sorted( + {57, 120, 303, *immigration._CUBAN_HAITIAN_BIRTH_CODES} + | set(_HUMANITARIAN_ORIGIN_CODES) + ) +) + +#: ASEC-UA legal-status indicators as the module docstring lists them +#: ("Medicare/Medicaid/SSI/Social Security receipt, federal pensions, +#: IHS/CHAMPVA/military coverage, government employment, subsidized housing, +#: veteran status"), with the neutral value each column takes otherwise. +#: Pre-1982 arrival is drawn through ``PEINUSYR``. Naturalization-eligibility +#: is omitted: the code reads it from ``PRCITSHP == 4``, a citizen code, so it +#: can never move a ``PRCITSHP == 5`` row and no property here depends on it. +_NEUTRAL_INDICATORS: Mapping[str, float | int] = { + "MCARE": 2, + "CAID": 2, + "IHSFLG": 2, + "CHAMPVA": 2, + "MIL": 2, + "PEN_SC1": 0, + "PEN_SC2": 0, + "RESNSS1": 0, + "RESNSS2": 0, + "SS_YN": 2, + "SSI_YN": 2, + "PEIO1COW": 0, + "A_MJOCC": 0, + "PEAFEVER": 2, + "SPM_CAPHOUSESUB": 0.0, +} +_INDICATORS: tuple[tuple[str, float | int], ...] = ( + ("MCARE", 1), + ("CAID", 1), + ("IHSFLG", 1), + ("CHAMPVA", 1), + ("MIL", 1), + ("PEN_SC1", 3), + ("PEN_SC2", 3), + ("RESNSS1", 2), + ("RESNSS2", 2), + ("SS_YN", 1), + ("SSI_YN", 1), + ("PEIO1COW", 1), + ("PEIO1COW", 2), + ("PEIO1COW", 3), + ("A_MJOCC", 11), + ("PEAFEVER", 1), + ("SPM_CAPHOUSESUB", 250.0), +) + +#: ASEC arrival windows transcribed from the module's comments, not read from +#: its tables. Birth codes are the module's own (its comment says they were +#: "verified against Census CPS technical documentation"); the windows are the +#: comments' words. ``_RECENT_ARRIVAL_CODES``: "The 2024 ASEC distinguishes +#: 2020-2021 (27) from 2022-2024 (28) ... U4U and CHNV cannot draw from code +#: 27, while Operation Allies Welcome can"; ``_PAROLE_ORIGIN_CODES``: "Operation +#: Allies Welcome (Afghanistan), Uniting for Ukraine, and the +#: non-Cuban/Haitian CHNV nationalities". +_PAROLE_DOCUMENTED_WINDOWS: Mapping[int, frozenset[int]] = { + 200: frozenset({27, 28}), # Afghanistan (Operation Allies Welcome) + 164: frozenset({28}), # Ukraine (Uniting for Ukraine) + 315: frozenset({28}), # Nicaragua (CHNV) + 373: frozenset({28}), # Venezuela (CHNV) +} +#: ``_TPS_ORIGIN_CODES``: "Legacy designations bind hard (El Salvador +#: 2001-02-13 -> code 17; Honduras/Nicaragua 1998-12-30 -> code 16; Nepal +#: 2015-06-24 -> code 24)". +_TPS_DOCUMENTED_MAX_ARRIVAL_CODE: Mapping[int, int] = { + 312: 17, # El Salvador + 314: 16, # Honduras + 315: 16, # Nicaragua + 229: 24, # Nepal +} +#: ``_TPS_ORIGIN_CODES``: "Ukraine and Afghanistan TPS registrants are carried +#: by the parole draw ...; Haiti TPS is carried by CUBAN_HAITIAN_ENTRANT", and +#: Cuba/Haiti-born persons "are excluded from every humanitarian draw". +_TPS_DOCUMENTED_EXCLUDED_BIRTHS = frozenset({164, 200, 327, 332}) +#: ``_ASYLEE_ARRIVAL_CODES``: "grantees in 2022-2024 overwhelmingly arrived +#: 2016+"; the 2016-2017 bin (midpoint 2017) is code 25. +_ASYLEE_DOCUMENTED_MIN_ARRIVAL_CODE = 25 + + +def _draw_labels() -> tuple[str, ...]: + labels: list[str] = [] + for category in immigration.HUMANITARIAN_STATUS_CATEGORIES: + origins = immigration._PER_ORIGIN_HUMANITARIAN_CATEGORIES.get(category) + if origins is None: + labels.append(category) + else: + labels.extend(f"{category}:{origin}" for origin in origins) + return tuple(labels) + + +_DRAW_LABELS = _draw_labels() + +#: One zero-target draw per manifest label, for evaluating candidate pools. +_TEMPLATE_DRAWS: tuple[immigration.HumanitarianDraw, ...] = tuple( + immigration.HumanitarianDraw( + category=label.split(":", 1)[0], + origin=label.split(":", 1)[1] if ":" in label else None, + status=immigration._HUMANITARIAN_STATUS_BY_CATEGORY[label.split(":", 1)[0]], + target=0.0, + source="https://example.org/template", + ) + for label in _DRAW_LABELS +) + + +def _tolerance(weights: np.ndarray) -> float: + return 1e-9 * max(1.0, float(np.abs(weights).sum())) + + +# --------------------------------------------------------------------------- +# _select_weight_to_target +# --------------------------------------------------------------------------- + +#: A few repeated draw values force the tie groups clones produce. +_TIED_DRAWS = (0.0, 0.125, 0.25, 0.5, 0.75, 0.875) + + +@st.composite +def _selection_cases(draw, *, integer_weights: bool = False): + n = draw(st.integers(0, 24)) + weight = ( + st.integers(0, 50).map(float) + if integer_weights + else st.floats(0.0, 1e6, allow_nan=False, allow_subnormal=False) + ) + weights = np.asarray( + draw(st.lists(weight, min_size=n, max_size=n)), dtype=np.float64 + ) + draws = np.asarray( + draw( + st.lists( + st.one_of( + st.sampled_from(_TIED_DRAWS), + st.floats(0.0, 1.0, exclude_max=True), + ), + min_size=n, + max_size=n, + ) + ), + dtype=np.float64, + ) + candidates = np.asarray( + draw(st.lists(st.booleans(), min_size=n, max_size=n)), dtype=bool + ) + total = float(weights[candidates].sum()) + amount = draw( + st.one_of( + st.floats(-10.0, 0.0), + st.floats(0.0, max(1.5 * total, 1.0)), + st.just(total), + ) + ) + return candidates, weights, draws, amount + + +def _reference_selection( + candidates: np.ndarray, weights: np.ndarray, draws: np.ndarray, amount: float +) -> np.ndarray: + """Walk distinct draw values upward; stop at the first group reaching it.""" + + if amount <= 0 or not candidates.any(): + return np.zeros(len(candidates), dtype=bool) + cumulative = 0.0 + for value in sorted(set(draws[candidates].tolist())): + cumulative += float(weights[candidates & (draws == value)].sum()) + if cumulative >= amount: + return candidates & (draws <= value) + return candidates.copy() + + +@_UNIT +@given(_selection_cases()) +def test_selection_is_a_draw_threshold_prefix_that_never_splits_ties(case) -> None: + """``_select_weight_to_target``: "every candidate at or below the draw + where the cumulative weight first reaches ``amount`` is selected. Records + sharing a draw ... land on the same side of the threshold" and "A + non-positive ``amount`` selects nothing".""" + + candidates, weights, draws, amount = case + selected = immigration._select_weight_to_target(candidates, weights, draws, amount) + assert selected.dtype == bool and selected.shape == candidates.shape + assert not (selected & ~candidates).any() + if amount <= 0 or not candidates.any(): + assert not selected.any() + return + threshold = draws[selected].max() + np.testing.assert_array_equal(selected, candidates & (draws <= threshold)) + + +@_UNIT +@given(_selection_cases()) +def test_selection_reaches_amount_and_overshoots_by_less_than_its_tie_group( + case, +) -> None: + """``_select_weight_to_target`` selects "~``amount`` of weighted mass": + the cumulative weight "first reaches ``amount``" at the threshold and "the + final tie group may overshoot the amount". When the candidates cannot + reach it, every candidate is selected.""" + + candidates, weights, draws, amount = case + selected = immigration._select_weight_to_target(candidates, weights, draws, amount) + if amount <= 0 or not candidates.any(): + return + tolerance = _tolerance(weights) + total = float(weights[candidates].sum()) + chosen = float(weights[selected].sum()) + if total < amount - tolerance: + event("amount infeasible") + np.testing.assert_array_equal(selected, candidates) + return + event("amount feasible") + tie = selected & (draws == draws[selected].max()) + assert chosen >= amount - tolerance + assert chosen - float(weights[tie].sum()) < amount + tolerance + + +@_UNIT +@given(_selection_cases(integer_weights=True)) +def test_selection_matches_an_independent_group_walk(case) -> None: + """Differential: the ``np.searchsorted`` implementation equals a plain + group walk over distinct draw values (integer weights keep both sums + exact, so the comparison is bitwise).""" + + candidates, weights, draws, amount = case + np.testing.assert_array_equal( + immigration._select_weight_to_target(candidates, weights, draws, amount), + _reference_selection(candidates, weights, draws, amount), + ) + + +@_UNIT +@given(_selection_cases(integer_weights=True), st.data()) +def test_selection_is_row_order_equivariant(case, data) -> None: + """Deduced: a threshold on draws with whole tie groups on one side depends + only on the (draw, weight) multiset, not on row positions. Integer weights + keep cumulative sums order-independent.""" + + candidates, weights, draws, amount = case + order = np.asarray( + data.draw(st.permutations(range(len(candidates)))), dtype=np.int64 + ) + selected = immigration._select_weight_to_target(candidates, weights, draws, amount) + permuted = immigration._select_weight_to_target( + candidates[order], weights[order], draws[order], amount + ) + np.testing.assert_array_equal(permuted, selected[order]) + + +# --------------------------------------------------------------------------- +# _stable_person_draws +# --------------------------------------------------------------------------- + + +@_UNIT +@given( + keys=st.lists(st.integers(0, 6), min_size=1, max_size=16), + seed=st.integers(0, 2**32), + salt=st.text(max_size=8), + data=st.data(), +) +def test_person_draws_are_keyed_by_identity_alone(keys, seed, salt, data) -> None: + """``_stable_person_draws``: "Deterministic uniform draws keyed by stable + person identity" that give "every clone of one source person the same + draw". Row-order equivariance is deduced from the per-key hash.""" + + person = pd.DataFrame({"person_id": keys}) + draws = immigration._stable_person_draws(person, seed=seed, salt=salt) + assert draws.shape == (len(keys),) + assert ((draws >= 0.0) & (draws < 1.0)).all() + key_array = np.asarray(keys) + for key in set(keys): + assert len(set(draws[key_array == key].tolist())) == 1 + np.testing.assert_array_equal( + draws, immigration._stable_person_draws(person, seed=seed, salt=salt) + ) + order = list(data.draw(st.permutations(range(len(keys))))) + reordered = person.iloc[order].reset_index(drop=True) + np.testing.assert_array_equal( + immigration._stable_person_draws(reordered, seed=seed, salt=salt), + draws[order], + ) + + +def _equal_draws_iff_equal_keys(draws: np.ndarray, keys: list[tuple]) -> None: + """Draws agree exactly for equal keys and differ for different ones (a + 64-bit blake2b collision between distinct keys has probability ~2**-64).""" + + for i in range(len(keys)): + for j in range(i + 1, len(keys)): + assert (draws[i] == draws[j]) == (keys[i] == keys[j]), (i, j) + + +@st.composite +def _lineage_rows(draw) -> list[tuple[int | None, int | None]]: + """(source_household_id, source_person_id) rows, each lineage alternative + about equally often: complete, household missing somewhere (legacy pair), + or source person missing somewhere (``person_id`` fallback). Small id + ranges make one source person recur across households.""" + + n = draw(st.integers(2, 10)) + ids = st.integers(1, 3) + rows = draw(st.lists(st.tuples(ids, ids), min_size=n, max_size=n)) + mode = draw(st.sampled_from(("full", "legacy", "person_id"))) + if mode == "full": + return rows + gaps = draw(st.sets(st.integers(0, n - 1), min_size=1)) + if mode == "legacy": + return [(None, p) if i in gaps else (h, p) for i, (h, p) in enumerate(rows)] + household_gaps = draw(st.booleans()) + return [ + ((None if household_gaps else h), None) if i in gaps else (h, p) + for i, (h, p) in enumerate(rows) + ] + + +@_UNIT +@given(rows=_lineage_rows(), seed=st.integers(0, 2**32)) +def test_person_draws_use_the_most_complete_lineage(rows, seed) -> None: + """``_stable_person_draws``: support clones key on ``source_year`` / + ``source_household_id`` / ``source_person_id``, and "frames without full + source ids fall back to the legacy source pair or ``person_id``"; with no + complete alternative it raises. Draws are equal exactly when the chosen + lineage is equal, so two rows sharing ``source_year`` and + ``source_person_id`` but not ``source_household_id`` draw differently + whenever the full lineage is complete (a draw that ignored the household + would tie them).""" + + household = [household for household, _ in rows] + source_person = [person for _, person in rows] + person = pd.DataFrame( + { + "source_year": [2024] * len(rows), + "source_household_id": pd.array(household, dtype="Int64"), + "source_person_id": pd.array(source_person, dtype="Int64"), + "person_id": np.arange(1, len(rows) + 1), + } + ) + + def draws(frame: pd.DataFrame) -> np.ndarray: + return immigration._stable_person_draws(frame, seed=seed, salt="lineage") + + full = draws(person) + if None not in source_person and None not in household: + event("full lineage complete") + if len(set(source_person)) < len( + set(zip(household, source_person, strict=True)) + ): + event("same source person in different households") + np.testing.assert_array_equal(full, draws(person.drop(columns="person_id"))) + _equal_draws_iff_equal_keys( + full, list(zip(household, source_person, strict=True)) + ) + elif None not in source_person: + event("legacy lineage") + np.testing.assert_array_equal( + full, draws(person.drop(columns="source_household_id")) + ) + _equal_draws_iff_equal_keys(full, [(key,) for key in source_person]) + else: + event("person_id fallback") + np.testing.assert_array_equal(full, draws(person[["person_id"]])) + _equal_draws_iff_equal_keys(full, [(key,) for key in range(len(rows))]) + with pytest.raises(SourceRuntimeError, match="lineage"): + draws(person.drop(columns="person_id")) + + +# --------------------------------------------------------------------------- +# _spill_pew_unauthorized_excess +# --------------------------------------------------------------------------- + +_MARKS = ("", "", *sorted(_HUMANITARIAN_STATUSES)) +#: Decimal weights whose binary sums leave rounding residue (0.1 + 0.2 is not +#: 0.3), the case the spill's closure tolerance exists for. +_DECIMAL_WEIGHTS = (0.1, 0.2, 0.3, 0.7, 1.1) +_WEIGHT = st.one_of( + st.just(0.0), + st.integers(1, 5_000).map(float), + st.sampled_from(_DECIMAL_WEIGHTS), + st.floats(0.5, 5e5, allow_nan=False, allow_subnormal=False), +) +#: Spill rows lean further toward decimal weights: the exact-closure residue +#: needs them on both sides of the tier boundary. +_SPILL_WEIGHT = st.one_of(st.sampled_from(_DECIMAL_WEIGHTS), _WEIGHT) +#: ``_spill_pew_unauthorized_excess``: "Treat anything within a billionth of +#: the in-scope weight as closed". +_DOCUMENTED_CLOSURE_TOLERANCE = 1e-9 + + +@dataclass(frozen=True) +class _SpillCase: + ssn: np.ndarray + marks: np.ndarray + weights: np.ndarray + noncitizens: np.ndarray + scope: np.ndarray + preserve: np.ndarray + retained: np.ndarray + target: float + seed: int + + +def _projection_of( + ssn: np.ndarray, marks: np.ndarray, retained: np.ndarray +) -> np.ndarray: + return immigration._pew_unauthorized_projection_mask( + ssn_codes=ssn, + humanitarian_marks=marks, + retained_ead_cohort=retained, + ) + + +@st.composite +def _spill_cases(draw) -> _SpillCase: + """Mostly residual in-scope non-citizens, with a target that is a share of + the in-scope Pew count (so the spill has work to do), exactly that count + less a subset of first-tier removable rows (the exact-closure case), or at + or above the count (so it must do nothing).""" + + n = draw(st.integers(2, 24), label="rows") + rows = draw( + st.lists( + st.tuples( + st.sampled_from((True, True, True, False)), + st.sampled_from((0, 0, 0, 2, 3)), + st.sampled_from(_MARKS), + st.sampled_from((True, True, False)), + st.booleans(), + st.sampled_from((False, False, False, True)), + _SPILL_WEIGHT, + ), + min_size=n, + max_size=n, + ) + ) + noncitizens = np.asarray([row[0] for row in rows], dtype=bool) + ssn = np.where(noncitizens, [row[1] for row in rows], 1).astype(np.int64) + # The stage never marks a residual (code 0) row: parole/TPS selections + # leave that pool when they are marked. + marks = np.where(noncitizens & (ssn != 0), [row[2] for row in rows], "").astype( + "U24" + ) + scope = np.asarray([row[3] for row in rows], dtype=bool) + preserve = np.asarray([row[4] for row in rows], dtype=bool) + retained = np.asarray([row[5] for row in rows], dtype=bool) + weights = np.asarray([row[6] for row in rows], dtype=np.float64) + initial = float(weights[scope & _projection_of(ssn, marks, retained)].sum()) + kind = draw( + st.sampled_from(("share", "exact", "exact", "above")), label="target kind" + ) + if kind == "share": + target = draw(st.floats(0.0, 1.0)) * initial + elif kind == "exact": + # Removing the whole first tier closes the gap exactly in real + # arithmetic, so the first tier's passes must take all of it; with + # decimal weights the recomputed sum then differs from the target by + # rounding residue alone. + first_tier = scope & ~preserve & noncitizens & (ssn == 0) & ~retained + subset = first_tier + if not draw(st.booleans(), label="whole first tier"): + subset = first_tier & np.asarray( + draw(st.lists(st.booleans(), min_size=n, max_size=n)), dtype=bool + ) + target = max(0.0, initial - float(weights[subset].sum())) + else: + target = initial * draw(st.floats(1.0, 1.5)) + event(f"spill target: {kind}") + return _SpillCase( + ssn=ssn, + marks=marks, + weights=weights, + noncitizens=noncitizens, + scope=scope, + preserve=preserve, + retained=retained, + target=target, + seed=draw(st.integers(0, 2**31 - 1)), + ) + + +def _projection(case: _SpillCase, ssn: np.ndarray) -> np.ndarray: + return _projection_of(ssn, case.marks, case.retained) + + +def _spill(case: _SpillCase) -> tuple[np.ndarray, np.ndarray]: + """Run the spill on copies; return (codes after, marks after).""" + + ssn = case.ssn.copy() + marks = case.marks.copy() + immigration._spill_pew_unauthorized_excess( + pd.DataFrame({"person_id": np.arange(1, len(ssn) + 1)}), + ssn, + marks, + case.weights.copy(), + noncitizens=case.noncitizens, + scope=case.scope, + preserve_scope=case.preserve, + retained_ead_cohort=case.retained, + target=case.target, + seed=case.seed, + salt="immigration:property", + ) + return ssn, marks + + +@_UNIT +@given(_spill_cases()) +def test_spill_moves_only_in_scope_residual_noncitizens_and_only_to_ead( + case, +) -> None: + """``_spill_pew_unauthorized_excess`` "Spill[s] a broad Pew-universe margin + to EAD": candidates are ``(ssn_codes == 0) & noncitizens & scope`` and the + only write is ``ssn_codes[selected] = 2``. It is deterministic.""" + + after, marks = _spill(case) + changed = after != case.ssn + if changed.any(): + event("the spill moved rows") + assert not (changed & ~((case.ssn == 0) & case.noncitizens & case.scope)).any() + assert (after[changed] == 2).all() + np.testing.assert_array_equal(marks, case.marks) + np.testing.assert_array_equal(_spill(case)[0], after) + + +@_UNIT +@given(_spill_cases()) +def test_spill_never_grows_the_pew_universe(case) -> None: + """The spill exists to reduce a Pew-universe excess: moving a residual row + to EAD either removes it from ``_pew_unauthorized_projection_mask`` or, + for "DACA and residual Cuban/Haitian cohort rows", leaves it counted. + Membership therefore only shrinks, in scope and everywhere else, so the + weighted Pew count never rises.""" + + after, _ = _spill(case) + before_mask = _projection(case, case.ssn) + after_mask = _projection(case, after) + assert not (after_mask & ~before_mask).any() + + +@_UNIT +@given(_spill_cases()) +def test_spill_leaves_a_margin_within_tolerance_of_its_control_untouched( + case, +) -> None: + """``_select_weight_to_target``: "a count already at or below its control + spills nothing"; the spill returns while ``current_excess() <= + closed_within``, and its comment says "Treat anything within a billionth + of the in-scope weight as closed".""" + + included = _projection(case, case.ssn) + excess = float(case.weights[case.scope & included].sum()) - case.target + closed_within = _DOCUMENTED_CLOSURE_TOLERANCE * max( + 1.0, float(case.weights[case.scope].sum()) + ) + if excess > closed_within: + return + event("margin already closed") + after, _ = _spill(case) + np.testing.assert_array_equal(after, case.ssn) + + +@_UNIT +@given(_spill_cases()) +def test_spill_closes_every_closable_excess_and_otherwise_exhausts_the_pool( + case, +) -> None: + """The workers/students controls are enforced "until the broad reported + universes match their controls" (module docstring, step 3): each pass + selects the current excess, and the corrective draw "spills rows outside + those retained cohorts". So the in-scope Pew count ends at or below the + target whenever removing every in-scope residual non-retained row could + get it there, and otherwise every such row has been spilled.""" + + tolerance = _tolerance(case.weights) + after, _ = _spill(case) + initial = float(case.weights[case.scope & _projection(case, case.ssn)].sum()) + final = float(case.weights[case.scope & _projection(case, after)].sum()) + removable = case.scope & case.noncitizens & (case.ssn == 0) & ~case.retained + floor = initial - float(case.weights[removable].sum()) + if floor <= case.target - tolerance: + event("excess closable") + assert final <= case.target + tolerance + elif floor > case.target + tolerance: + event("excess not closable") + assert (after[removable] == 2).all() + + +@_UNIT +@given(_spill_cases()) +def test_spill_undershoots_its_control_by_less_than_one_spilled_row(case) -> None: + """Each pass asks ``_select_weight_to_target`` for exactly the current + excess, which "may overshoot the amount" only by its final tie group + (one row here: every row has its own ``person_id`` draw). The in-scope + Pew count therefore never lands a full spilled row below the target.""" + + after, _ = _spill(case) + changed = after != case.ssn + if not changed.any(): + return + final = float(case.weights[case.scope & _projection(case, after)].sum()) + largest = float(case.weights[changed].max()) + assert final > case.target - largest - _tolerance(case.weights) + + +#: The minimal exact-closure case: in real arithmetic the student control 0.3 +#: equals the two worker-students' 0.1 + 0.2, so spilling the one non-worker +#: student (weight 1.0) closes the gap, yet ``0.1 + 0.2 - 0.3`` is 5.6e-17 in +#: binary floating point. ``test_us_immigration.py`` pins the same case end +#: to end through the stage. +_EXACT_CLOSURE_RESIDUE_CASE = _SpillCase( + ssn=np.zeros(3, dtype=np.int64), + marks=np.full(3, "", dtype="U24"), + weights=np.asarray([1.0, 0.1, 0.2]), + noncitizens=np.ones(3, dtype=bool), + scope=np.ones(3, dtype=bool), + preserve=np.asarray([False, True, True]), + retained=np.zeros(3, dtype=bool), + target=0.3, + seed=0, +) + + +@_UNIT +@given(_spill_cases()) +@example(_EXACT_CLOSURE_RESIDUE_CASE) +def test_spill_exhausts_rows_outside_the_other_margin_first(case) -> None: + """``_spill_pew_unauthorized_excess``: "Rows outside the other controlled + margin are exhausted before any overlapping row is touched" and "a row + inside the other margin is spilled only once no row outside it can close + the gap"; the closure tolerance ("anything within a billionth of the + in-scope weight") exists so that rounding residue "never opens the next + tier". If any row inside the preserved margin moved, every removable + (non-retained residual) in-scope row outside it moved too, and removing + all of them still left more than rounding residue above the target.""" + + after, _ = _spill(case) + changed = after != case.ssn + if not (changed & case.preserve).any(): + return + event("a row inside the other margin moved") + outside = ( + case.scope + & ~case.preserve + & case.noncitizens + & (case.ssn == 0) + & ~case.retained + ) + assert (after[outside] == 2).all() + initial = float(case.weights[case.scope & _projection(case, case.ssn)].sum()) + closed_within = _DOCUMENTED_CLOSURE_TOLERANCE * max( + 1.0, float(case.weights[case.scope].sum()) + ) + left_after_outside = initial - float(case.weights[outside].sum()) - case.target + assert left_after_outside > 0.5 * closed_within + + +# --------------------------------------------------------------------------- +# The stage +# --------------------------------------------------------------------------- + + +def _generic_asec_row(weight): + return st.fixed_dictionaries( + { + "PRCITSHP": st.sampled_from((1, 2, 3, 4, 5, 5, 5, 5, 5)), + # Biased toward the origins and recent arrivals the humanitarian + # draws need. + "PENATVTY": st.one_of( + st.sampled_from(_BIRTH_CODES), + st.sampled_from(_HUMANITARIAN_ORIGIN_CODES), + ), + "PEINUSYR": st.one_of( + st.integers(0, 28), st.integers(17, 28), st.integers(24, 28) + ), + "A_AGE": st.integers(0, 85), + "A_MARITL": st.sampled_from((1, 2, 4, 7)), + "A_SPOUSE": st.sampled_from((0, 1)), + "A_HSCOL": st.sampled_from((0, 1, 2)), + "A_LFSR": st.sampled_from(immigration._CPS_LABOR_FORCE_STATUS_DOMAIN), + "indicator": st.one_of(st.none(), st.none(), st.sampled_from(_INDICATORS)), + "person_weight": weight, + } + ) + + +@st.composite +def _humanitarian_profile_row(draw, weight) -> dict: + """A non-citizen shaped for one humanitarian draw or for the DACA cohort, + or one arrival bin outside its window (so windows are exercised from both + sides). + + Inputs only: the origin and arrival tables here come from the module so + the candidate pools are reachable; no oracle reads them. + """ + + category = draw( + st.sampled_from((*immigration.HUMANITARIAN_STATUS_CATEGORIES, "daca")) + ) + age = draw(st.one_of(st.integers(25, 60), st.integers(0, 85))) + if category == "daca": + # Arrived by 2007 (module docstring, step 6) at an age measured at + # 2024: under 16 (the cohort at every period drawn) or 16-21 (outside + # it at 2024, inside it by 2030, so cohort membership depends on the + # period). Sometimes Cuba/Haiti-born, where DACA and the entrant + # class compete. + birth = draw(st.sampled_from((303, 312, 327, 332))) + arrival = draw(st.integers(8, 20)) + midpoint = immigration._ARRIVAL_YEAR_MIDPOINTS[arrival] + entry_age_at_2024 = draw(st.one_of(st.integers(0, 15), st.integers(16, 21))) + age = max(15, (_TIME_PERIOD - midpoint) + entry_age_at_2024) + elif category == "paroled_one_year": + origin = draw(st.sampled_from(tuple(immigration._PAROLE_ORIGIN_CODES))) + birth = draw(st.sampled_from(immigration._PAROLE_ORIGIN_CODES[origin])) + arrival = draw(st.sampled_from(immigration._PAROLE_ASEC_ARRIVAL_CODES[origin])) + elif category == "refugee": + birth = draw(st.sampled_from(immigration._REFUGEE_ORIGIN_CODES)) + arrival = 28 + elif category == "asylee": + birth = draw(st.sampled_from(immigration._ASYLEE_ORIGIN_CODES)) + arrival = draw(st.sampled_from(immigration._ASYLEE_ARRIVAL_CODES)) + elif category == "deportation_withheld": + birth = draw(st.sampled_from(_BIRTH_CODES)) + arrival = draw(st.integers(1, immigration._WITHHELD_MAX_ARRIVAL_CODE)) + else: + codes, latest = draw( + st.sampled_from(tuple(immigration._TPS_ORIGIN_CODES.values())) + ) + birth = draw(st.sampled_from(codes)) + arrival = draw(st.integers(1, latest)) + if draw(st.integers(0, 4)) == 0: + arrival = min(28, max(1, arrival + draw(st.sampled_from((-1, 1))))) + return { + "PRCITSHP": draw(st.sampled_from((5, 5, 5, 4))), + "PENATVTY": birth, + "PEINUSYR": arrival, + "A_AGE": age, + "A_MARITL": draw(st.sampled_from((1, 2, 4, 7))), + "A_SPOUSE": draw(st.sampled_from((0, 1))), + "A_HSCOL": draw(st.sampled_from((0, 1, 2))), + "A_LFSR": draw(st.sampled_from(immigration._CPS_LABOR_FORCE_STATUS_DOMAIN)), + # Refugee, asylee and withholding draws need an indicator; parole and + # TPS draw from either pool; a DACA-cohort row needs the residual + # pool to reach an EAD. + "indicator": draw( + st.none() + if category == "daca" + else st.one_of( + st.none(), st.sampled_from(_INDICATORS), st.sampled_from(_INDICATORS) + ) + ), + "person_weight": draw(weight), + } + + +def _asec_row(weight): + return st.one_of( + _generic_asec_row(weight), + _generic_asec_row(weight), + _humanitarian_profile_row(weight), + ) + + +def _asec_table(rows: list[dict]) -> pd.DataFrame: + records = [] + for index, row in enumerate(rows): + record = {**_NEUTRAL_INDICATORS, **row, "person_id": index + 1} + indicator = record.pop("indicator") + if indicator is not None: + column, value = indicator + record[column] = value + records.append(record) + return pd.DataFrame(records) + + +def _manifest_parameters( + *, + workers: float, + students: float, + anchor: float, + targets: Mapping[str, float], +) -> dict[str, object]: + """``derive_immigration_status`` parameters in the packaged manifest shape.""" + + humanitarian: dict[str, object] = {} + for category in immigration.HUMANITARIAN_STATUS_CATEGORIES: + origins = immigration._PER_ORIGIN_HUMANITARIAN_CATEGORIES.get(category) + if origins is None: + humanitarian[category] = { + "target": targets[category], + "source": f"https://example.org/{category}", + } + else: + humanitarian[category] = { + origin: { + "target": targets[f"{category}:{origin}"], + "source": f"https://example.org/{category}/{origin}", + } + for origin in origins + } + return { + "seed_from_build_config": True, + "time_period_from_build_config": True, + "undocumented_workers": { + "target": workers, + "source": "https://example.org/workers", + }, + "undocumented_students": { + "target": students, + "source": "https://example.org/students", + }, + "undocumented_population_anchor": { + "value": anchor, + "source": "https://example.org/population", + }, + "humanitarian_status_stocks": humanitarian, + } + + +@dataclass(frozen=True) +class _StageCase: + person: pd.DataFrame + workers: float + students: float + anchor: float + targets: Mapping[str, float] + seed: int + time_period: int + + @property + def parameters(self) -> dict[str, object]: + return _manifest_parameters( + workers=self.workers, + students=self.students, + anchor=self.anchor, + targets=self.targets, + ) + + @property + def controls(self) -> immigration.ImmigrationControls: + return immigration._controls_from_parameters(self.parameters) + + +def _indicator_mask(person: pd.DataFrame) -> np.ndarray: + """Rows carrying a documented indicator (see ``_INDICATORS``).""" + + indicator = np.isin(person["PEINUSYR"].to_numpy(), range(1, 8)) + for name, value in _INDICATORS: + indicator |= person[name].to_numpy() == value + return indicator + + +def _scopes(person: pd.DataFrame) -> dict[str, np.ndarray]: + """The two controlled margins as step 3 defines them: workers "measured + with ASEC ``A_LFSR`` at age 16+" and students.""" + + return { + "workers": (person["A_AGE"].to_numpy() >= 16) + & np.isin(person["A_LFSR"].to_numpy(), (1, 2, 3, 4)), + "students": person["A_HSCOL"].to_numpy() == 2, + } + + +def _residual_masses(person: pd.DataFrame) -> dict[str, float]: + """Each margin's pre-stage residual mass (indicator-free non-citizens). + + The stage's pre-spill Pew count in a margin is at least this: residual + rows either stay residual or, if a parole/TPS draw takes them, keep a + Pew-included mark. A control below it therefore makes the spill work. + """ + + residual = (person["PRCITSHP"].to_numpy() == 5) & ~_indicator_mask(person) + weights = person["person_weight"].to_numpy(dtype=np.float64) + return { + name: float(weights[residual & scope].sum()) + for name, scope in _scopes(person).items() + } + + +def _pool_masses(person: pd.DataFrame, time_period: int) -> dict[str, float]: + """Each draw's weighted candidate pool before any draw runs. + + Scaling targets to these pools exercises both sides of every draw (a pool + that suffices and one that runs short) instead of mostly saturating. + """ + + noncitizen = person["PRCITSHP"].to_numpy() == 5 + codes = np.where(noncitizen, np.where(_indicator_mask(person), 3, 0), 1) + profile = immigration._source_aware_immigration_profile( + person, time_period=time_period + ) + weights = person["person_weight"].to_numpy(dtype=np.float64) + return { + template.label: float( + weights[ + immigration._humanitarian_draw_candidates( + template, ssn_codes=codes, profile=profile + ) + ].sum() + ) + for template in _TEMPLATE_DRAWS + } + + +@st.composite +def _target(draw, pool: float, scale: float) -> float: + """Zero, a share of the draw's own pool, or occasionally a pool-blind stock.""" + + if draw(st.integers(0, 9)) == 0: + return draw(st.floats(1e-3, 0.1)) * scale + return draw(st.one_of(st.just(0.0), st.floats(0.01, 1.5))) * pool + + +@st.composite +def _control(draw, mass: float, scale: float) -> float: + """A worker/student control as a share of the margin's residual mass: + below it (the spill must act), near it, or above it (it may not).""" + + share = draw( + st.one_of(st.floats(0.01, 0.99), st.floats(0.99, 1.01), st.floats(1.01, 2.0)) + ) + return share * (mass if mass > 0 else scale) + + +@st.composite +def _stage_cases( + draw, + *, + integer_weights: bool = False, + min_size: int = 6, + max_size: int = 32, + time_periods: tuple[int, ...] = _PERIODS, +) -> _StageCase: + weight = st.integers(0, 2_000).map(float) if integer_weights else _WEIGHT + n = draw(st.integers(min_size, max_size), label="rows") + rows = draw(st.lists(_asec_row(weight), min_size=n, max_size=n)) + person = _asec_table(rows) + time_period = draw(st.sampled_from(time_periods), label="time period") + scale = max(1.0, float(person["person_weight"].sum())) + pools = _pool_masses(person, time_period) + residual = _residual_masses(person) + return _StageCase( + person=person, + workers=draw(_control(residual["workers"], scale), label="workers"), + students=draw(_control(residual["students"], scale), label="students"), + anchor=draw(st.floats(0.1, 2.0)) * scale, + targets={label: draw(_target(pools[label], scale)) for label in _DRAW_LABELS}, + seed=draw(st.integers(0, 2**31 - 1)), + time_period=time_period, + ) + + +def _derive( + person: pd.DataFrame, + parameters: Mapping[str, object], + *, + seed: int, + time_period: int, +) -> pd.DataFrame: + operation = SourceOperationSpec.from_mapping( + {"kind": "derive_immigration_status", **parameters} + ) + context = SourceRuntimeContext( + config=SourceRuntimeConfig(seed=seed, target_year=time_period), + tables={}, + ) + return immigration.derive_us_immigration_status_from_manifest( + person, operation, context + ) + + +def _run(case: _StageCase) -> pd.DataFrame: + return _derive( + case.person, + case.parameters, + seed=case.seed, + time_period=case.time_period, + ) + + +@dataclass(frozen=True) +class _Facts: + """Evidence recomputed from the raw columns and the documented rules.""" + + weights: np.ndarray + noncitizen: np.ndarray + ssn: np.ndarray + status: np.ndarray + humanitarian: np.ndarray + cuban_haitian_born: np.ndarray + cuban_haitian_class: np.ndarray + daca_cohort: np.ndarray + indicator: np.ndarray + worker: np.ndarray + student: np.ndarray + + @property + def retained(self) -> np.ndarray: + """The spill's retained EAD cohort: DACA plus the Cuban/Haitian class + outside it (module docstring, step 3: "Residual Cuban/Haitian rows + that receive an EAD also stay in that universe").""" + + return self.daca_cohort | (self.cuban_haitian_class & ~self.daca_cohort) + + +def _facts(person: pd.DataFrame, output: pd.DataFrame, time_period: int) -> _Facts: + def column(name: str) -> np.ndarray: + return person[name].to_numpy() + + arrival_code = column("PEINUSYR").astype(np.int64) + arrival_year = np.asarray( + [ + immigration._ARRIVAL_YEAR_MIDPOINTS.get(int(code), time_period) + for code in arrival_code + ], + dtype=np.int64, + ) + age = column("A_AGE").astype(np.int64) + age_at_entry = np.maximum(0, age - (time_period - arrival_year)) + # "DACA applies the statutory cohort test (arrived by 2007 before age 16, + # aged 15+)" (module docstring, step 6). + daca_cohort = (arrival_year <= 2007) & (age_at_entry < 16) & (age >= 15) + cuban_haitian_born = np.isin( + column("PENATVTY"), immigration._CUBAN_HAITIAN_BIRTH_CODES + ) + status = output["immigration_status_str"].astype(str).to_numpy() + scopes = _scopes(person) + return _Facts( + weights=person["person_weight"].to_numpy(dtype=np.float64), + noncitizen=column("PRCITSHP") == 5, + ssn=output["ssn_card_type"].astype(str).to_numpy(), + status=status, + humanitarian=np.isin(status, list(_HUMANITARIAN_STATUSES)), + cuban_haitian_born=cuban_haitian_born, + # "the nationality-plus-arrival class" (module docstring, step 6). + cuban_haitian_class=cuban_haitian_born + & (arrival_year >= immigration._CUBAN_HAITIAN_ARRIVAL_CUTOFF), + daca_cohort=daca_cohort, + indicator=_indicator_mask(person), + worker=scopes["workers"], + student=scopes["students"], + ) + + +def _frame(person: pd.DataFrame, weights: np.ndarray) -> Frame: + """One person per household, weighted at the household.""" + + person = person.drop(columns=["person_weight"], errors="ignore").copy() + ids = np.arange(1, len(person) + 1, dtype=np.int64) + person = person.reset_index(drop=True) + for entity in US_SCHEMA.group_entities: + person[US_SCHEMA.membership_column(entity)] = ids + tables = { + "person": person, + **{ + entity: pd.DataFrame({US_SCHEMA.id_column(entity): ids}) + for entity in US_SCHEMA.group_entities + }, + } + return Frame( + tables, + US_SCHEMA, + { + "household": Weights( + np.asarray(weights, dtype=np.float64), WeightKind.DESIGN + ) + }, + ) + + +def _codes(ssn: np.ndarray) -> np.ndarray: + """Engine SSN names to the stage's integer codes, via the module's map.""" + + return immigration._ssn_name_codes(pd.DataFrame({"ssn_card_type": ssn})) + + +def _pre_draw_codes(ssn: np.ndarray) -> np.ndarray: + """SSN codes as the humanitarian draws saw them. + + Indicators set code 3 and nothing later changes it; the draws and the + spill only move code 0 to code 2. So every final EAD row was residual when + the draws ran. + """ + + return np.select( + [ssn == "CITIZEN", ssn == "OTHER_NON_CITIZEN"], [1, 3], default=0 + ).astype(np.int64) + + +@_STAGE +@given(_stage_cases()) +def test_stage_pairs_undocumented_with_no_ssn_and_citizens_with_citizenship( + case, +) -> None: + """Module docstring: "``UNDOCUMENTED`` enum remains exactly paired with + ``ssn_card_type=NONE``"; "Citizenship is measured, not imputed. + ``PRCITSHP`` in {1..4} → citizens"; "``CONDITIONAL_ENTRANT`` is + deliberately not emitted"; and "The stage writes two PolicyEngine-US + person input columns" (the source columns pass through unchanged).""" + + output = _run(case) + assert list(output.columns) == [ + *case.person.columns, + *immigration.US_IMMIGRATION_OUTPUT_COLUMNS, + ] + pd.testing.assert_frame_equal(output[list(case.person.columns)], case.person) + ssn = output["ssn_card_type"].to_numpy() + status = output["immigration_status_str"].to_numpy() + assert set(ssn) <= set(immigration.SSN_CARD_TYPE_VALUES) + assert set(status) <= set(immigration.IMMIGRATION_STATUS_VALUES) + assert "CONDITIONAL_ENTRANT" not in set(status) + np.testing.assert_array_equal(ssn == "NONE", status == "UNDOCUMENTED") + citizen = np.isin(case.person["PRCITSHP"].to_numpy(), (1, 2, 3, 4)) + np.testing.assert_array_equal(ssn == "CITIZEN", citizen) + np.testing.assert_array_equal(status == "CITIZEN", citizen) + if (ssn == "NONE").any() and (ssn == "NON_CITIZEN_VALID_EAD").any(): + event("output carries both NONE and EAD rows") + + +@_STAGE +@given(_stage_cases()) +def test_stage_status_is_the_documented_function_of_ssn_marks_and_evidence( + case, +) -> None: + """Reference implementation of step 6 and ``_derive_immigration_status``: + citizens are ``CITIZEN``, ``NONE`` is ``UNDOCUMENTED``, a humanitarian + mark wins, then "``DACA`` applies the statutory cohort test ... to EAD + holders; ``CUBAN_HAITIAN_ENTRANT`` applies the nationality-plus-arrival + class to documented non-citizens; every other documented non-citizen stays + ``LEGAL_PERMANENT_RESIDENT``". DACA outranks the Cuban/Haitian class only + for EAD holders (``_special_status_masks``: "a qualifying EAD holder in + both cohorts retains DACA"), so an indicator-documented Cuba-born member + of the DACA cohort is a Cuban/Haitian entrant. A mark never hides one of + those tags: "Humanitarian draws exclude Cuba/Haiti-born and the DACA + cohort, so the marks are disjoint from both tags".""" + + output = _run(case) + facts = _facts(case.person, output, case.time_period) + documented = np.isin(facts.ssn, ("NON_CITIZEN_VALID_EAD", "OTHER_NON_CITIZEN")) + daca = (facts.ssn == "NON_CITIZEN_VALID_EAD") & facts.daca_cohort + cuban_haitian = documented & facts.cuban_haitian_class & ~daca + if daca.any(): + event("output carries DACA rows") + if cuban_haitian.any(): + event("output carries Cuban/Haitian entrants") + expected = np.select( + [ + facts.ssn == "CITIZEN", + facts.ssn == "NONE", + facts.humanitarian, + daca, + cuban_haitian, + ], + ["CITIZEN", "UNDOCUMENTED", facts.status, "DACA", "CUBAN_HAITIAN_ENTRANT"], + default="LEGAL_PERMANENT_RESIDENT", + ) + np.testing.assert_array_equal(facts.status, expected) + assert not (facts.humanitarian & (daca | cuban_haitian)).any() + + +@_STAGE +@given(_stage_cases()) +def test_indicator_pool_is_exactly_the_noncitizens_with_a_documented_indicator( + case, +) -> None: + """Module docstring, step 2: "Non-citizens with any indicator of authorized + status — pre-1982 IRCA-cohort arrival, ... — move to + ``OTHER_NON_CITIZEN``", and nothing later moves them (the draws and the + spill only rewrite code 0). No other row lands there. (The docstring also + lists naturalization-eligibility, which the code cannot fire for a + non-citizen; see ``_NEUTRAL_INDICATORS``.)""" + + output = _run(case) + facts = _facts(case.person, output, case.time_period) + np.testing.assert_array_equal( + facts.ssn == "OTHER_NON_CITIZEN", facts.noncitizen & facts.indicator + ) + + +@_STAGE +@given(_stage_cases()) +def test_humanitarian_labels_respect_exclusions_pools_and_draw_masks(case) -> None: + """Module docstring, step 5: "Cuba/Haiti-born persons are excluded from + every humanitarian draw ..., as is the DACA statutory cohort"; draws are + "mutually exclusive"; "Draws from the residual pool (TPS and parole only) + flip ``ssn_card_type`` ``NONE`` to ``NON_CITIZEN_VALID_EAD``"; + ``_humanitarian_draw_candidates``: "``REFUGEE``/``ASYLEE``/ + ``DEPORTATION_WITHHELD`` draw from the indicator-documented pool only". + The public ``us_immigration_humanitarian_draw_mask`` partitions the + humanitarian rows by draw, and an explicit zero target emits nothing + (``HumanitarianDraw``: "Zero ... means the category is explicitly not + imputed").""" + + output = _run(case) + facts = _facts(case.person, output, case.time_period) + if facts.humanitarian.any(): + event("output carries humanitarian rows") + assert not (facts.humanitarian & ~facts.noncitizen).any() + assert not (facts.humanitarian & facts.cuban_haitian_born).any() + assert not (facts.humanitarian & facts.daca_cohort).any() + indicator_pool = np.isin(facts.status, list(_INDICATOR_POOL_STATUSES)) + assert (facts.ssn[indicator_pool] == "OTHER_NON_CITIZEN").all() + temporary = facts.humanitarian & ~indicator_pool + if (temporary & (facts.ssn == "NON_CITIZEN_VALID_EAD")).any(): + event("a residual row drew parole/TPS") + assert np.isin( + facts.ssn[temporary], ("OTHER_NON_CITIZEN", "NON_CITIZEN_VALID_EAD") + ).all() + + frame = _frame(output, np.ones(len(output))) + covered = np.zeros(len(output), dtype=int) + for draw in case.controls.humanitarian: + mask = immigration.us_immigration_humanitarian_draw_mask( + frame, draw, time_period=case.time_period + ) + if draw.target <= 0: + assert not mask.any(), draw.label + covered += mask + assert (covered <= 1).all() + np.testing.assert_array_equal(covered == 1, facts.humanitarian) + + +@_STAGE +@given(_stage_cases()) +def test_humanitarian_rows_sit_inside_the_asec_windows_the_comments_state( + case, +) -> None: + """Independent of ``_humanitarian_draw_candidates``: every emitted parole, + TPS and asylee row satisfies the ASEC origin/arrival windows transcribed + above from the module's comments (``_PAROLE_DOCUMENTED_WINDOWS``, + ``_TPS_DOCUMENTED_MAX_ARRIVAL_CODE``, ``_TPS_DOCUMENTED_EXCLUDED_BIRTHS``, + ``_ASYLEE_DOCUMENTED_MIN_ARRIVAL_CODE``). Refugee and withholding windows + have no numeric statement in a comment and are not checked here.""" + + output = _run(case) + status = output["immigration_status_str"].to_numpy() + birth = case.person["PENATVTY"].to_numpy() + arrival = case.person["PEINUSYR"].to_numpy() + + parole = status == "PAROLED_ONE_YEAR" + assert set(birth[parole].tolist()) <= set(_PAROLE_DOCUMENTED_WINDOWS) + for origin, window in _PAROLE_DOCUMENTED_WINDOWS.items(): + rows = parole & (birth == origin) + if rows.any(): + event("parole rows emitted") + if ((birth == origin) & ~np.isin(arrival, list(window))).any(): + event("a parole-origin row sits outside its window") + assert np.isin(arrival[rows], list(window)).all(), origin + + tps = status == "TPS" + assert not np.isin(birth[tps], list(_TPS_DOCUMENTED_EXCLUDED_BIRTHS)).any() + assert (arrival[tps] >= 1).all() + for origin, latest in _TPS_DOCUMENTED_MAX_ARRIVAL_CODE.items(): + rows = tps & (birth == origin) + if rows.any(): + event("legacy-designation TPS rows emitted") + if ((birth == origin) & (arrival > latest)).any(): + event("a legacy TPS-origin row arrived after its cutoff") + assert (arrival[rows] <= latest).all(), origin + + asylee = status == "ASYLEE" + if asylee.any(): + event("asylee rows emitted") + assert (arrival[asylee] >= _ASYLEE_DOCUMENTED_MIN_ARRIVAL_CODE).all() + + +@_STAGE +@given(_stage_cases()) +def test_each_humanitarian_draw_reaches_its_target_or_exhausts_its_pool( + case, +) -> None: + """Step 5 draws each status "to a manifest-cited weighted stock target", + "in that order", and "a person two cohorts could claim takes the earlier + status" (``_assign_humanitarian_statuses``: "a person marked by an earlier + draw is out of every later candidate pool"). The band constant adds: "The + draw forces the target when candidates suffice". Replaying the candidate + pools in draw order, each draw's rows reach its target and overshoot it by + less than the last-drawn row, or take the whole pool when it is short; a + zero target takes nothing; and the draws account for every humanitarian + row. The pools come from the module's ``_humanitarian_draw_candidates``, + so this checks selection against target, not the windows themselves.""" + + output = _run(case) + facts = _facts(case.person, output, case.time_period) + tolerance = _tolerance(facts.weights) + profile = immigration._source_aware_immigration_profile( + case.person, time_period=case.time_period + ) + pre_draw = _pre_draw_codes(facts.ssn) + marked = np.zeros(len(output), dtype=bool) + for draw in case.controls.humanitarian: + pool = ( + immigration._humanitarian_draw_candidates( + draw, ssn_codes=pre_draw, profile=profile + ) + & ~marked + ) + chosen = pool & (facts.status == draw.status) + weight = float(facts.weights[chosen].sum()) + available = float(facts.weights[pool].sum()) + if draw.target <= 0: + assert not chosen.any(), draw.label + elif available >= draw.target + tolerance: + event("a draw's pool sufficed") + assert weight >= draw.target - tolerance, draw.label + draws = immigration._stable_person_draws( + case.person, seed=case.seed, salt=draw.salt + ) + last = chosen & (draws == draws[chosen].max()) + assert ( + weight - float(facts.weights[last].sum()) < draw.target + tolerance + ), draw.label + elif available < draw.target - tolerance: + event("a draw's pool was short") + np.testing.assert_array_equal(chosen, pool, err_msg=draw.label) + marked |= chosen + np.testing.assert_array_equal(marked, facts.humanitarian) + + +@_STAGE +@given(_stage_cases(), st.data()) +def test_humanitarian_draws_are_sequential_and_precede_the_spill(case, data) -> None: + """``HUMANITARIAN_STATUS_CATEGORIES``: "Draws are sequential and mutually + exclusive"; ``_assign_humanitarian_statuses`` "Runs after the legal-status + indicators and before the EAD worker/student spill". So retargeting every + category from some point on, and changing the worker and student controls, + leaves the rows of every earlier category unchanged.""" + + categories = immigration.HUMANITARIAN_STATUS_CATEGORIES + cut = data.draw(st.integers(0, len(categories)), label="first retargeted") + scale = max(1.0, float(case.person["person_weight"].sum())) + pools = _pool_masses(case.person, case.time_period) + residual = _residual_masses(case.person) + later = set(categories[cut:]) + targets = { + label: ( + data.draw(_target(pools[label], scale), label=label) + if label.split(":", 1)[0] in later + else target + ) + for label, target in case.targets.items() + } + changed = replace( + case, + workers=data.draw(_control(residual["workers"], scale), label="workers"), + students=data.draw(_control(residual["students"], scale), label="students"), + targets=targets, + ) + first = _run(case)["immigration_status_str"].to_numpy() + second = _run(changed)["immigration_status_str"].to_numpy() + for category in categories[:cut]: + status = immigration._HUMANITARIAN_STATUS_BY_CATEGORY[category] + if (first == status).any(): + event("an earlier category emitted rows") + np.testing.assert_array_equal( + first == status, second == status, err_msg=category + ) + + +@_STAGE +@given(_stage_cases(integer_weights=True), st.data()) +def test_stage_is_bit_reproducible_and_row_order_equivariant(case, data) -> None: + """Module docstring: "reruns are bit-reproducible without global RNG + state". Row-order equivariance is deduced: draws are "keyed by the + person's stable source identity" and every selection is a draw threshold + (integer weights keep the weighted sums order-independent).""" + + output = _run(case) + pd.testing.assert_frame_equal(_run(case), output) + order = list(data.draw(st.permutations(range(len(case.person))))) + shuffled = replace(case, person=case.person.iloc[order].reset_index(drop=True)) + pd.testing.assert_frame_equal( + _run(shuffled), + output.iloc[order].reset_index(drop=True), + ) + + +@_STAGE +@given(_stage_cases(max_size=16), st.data()) +def test_support_clones_share_their_source_persons_labels(case, data) -> None: + """Module docstring: draws keyed by + ``source_year``/``source_household_id``/``source_person_id`` mean + "support-channel clones of one source person always receive the same + status"; ``_select_weight_to_target``: "a clone pair is never split". + Clones copy their source person's survey record but may carry any + weight.""" + + source = case.person.assign( + source_year=2024, + source_household_id=case.person["person_id"] + 100, + source_person_id=case.person["person_id"], + ) + copies = data.draw( + st.lists(st.integers(0, len(source) - 1), min_size=1, max_size=16), + label="cloned rows", + ) + clones = source.iloc[copies].copy() + clones["person_weight"] = data.draw( + st.lists(_WEIGHT, min_size=len(copies), max_size=len(copies)), + label="clone weights", + ) + person = pd.concat([source, clones], ignore_index=True) + person["person_id"] = np.arange(1, len(person) + 1) + output = _derive( + person, case.parameters, seed=case.seed, time_period=case.time_period + ) + grouped = output.groupby("source_person_id")[ + list(immigration.US_IMMIGRATION_OUTPUT_COLUMNS) + ].nunique() + assert (grouped == 1).all().all() + + +@_FRAME +@given(_stage_cases(max_size=16), st.floats(0.25, 4.0), st.data()) +def test_frame_entry_point_matches_the_manifest_handler_and_is_idempotent( + case, weight_scale, data +) -> None: + """``with_us_immigration_inputs`` runs "the ``immigration_status`` + manifest stage over a US frame" and returns "A new frame whose person + table carries ``ssn_card_type`` and ``immigration_status_str``"; + ``person_weight_scale`` is a "Stage-only multiplier for person design + weights"; and "Existing output columns are preserved, making the + transform idempotent". Differential: with non-monotone person ids, the + frame entry point labels every person exactly as the packaged manifest + handler does on design weight x scale, leaves the frame's weights and + every other column and table unchanged, and a second call (any seed or + period) returns the frame as it is.""" + + ids = data.draw(st.permutations(range(1, len(case.person) + 1)), label="ids") + person = case.person.assign(person_id=np.asarray(ids, dtype=np.int64)) + # A Frame refuses all-zero weights, which the handler alone accepts. + weights = (person["person_weight"].to_numpy(dtype=np.float64) + 1.0) * 1e3 + frame = _frame(person, weights) + result = immigration.with_us_immigration_inputs( + frame, + seed=case.seed, + time_period=case.time_period, + person_weight_scale=weight_scale, + ) + packaged = [ + operation + for operation in immigration.us_immigration_stage_spec().operations + if operation.kind == "derive_immigration_status" + ] + assert len(packaged) == 1 + expected = _derive( + frame.table("person").assign(person_weight=weights * weight_scale), + packaged[0].parameters, + seed=case.seed, + time_period=case.time_period, + ) + labelled = result.table("person") + columns = list(immigration.US_IMMIGRATION_OUTPUT_COLUMNS) + for column in columns: + np.testing.assert_array_equal( + labelled[column].to_numpy(), expected[column].to_numpy(), err_msg=column + ) + pd.testing.assert_frame_equal(labelled.drop(columns=columns), frame.table("person")) + for entity in US_SCHEMA.group_entities: + pd.testing.assert_frame_equal(result.table(entity), frame.table(entity)) + np.testing.assert_array_equal( + np.asarray(result.resolve_weights("household").values), + np.asarray(frame.resolve_weights("household").values), + ) + again = immigration.with_us_immigration_inputs( + result, + seed=case.seed + 1, + time_period=case.time_period + 1, + person_weight_scale=2.0, + ) + pd.testing.assert_frame_equal(again.table("person"), labelled) + + +#: Composition-gate failures that judge plausibility of the weighted totals +#: (non-citizen share, Pew anchor band, humanitarian bands, a small table +#: that happens to be all one value). Every other failure is structural. +_PLAUSIBILITY_FAILURES = ( + "constant value", + "non-citizen weighted share", + "emergent Pew-defined unauthorized population", + "x the cited stock target", +) + + +def _structural_failures(gate) -> list[str]: + return [ + failure + for failure in gate.failures + if not any(fragment in failure for fragment in _PLAUSIBILITY_FAILURES) + ] + + +def _binary_weights(n: int) -> np.ndarray: + """Distinct powers of two: a weighted total then names its rows exactly.""" + + return 2.0 ** np.arange(n, dtype=np.float64) + + +def _rows_from_binary_total(total: float, n: int) -> np.ndarray: + bits = int(round(total)) + assert float(bits) == total + return np.asarray([(bits >> index) & 1 for index in range(n)], dtype=bool) + + +def _person_row(**overrides) -> dict: + """One ``_asec_row``-shaped record: a US-born citizen adult by default.""" + + row = { + "PRCITSHP": 1, + "PENATVTY": 57, + "PEINUSYR": 0, + "A_AGE": 30, + "A_MARITL": 7, + "A_SPOUSE": 0, + "A_HSCOL": 0, + "A_LFSR": 7, + "indicator": None, + "person_weight": 1.0, + } + return row | overrides + + +#: A worker who arrived in 2007 (PEINUSYR 20) and is 33 entered at 16 when +#: measured in 2024 but at 15 in 2025, so the stage at 2025 emits DACA once +#: the worker spill gives the row an EAD. A gate that re-derived the cohort +#: at 2024 (the defect fixed in 5a89fed1e) flags that row. +_PERIOD_SENSITIVE_DACA_CASE = _StageCase( + person=_asec_table( + [ + _person_row(), + _person_row(PRCITSHP=5, PENATVTY=303, PEINUSYR=20, A_AGE=33, A_LFSR=1), + ] + ), + workers=0.5, + students=1.0, + anchor=1.0, + targets=dict.fromkeys(_DRAW_LABELS, 0.0), + seed=0, + time_period=2025, +) + + +@_STAGE +@given(_stage_cases()) +@example(_PERIOD_SENSITIVE_DACA_CASE) +def test_stage_output_satisfies_the_composition_gates_structural_contract( + case, +) -> None: + """Differential against ``us_immigration_composition_gate``, which fails + "when a value falls outside the engine enum domain, when the two columns + disagree about citizenship or undocumented status", on source-evidence and + Cuban/Haitian/DACA cohort violations, on humanitarian rows outside every + draw's cohort, and when "an explicit zero target" emits. Its + ``time_period`` "must be the period the stage ran for", so the gate runs + at the stage's period (several periods are drawn). The stage and the gate + encode those contracts separately; stage output must pass all of them + (the weighted plausibility bands are not structural).""" + + output = _run(case) + event(f"time period {case.time_period}") + gate = immigration.us_immigration_composition_gate( + _frame(output, _binary_weights(len(output))), + time_period=case.time_period, + controls=case.controls, + ) + assert _structural_failures(gate) == [] + assert gate.details["evidence_derived_status_compatibility"]["invalid_rows"] == 0 + + +@_STAGE +@given(_stage_cases()) +def test_spill_projection_selects_exactly_the_gates_pew_universe(case) -> None: + """Differential: ``_pew_unauthorized_projection_mask`` "Project[s] final + membership in Pew's broad unauthorized universe" that the composition + gate then measures (``_PEW_UNAUTHORIZED_STATUS_VALUES`` plus residual + Cuban/Haitian EAD rows). On final stage output the two definitions select + the same rows; power-of-two gate weights make the gate's total name its + rows.""" + + output = _run(case) + facts = _facts(case.person, output, case.time_period) + gate = immigration.us_immigration_composition_gate( + _frame(output, _binary_weights(len(output))), + time_period=case.time_period, + controls=case.controls, + ) + gate_rows = _rows_from_binary_total( + gate.details["pew_unauthorized_population"], len(output) + ) + projection = immigration._pew_unauthorized_projection_mask( + ssn_codes=_codes(facts.ssn), + humanitarian_marks=np.where(facts.humanitarian, facts.status, ""), + retained_ead_cohort=facts.retained, + ) + if (projection & (facts.ssn != "NONE")).any(): + event("Pew universe includes non-NONE rows") + np.testing.assert_array_equal(projection, gate_rows) + + +@_STAGE +@given(_stage_cases()) +def test_pew_margins_close_whenever_the_residual_pool_can_close_them(case) -> None: + """Module docstring, step 3: workers and students "spill to + ``NON_CITIZEN_VALID_EAD`` in deterministic seeded order until the broad + reported universes match their controls"; the worker spill "runs last and + is therefore authoritative"; and "a count already at or below its control + spills nothing". Each margin ends at or below its control unless every + removable (residual, non-retained) row in it was spilled, and no row is + spilled when both margins start below their controls.""" + + output = _run(case) + facts = _facts(case.person, output, case.time_period) + tolerance = _tolerance(facts.weights) + pew = immigration._pew_unauthorized_projection_mask( + ssn_codes=_codes(facts.ssn), + humanitarian_marks=np.where(facts.humanitarian, facts.status, ""), + retained_ead_cohort=facts.retained, + ) + residual = ( + facts.noncitizen + & ~facts.humanitarian + & np.isin(facts.ssn, ("NONE", "NON_CITIZEN_VALID_EAD")) + ) + spilled = residual & (facts.ssn == "NON_CITIZEN_VALID_EAD") + initial: dict[str, float] = {} + for name, scope, target in ( + ("students", facts.student, case.students), + ("workers", facts.worker, case.workers), + ): + final = float(facts.weights[scope & pew].sum()) + removable = scope & residual & ~facts.retained + if (scope & spilled).any(): + event(f"the {name} margin spilled rows") + if final <= target + tolerance and (removable & ~spilled).any(): + event(f"the {name} margin closed with residual rows left") + assert final <= target + tolerance or not (removable & ~spilled).any(), name + initial[name] = final + float( + facts.weights[scope & spilled & ~facts.retained].sum() + ) + if ( + initial["students"] < case.students - tolerance + and initial["workers"] < case.workers - tolerance + ): + assert not spilled.any() + + +# --------------------------------------------------------------------------- +# ACS transfer reconciliation +# --------------------------------------------------------------------------- + +_ACS_ROW = st.fixed_dictionaries( + { + "CIT": st.sampled_from((1, 4, 5, 5, 5, 5)), + # Biased toward the humanitarian origins so recipients can fill draws. + "POBP": st.one_of( + st.sampled_from(_BIRTH_CODES), st.sampled_from(_HUMANITARIAN_ORIGIN_CODES) + ), + "YOEP": st.one_of(st.integers(1975, 2024), st.integers(2016, 2024)), + "age": st.integers(0, 85), + "ssn_card_type": st.sampled_from(immigration.SSN_CARD_TYPE_VALUES), + # The joint-QRF baseline never emits an evidence-constrained label. + "immigration_status_str": st.sampled_from( + ("CITIZEN", "LEGAL_PERMANENT_RESIDENT", "UNDOCUMENTED") + ), + "weight": st.one_of( + st.integers(1, 5_000).map(float), + st.floats(0.5, 5e5, allow_nan=False, allow_subnormal=False), + ), + } +) + + +@dataclass(frozen=True) +class _TransferCase: + person: pd.DataFrame + weights: np.ndarray + mutable: np.ndarray + controls: immigration.ImmigrationControls + seed: int + + +@st.composite +def _transfer_cases(draw) -> _TransferCase: + """ASEC rows labeled by the stage (immutable) stacked with ACS recipients + carrying unconstrained donor labels (mutable), as in the ACS transfer. + + Reconciliation targets are feasible by construction, so the documented + refusal ("candidate exhaustion is a hard error") never applies and any + error is a finding. Each draw's target is at least half its immutable ASEC + mass (so a draw with immutable rows never has an explicit-zero target) + plus a share of the recipient mass that this draw, and no earlier draw, + can claim; ``selected_once`` can only remove rows an earlier draw claims. + Some recipients are ACS twins of ASEC rows (same origin, age and the + ASEC bin's midpoint year), so a draw often has both an immutable + contribution and recipients to select. + """ + + stage = draw(_stage_cases(min_size=4, max_size=16, time_periods=(_TIME_PERIOD,))) + asec = _run(stage).assign(CIT=np.nan, POBP=np.nan, YOEP=np.nan, age=np.nan) + acs_rows = draw(st.lists(_ACS_ROW, min_size=2, max_size=12)) + for index in draw( + st.lists(st.integers(0, len(asec) - 1), max_size=8), label="ACS twins" + ): + source = asec.iloc[index] + noncitizen = int(source["PRCITSHP"]) == 5 + acs_rows.append( + { + "CIT": 5 if noncitizen else 1, + "POBP": int(source["PENATVTY"]), + "YOEP": immigration._ARRIVAL_YEAR_MIDPOINTS.get( + int(source["PEINUSYR"]), _TIME_PERIOD + ), + "age": int(source["A_AGE"]), + "ssn_card_type": draw( + st.sampled_from( + ("OTHER_NON_CITIZEN", "NONE", "NON_CITIZEN_VALID_EAD") + if noncitizen + else ("CITIZEN",) + ) + ), + "immigration_status_str": draw( + st.sampled_from(("LEGAL_PERMANENT_RESIDENT", "UNDOCUMENTED")) + if noncitizen + else st.just("CITIZEN") + ), + "weight": draw(st.integers(1, 5_000).map(float)), + } + ) + acs = pd.DataFrame(acs_rows) + acs = acs.rename(columns={"weight": "person_weight"}).assign( + PRCITSHP=np.nan, PENATVTY=np.nan, PEINUSYR=np.nan, A_AGE=np.nan + ) + person = pd.concat([asec, acs], ignore_index=True) + person["person_id"] = np.arange(1, len(person) + 1) + weights = person.pop("person_weight").to_numpy(dtype=np.float64) + mutable = np.arange(len(person)) >= len(asec) + + frame = _frame(person, weights) + profile = immigration._source_aware_immigration_profile( + person, time_period=_TIME_PERIOD + ) + # Pair repair turns a recipient non-citizen's CITIZEN code into + # OTHER_NON_CITIZEN and leaves every other non-citizen code alone; no + # citizen is ever a humanitarian candidate. + codes = immigration._ssn_name_codes(person) + codes = np.where(~profile.is_citizen & (codes == 1), 3, codes) + claimed = np.zeros(len(person), dtype=bool) + targets: dict[str, float] = {} + for control in stage.controls.humanitarian: + immutable_mass = float( + weights[ + immigration.us_immigration_humanitarian_draw_mask( + frame, control, time_period=_TIME_PERIOD + ) + & ~mutable + ].sum() + ) + candidate_codes = ( + np.where(codes == 2, 0, codes) + if control.category in {"paroled_one_year", "tps"} + else codes + ) + pool = mutable & immigration._humanitarian_draw_candidates( + control, ssn_codes=candidate_codes, profile=profile + ) + exclusive = float(weights[pool & ~claimed].sum()) + claimed |= pool + immutable_share = draw( + st.one_of(st.just(1.0), st.floats(0.5, 1.0)), label="immutable share" + ) + exclusive_share = draw( + st.one_of(st.just(0.0), st.just(1.0), st.floats(0.0, 1.0)), + label="exclusive share", + ) + targets[control.label] = ( + immutable_share * immutable_mass + exclusive_share * exclusive + ) + return _TransferCase( + person=person, + weights=weights, + mutable=mutable, + controls=immigration._controls_from_parameters( + _manifest_parameters( + workers=stage.workers, + students=stage.students, + anchor=stage.anchor, + targets=targets, + ) + ), + seed=draw(st.integers(0, 2**31 - 1)), + ) + + +def _reconcile(case: _TransferCase): + return immigration.reconcile_us_immigration_humanitarian_transfer( + case.person, + weights=case.weights, + mutable_rows=case.mutable, + seed=case.seed, + time_period=_TIME_PERIOD, + controls=case.controls, + ) + + +@_TRANSFER +@given(_transfer_cases()) +def test_reconciliation_output_satisfies_the_gates_structural_contract(case) -> None: + """Differential: ``reconcile_us_immigration_humanitarian_transfer``'s own + final checks restate the gate's citizenship/``NONE``⇔``UNDOCUMENTED`` + pairing and Cuban/Haitian/DACA cohort contracts; the independently coded + ``us_immigration_composition_gate`` must find no structural failure in + the reconciled frame. Reconciliation is deterministic. Immutable rows are + compared too, but the function already raises "changed immutable ASEC + rows" before returning otherwise, so that comparison confirms the guard + rather than adding evidence.""" + + result, receipt = _reconcile(case) + if any( + entry["selected_recipient_population"] > 0 + for entry in receipt["draws"].values() + ): + event("reconciliation selected recipients") + immutable = ~case.mutable + for column in immigration.US_IMMIGRATION_OUTPUT_COLUMNS: + np.testing.assert_array_equal( + result.loc[immutable, column].to_numpy(), + case.person.loc[immutable, column].to_numpy(), + ) + gate = immigration.us_immigration_composition_gate( + _frame(result, case.weights), + time_period=_TIME_PERIOD, + controls=case.controls, + ) + assert _structural_failures(gate) == [] + again_result, again_receipt = _reconcile(case) + pd.testing.assert_frame_equal(again_result, result) + assert again_receipt == receipt + + +@_TRANSFER +@given(_transfer_cases()) +def test_reconciliation_receipts_obey_their_accounting_identities(case) -> None: + """Each draw "receives the residual of its national target after its + compatible, immutable contribution, in manifest order", and the receipt + records it. + + Independent evidence: ``achieved_population`` is recomputed from the + post-selection emitted mask, separately from the immutable and selected + masks, and must equal immutable + selected; a zero target selects + nothing; the selection never exceeds the eligible pool. + + Receipt consistency only (not independent evidence): the function raises + ``RuntimeError`` before returning a receipt whose selection misses its + residual by more than its threshold tie mass or whose achieved error + exceeds the discrete bound, and it hard-codes + ``within_residual_discrete_weight_bound``. Those assertions confirm the + receipt records the quantities it checked.""" + + _, receipt = _reconcile(case) + tolerance = float(receipt["floating_tolerance"]) + assert receipt["selection_order"] == [ + draw.label for draw in case.controls.humanitarian + ] + assert receipt["mutable_rows"] == int(case.mutable.sum()) + for draw in case.controls.humanitarian: + entry = receipt["draws"][draw.label] + target = float(draw.target) + immutable = entry["immutable_population"] + selected = entry["selected_recipient_population"] + achieved = entry["achieved_population"] + tie = entry["selection_threshold_tie_population"] + if selected > 0 and immutable > 0: + event("a draw combined immutable and selected rows") + if immutable > target: + event("immutable rows overshoot a target") + # Independent identities. + assert achieved == pytest.approx(immutable + selected, abs=tolerance) + assert selected <= entry["eligible_recipient_population"] + tolerance + if target <= 0: + assert selected == 0.0 + # Receipt consistency with the function's own guards. + assert entry["target"] == target + assert entry["residual_target"] == pytest.approx( + max(0.0, target - immutable), abs=tolerance + ) + assert abs(selected - entry["residual_target"]) <= tie + tolerance + assert abs(achieved - target) <= ( + max(0.0, immutable - target) + tie + tolerance + ) + assert entry["within_residual_discrete_weight_bound"] is True + assert entry["within_discrete_weight_bound"] is True + + +@_TRANSFER +@given(_transfer_cases()) +def test_selection_replay_reproduces_the_reconciled_mutable_rows(case) -> None: + """Differential: ``us_immigration_humanitarian_transfer_selection_masks`` + promises to "Replay the exact deterministic mutable-row selection for + every draw" from the post-transfer frame. For every draw, its replayed + mask equals the reconciled mutable rows that + ``us_immigration_humanitarian_draw_mask`` reports as emitted (the + comparison ``stacked_spine`` makes at release).""" + + result, _ = _reconcile(case) + frame = _frame(result, case.weights) + replay = immigration.us_immigration_humanitarian_transfer_selection_masks( + frame, + mutable_rows=case.mutable, + seed=case.seed, + time_period=_TIME_PERIOD, + controls=case.controls, + ) + assert list(replay) == [draw.label for draw in case.controls.humanitarian] + for draw in case.controls.humanitarian: + emitted = immigration.us_immigration_humanitarian_draw_mask( + frame, draw, time_period=_TIME_PERIOD + ) + if (emitted & case.mutable).any(): + event("a replayed draw selected recipients") + np.testing.assert_array_equal( + replay[draw.label], emitted & case.mutable, err_msg=draw.label + ) From 3468180ab0d10015a2e0d55ea26e81a16c8670dc Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 29 Sep 2026 20:51:55 -0400 Subject: [PATCH 39/49] Seal the legacy fixture's execution rows under their schema-16 contracts Astra round 2 (second half): the positive schema-9 tests kept today's declared-input surfaces, so nothing proved a real schema-9 pool's historical rows authenticate. Every producer whose schema-16 contract differs from today's now has its row sealed by the production readiness, absence, input-evidence and output-evidence builders, run against the historical contract on the fixture's own late-DAG input frame (with the WSAL_VAL/SEMP_VAL inputs the schema-16 immigration source read). The positive test asserts the historical surfaces (57 and 93 inputs, WSAL_VAL and SEMP_VAL present, no A_LFSR); a legacy pool whose immigration source row carries today's 56-input surface is refused, and a current pool whose rows carry the historical surfaces is refused, with a control proving the re-seal itself reproduces a loadable pool. Co-Authored-By: Claude Opus 5.5 --- .../us/test_us_multispine_pool_h5_io.py | 335 +++++++++++++++++- 1 file changed, 334 insertions(+), 1 deletion(-) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_multispine_pool_h5_io.py b/packages/microcosm-build/tests/engine_free/us/test_us_multispine_pool_h5_io.py index e44df57c20..8bb11c133d 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_multispine_pool_h5_io.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_multispine_pool_h5_io.py @@ -3,7 +3,7 @@ import hashlib import json import sys -from collections.abc import Callable +from collections.abc import Callable, Collection, Mapping from copy import deepcopy from dataclasses import replace from pathlib import Path @@ -16,6 +16,7 @@ import microcosm.build.us_runtime.h5_io as h5_io import microcosm.build.us_runtime.immigration as immigration_runtime import microcosm.build.us_runtime.post_transfer_calibration as post_transfer_calibration_runtime +import microcosm.build.us_runtime.puf_support as puf_support_runtime import microcosm.build.us_runtime.stacked_spine as stacked_spine_module import microcosm.build.us_runtime.us_late_producer_registry as late_producer_registry_module import microcosm.build.us_runtime.worker_identity as worker_identity_module @@ -51,6 +52,7 @@ us_multispine_pool_release_receipt, write_nullable_us_h5, ) +from microcosm.build.us_runtime.late_producer_dag import ProducerContract from microcosm.build.us_runtime.support_provenance import ( BASE_ASEC_SUPPORT_CHANNEL, spine_assembly_manifest, @@ -1606,13 +1608,258 @@ def historical(target_key: str, receipt: dict[str, object]) -> None: historical(target_key, receipt) +_IMMIGRATION_SOURCE_PRODUCER = "source:with_us_immigration_inputs" +_IMMIGRATION_TRANSFER_PRODUCER = "transfer:person/source_operator_immigration" + +#: One wage-earning CPS ASEC person record: the raw fields the immigration +#: source producer reads that the persisted fixture stack does not carry. +#: WSAL_VAL and SEMP_VAL are the schema-16 requirement #767 replaced with +#: A_LFSR, which only today's contract reads; both contracts read the rest. +_ASEC_PERSON_RECORD = { + "WSAL_VAL": 52_000.0, + "SEMP_VAL": 0.0, + "A_LFSR": 1.0, + "A_HSCOL": 0.0, + "A_MARITL": 7.0, + "A_MJOCC": 2.0, + "A_SPOUSE": 0.0, + "CAID": 2.0, + "CHAMPVA": 2.0, + "IHSFLG": 2.0, + "MCARE": 2.0, + "MIL": 2.0, + "PEAFEVER": 2.0, + "PEIO1COW": 4.0, + "PEN_SC1": 0.0, + "PEN_SC2": 0.0, + "RESNSS1": 0.0, + "RESNSS2": 0.0, + "SPM_CAPHOUSESUB": 0.0, + "SSI_YN": 2.0, + "SS_YN": 2.0, +} + + +def _historically_changed_late_producers() -> tuple[str, ...]: + """Name every producer whose schema-16 contract differs from today's.""" + + legacy = late_producer_registry_module.legacy_us_late_producer_contracts() + live = stacked_spine_module.CANONICAL_US_LATE_PRODUCER_REGISTRY + return tuple( + name for name in legacy.schedule.order if legacy.registry[name] != live[name] + ) + + +def _late_dag_input_frame(manifest: Mapping[str, object]) -> Frame: + """The fixture stack plus what the immigration producers' contracts read. + + This is the frame ``_write_ready_pool(stacked=True)`` hands the fixture + DAG builder, plus exactly the inputs the immigration source and transfer + contracts (schema-16 and current) declare that the fixture stack does not + carry: the raw ASEC person record on the ASEC-origin rows (the ACS spine + has no CPS fields, so its rows stay null), ``is_female``, the person and + household spine source ids, ``state_fips``, ACS ``TYPEHUGQ``, and the + stack and PUF-attachment manifests in frame metadata. The transfer's + optional predictors stay absent, so its row seals their tolerated + absences exactly as the builder does. One frame serves every sealed row, + so a row's input and output evidence are computed on the same frame. + """ + + frame, _reconciliation = _stacked_pool_frame_with_live_immigration() + assert ( + spine_provenance_counts(frame, boundary="late DAG input fixture") + == manifest["provenance_counts"] + ), "the rebuilt fixture stack is not the one this pool persisted" + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + person = tables["person"] + asec_person = ( + person[support_channel_column("person")] + .astype(str) + .eq(BASE_ASEC_SUPPORT_CHANNEL) + .to_numpy() + ) + for column, value in _ASEC_PERSON_RECORD.items(): + person[column] = np.where(asec_person, value, np.nan) + person["is_female"] = (np.arange(len(person)) % 2).astype(np.float64) + person["person_spine_source_id"] = person[ + support_source_id_column("person") + ].to_numpy(dtype=np.float64) + household = tables["household"] + acs_household = ( + household[support_channel_column("household")] + .astype(str) + .eq(stacked_spine_module.ACS_STACKED_SUPPORT_CHANNEL) + .to_numpy() + ) + # Every fixture household is in PUMA 0600101 / county 06001. + household["state_fips"] = 6.0 + household["household_spine_source_id"] = household[ + support_source_id_column("household") + ].to_numpy(dtype=np.float64) + household["TYPEHUGQ"] = np.where(acs_household, 1.0, np.nan) + weights = {entity: frame.weights_for(entity) for entity in frame.weighted_entities} + _sampling, stack_manifest = _fixture_stacked_sampling(1.0) + # The attachment step records its manifest under the controls the + # fixture's primary-PUF resource receipts declare (fraction 1, seed 578). + attached = puf_support_runtime._attach_clone_arm_to_seeded_sample( + frame, + fraction=1.0, + seed=578, + ) + return Frame( + tables, + frame.schema, + weights, + frame.strata, + mass_log=frame.mass_log, + metadata={ + **frame.metadata, + stacked_spine_module.STACKED_SPINE_MANIFEST_KEY: stack_manifest, + puf_support_runtime.PUF_CLONE_ATTACHMENT_MANIFEST_KEY: attached.metadata[ + puf_support_runtime.PUF_CLONE_ATTACHMENT_MANIFEST_KEY + ], + }, + ) + + +def _seal_execution_rows( + dag: dict[str, object], + contracts: Mapping[str, ProducerContract], + frame: Frame, +) -> None: + """Seal the named rows' surfaces exactly as ``run_stacked_late_producer_dag``. + + Readiness counts, tolerated-absence receipts, content-bound declared-input + evidence, and output evidence all come from the production builders, + computed on ``frame`` against the given contract and the row's own sealed + available-input receipts and callback receipt. Callers re-chain digests. + """ + + for row in dag["execution"]: + contract = contracts.get(row["producer"]) + if contract is None: + continue + available = row["available_input_receipts"] + unfilled_rows, invalid_rows = stacked_spine_module._late_input_readiness_rows( + frame, + contract, + available_input_receipts=available, + ) + row["declared_inputs"] = stacked_spine_module._late_declared_input_evidence( + frame, + contract, + available_input_receipts=available, + unfilled_rows=unfilled_rows, + invalid_rows=invalid_rows, + ) + row["declared_absence_receipts"] = { + receipt_id: dict(receipt) + for receipt_id, receipt in ( + stacked_spine_module._late_declared_absence_receipts( + contract, + unfilled_rows, + invalid_rows=invalid_rows, + ).items() + ) + } + row["output_surface"] = [ + stacked_spine_module._late_output_column_evidence( + frame, + output=output, + producer_receipt=row["producer_receipt"], + ) + for output in contract.outputs + ] + + +def _rechain_late_dag(dag: dict[str, object]) -> None: + """Re-derive every row's surface digests, the chain, and the receipt digest. + + The chain starts from the genesis digest of the receipt's own version, so + this serves both a current receipt and a version-3 (schema-9) one. + """ + + previous_sha256 = stacked_spine_module._late_execution_genesis_sha256( + producer_schedule_sha256=dag["producer_schedule"]["payload_sha256"], + input_frame_sha256=dag["input_frame_sha256"], + receipt_schema_version=dag["version"], + ) + for row in dag["execution"]: + row["input_surface_sha256"] = _json_sha256(row["declared_inputs"]) + row["output_surface_sha256"] = _json_sha256(row["output_surface"]) + row["producer_receipt_sha256"] = _json_sha256(row["producer_receipt"]) + row["previous_execution_sha256"] = previous_sha256 + row.pop("sha256", None) + row["sha256"] = stacked_spine_module._canonical_sha256(row) + previous_sha256 = row["sha256"] + dag["execution_chain_sha256"] = previous_sha256 + dag.pop("sha256", None) + dag["sha256"] = stacked_spine_module._canonical_sha256(dag) + + +def _reseal_current_pool_rows( + manifest_path: Path, + contracts: Mapping[str, ProducerContract], +) -> None: + """Re-seal a current pool's named rows and re-chain its late DAG.""" + + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + dag = manifest["stage_receipts"]["impute"]["stacked_late_producer_dag"] + _seal_execution_rows(dag, contracts, _late_dag_input_frame(manifest)) + _rechain_late_dag(dag) + manifest["late_producer_transition_authority_sha256"] = ( + stacked_spine_module._late_producer_transition_authority_receipt(dag)["sha256"] + ) + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + + +def _immigration_row_contracts( + *, historical: Collection[str] +) -> dict[str, ProducerContract]: + """Every historically changed producer's contract, legacy only if named.""" + + legacy = late_producer_registry_module.legacy_us_late_producer_contracts() + live = stacked_spine_module.CANONICAL_US_LATE_PRODUCER_REGISTRY + changed = _historically_changed_late_producers() + unknown = set(historical) - set(changed) + if unknown: + raise ValueError(f"{sorted(unknown)} have no distinct historical contract.") + return { + name: legacy.registry[name] if name in historical else live[name] + for name in changed + } + + +def _late_row(manifest_path: Path, producer: str) -> dict[str, object]: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + dag = manifest["stage_receipts"]["impute"]["stacked_late_producer_dag"] + return next(row for row in dag["execution"] if row["producer"] == producer) + + +def _declared_input_columns(row: Mapping[str, object]) -> set[str]: + return { + column["column"] + for declared in row["declared_inputs"] + for alternative in declared["evidence"]["alternatives"] + for column in alternative + } + + def _rewrite_as_legacy_relocated_worker_pool( manifest_path: Path, *, historical_contract: bool = True, + current_surface_producers: Collection[str] = (), ) -> tuple[dict[str, object], dict[str, object]]: """Re-sign the tiny fixture with the frozen schema-9 worker binding. + With ``historical_contract`` (the default), every producer whose + schema-16 contract differs from today's has its execution row sealed by + the production row builder against that historical contract, as a real + schema-9 pool's row was. ``current_surface_producers`` seals the named + ones against today's contract instead: the stale surface the loader must + refuse. + ``historical_contract=False`` keeps today's schedule, operator order, and immigration evidence and only lowers the version numbers: the forgery a real schema-9 pool can never be, which the loader must refuse. @@ -1716,6 +1963,18 @@ def _rewrite_as_legacy_relocated_worker_pool( group_receipt = dag["post_puf_transfer"]["groups"].get(row["producer"]) if group_receipt is not None: row["producer_receipt"] = json.loads(json.dumps(group_receipt)) + # Seal every row whose schema-16 contract differs from today's with + # the production readiness and evidence builders, run against that + # historical contract on the fixture's own late-DAG input frame, so + # the rows carry the surfaces a schema-9 build sealed. + _seal_execution_rows( + dag, + _immigration_row_contracts( + historical=set(_historically_changed_late_producers()) + - set(current_surface_producers) + ), + _late_dag_input_frame(manifest), + ) manifest["operator_order"] = list(h5_io._SCHEMA9_STACKED_POOL_OPERATOR_ORDER) else: schedule = dag["producer_schedule"] @@ -2591,6 +2850,27 @@ def test_scoring_loader_accepts_legacy_worker_alias_relocation_only( recorded_worker, semantic_identity = _rewrite_as_legacy_relocated_worker_pool( manifest_path ) + # The sealed rows are the schema-16 ones, not today's: the immigration + # source still reads WSAL_VAL/SEMP_VAL (#767 replaced them with A_LFSR), + # and the immigration transfer lacks the seven requirements #767 added. + legacy = late_producer_registry_module.legacy_us_late_producer_contracts() + live = stacked_spine_module.CANONICAL_US_LATE_PRODUCER_REGISTRY + source_row = _late_row(manifest_path, _IMMIGRATION_SOURCE_PRODUCER) + transfer_row = _late_row(manifest_path, _IMMIGRATION_TRANSFER_PRODUCER) + assert ( + len(source_row["declared_inputs"]) + == 57 + == len(legacy.registry[_IMMIGRATION_SOURCE_PRODUCER].inputs) + ) + assert ( + len(transfer_row["declared_inputs"]) + == 93 + == len(legacy.registry[_IMMIGRATION_TRANSFER_PRODUCER].inputs) + ) + assert len(live[_IMMIGRATION_SOURCE_PRODUCER].inputs) == 56 + assert len(live[_IMMIGRATION_TRANSFER_PRODUCER].inputs) == 100 + assert {"WSAL_VAL", "SEMP_VAL"} <= _declared_input_columns(source_row) + assert "A_LFSR" not in _declared_input_columns(source_row) manifest_sha256 = _sha256(manifest_path) attestation_path = _write_legacy_worker_attestation( manifest_path, @@ -2652,6 +2932,59 @@ def test_scoring_loader_accepts_legacy_worker_alias_relocation_only( ) +def test_scoring_loader_refuses_legacy_rows_sealed_with_todays_surfaces( + tmp_path: Path, +) -> None: + """An attested schema-9 pool must carry the rows schema 16 sealed.""" + + pytest.importorskip("tables") + manifest_path = _write_gate_failed_pool(tmp_path) + recorded_worker, semantic_identity = _rewrite_as_legacy_relocated_worker_pool( + manifest_path, + current_surface_producers=(_IMMIGRATION_SOURCE_PRODUCER,), + ) + source_row = _late_row(manifest_path, _IMMIGRATION_SOURCE_PRODUCER) + assert len(source_row["declared_inputs"]) == 56 + attestation_path = _write_legacy_worker_attestation( + manifest_path, + recorded_worker=recorded_worker, + semantic_identity=semantic_identity, + ) + + with pytest.raises(ValueError, match="exact 57-input readiness surface"): + load_authenticated_us_multispine_pool_for_scoring( + manifest_path, + expected_manifest_sha256=_sha256(manifest_path), + worker_identity_attestation=attestation_path, + ) + + +def test_current_loader_refuses_rows_sealed_with_historical_surfaces( + tmp_path: Path, +) -> None: + """History authenticates only under an attestation, never on the current path.""" + + pytest.importorskip("tables") + (tmp_path / "control").mkdir() + control_path = _write_gate_failed_pool(tmp_path / "control") + # Control: re-sealing with today's contracts reproduces a loadable pool, + # so the refusal below comes from the historical surfaces, not the helper. + _reseal_current_pool_rows(control_path, _immigration_row_contracts(historical=())) + load_authenticated_us_multispine_pool_for_scoring(control_path) + + (tmp_path / "historical").mkdir() + manifest_path = _write_gate_failed_pool(tmp_path / "historical") + _reseal_current_pool_rows( + manifest_path, + _immigration_row_contracts(historical=_historically_changed_late_producers()), + ) + source_row = _late_row(manifest_path, _IMMIGRATION_SOURCE_PRODUCER) + assert len(source_row["declared_inputs"]) == 57 + + with pytest.raises(ValueError, match="exact 56-input readiness surface"): + load_authenticated_us_multispine_pool_for_scoring(manifest_path) + + def test_scoring_loader_refuses_version_downgraded_current_contract( tmp_path: Path, ) -> None: From bad5cb6679fe22c7560d14de2f9bbc86abcba440 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 9 Nov 2026 23:46:02 -0500 Subject: [PATCH 40/49] Add default-pool stage receipts for the immigration statuses The release-build rule asks this PR for a from-scratch build from its own tree; d2729 asks Max whether stage receipts stand in for it, on the terms of d2647. The receipts run the immigration stage alone on main's default pool (the 2024-2026 ASEC, the pool d2603 gives the next certified build) at this PR's merge of main 764ae3aa7, at that main, and at the pre-sync head 42bb166b1, then run the composition gate. At the merge the gate passes with no failures: refugees, asylees and TPS emit within 3% of their cited stocks, one-year parole reaches 0.90 and citizens match main. The pre-sync head refuses the 2026 ASEC's PEINUSYR code 29. The receipts hold aggregates only; the record-level labels behind the README's row-by-row table stay out of the repository, and scripts/xtab_labels.py rebuilds the table from two runs' labels. Refs #767 Co-Authored-By: Claude Opus 5.5 --- .../us-immigration-statuses-767/README.md | 110 ++++++ .../receipt.head.json | 288 ++++++++++++++ .../receipt.main.json | 101 +++++ .../receipt.pre_sync_42bb166b.json | 42 +++ .../scripts/stage_receipt_pooled.py | 354 ++++++++++++++++++ .../scripts/xtab_labels.py | 86 +++++ 6 files changed, 981 insertions(+) create mode 100644 docs/evidence/us-immigration-statuses-767/README.md create mode 100644 docs/evidence/us-immigration-statuses-767/receipt.head.json create mode 100644 docs/evidence/us-immigration-statuses-767/receipt.main.json create mode 100644 docs/evidence/us-immigration-statuses-767/receipt.pre_sync_42bb166b.json create mode 100644 docs/evidence/us-immigration-statuses-767/scripts/stage_receipt_pooled.py create mode 100644 docs/evidence/us-immigration-statuses-767/scripts/xtab_labels.py diff --git a/docs/evidence/us-immigration-statuses-767/README.md b/docs/evidence/us-immigration-statuses-767/README.md new file mode 100644 index 0000000000..1950854df2 --- /dev/null +++ b/docs/evidence/us-immigration-statuses-767/README.md @@ -0,0 +1,110 @@ +# US immigration statuses: default-pool stage receipts (microcosm#767) + +PR #779 draws humanitarian and temporary-protection statuses in the US +immigration source stage (`us_runtime/immigration.py`). +[`docs/us-release-build-rule.md`](../../us-release-build-rule.md) asks a pull +request that changes how the dataset is built to carry a from-scratch build from +its own tree before it merges. These receipts run that one stage alone, on the +inputs the next certified build uses, at this PR's head and at main. Max decides +whether they stand in for the from-scratch run (d2729, on the terms of +d2647). Nothing here changes a build. + +## What the script runs + +`scripts/stage_receipt_pooled.py` runs the stage on the three Census ASEC person +files of main's default pool: the 2024, 2025 and 2026 ASEC, income years 2023 to +2025 (`asec_sources.ASEC_DEFAULT_POOL_INCOME_YEARS`). Max's ruling d2603 moves +the next certified build to that pool. The script: + +- checks each archive against the SHA-256 that + `us_runtime/education_assistance_source.py` pins; +- weights each person by the household's `HSUP_WGT` times 0.01 and rescales each + file to one third of income year 2024's weighted population, as + `asec_pool.pool_asec_sources` does when `build_pooled_asec_unit_frame` pools + for target year 2024; +- keys the draws by income year, `PH_SEQ` and `PERIDNUM`, the pool's lineage + columns; +- runs at seed 0 and period 2024, the `--target-year` of + `tools/route_a/route_a.sh`; +- runs the composition gate on the output. + +The receipts hold aggregates only. Each records the tree's commit and the +SHA-256 of its `immigration.py` and of the script. + +## Receipts + +| File | Tree | Result | +|---|---|---| +| `receipt.head.json` | this PR at `4b3b6a7a6`, its merge of main `764ae3aa7` | the gate passes with no failures | +| `receipt.main.json` | main at `764ae3aa7` | main draws no humanitarian status | +| `receipt.pre_sync_42bb166b.json` | this PR at `42bb166b1`, before it merged main `1acd884fe` | the stage refuses the 2026 ASEC's `PEINUSYR` code 29 | + +The script reran main's and the pre-sync tree from `git archive +packages`, so those two receipts take their commit from `--source-commit`. Their +`immigration_module_sha256` values equal the SHA-256 of `immigration.py` at +those commits. + +## Results + +Weighted persons, in millions: + +| Status | Main | This PR | Cited target | +|---|---:|---:|---:| +| Citizen | 300.244 | 300.244 | | +| Lawful permanent resident | 11.680 | 10.427 | | +| Undocumented | 13.071 | 12.891 | | +| DACA | 0.376 | 0.277 | | +| Cuban/Haitian entrant | 0.581 | 0.539 | | +| Paroled, one year | 0 | 0.398 | 0.442 | +| Refugee | 0 | 0.160 | 0.160 | +| Asylee | 0 | 0.155 | 0.155 | +| TPS | 0 | 0.860 | 0.858 | +| All | 325.952 | 325.952 | | + +- Ten of the twelve positive-target draws emit within 3% of their targets. + Nicaraguan TPS emits 3,193 against 2,910. The U4U Ukrainian parole draw emits + 112,934 against 158,000: it exhausts its candidates, Ukraine-born non-citizens + in arrival codes 28 and 29, at 0.71. The composition gate's band runs from + 0.35 to 1.5 of each target. Deportation withheld carries a cited zero and + emits none. +- Pew's broad unauthorized universe totals 14.574 million against its 14.0 + million anchor, and that universe's labor force 9.699 million against the 9.7 + million control. +- In both trees `ssn_card_type` is `NONE` exactly on the undocumented rows. + +The totals move little, but the two trees label different rows undocumented. +Main's EAD spill counts a worker by prior-year wage or self-employment earnings +(`WSAL_VAL`, `SEMP_VAL`) and spills until the remaining undocumented rows match +the controls. This PR counts current labor-force status (`A_LFSR`, age 16 and +over) and spills until Pew's broad universe matches them. A spilled row takes an +EAD, and an EAD holder with no other mark is a lawful permanent resident. Row by +row (millions, main's label down the side, this PR's across): + +| Main \ this PR | LPR | Undocumented | DACA | Cuban/Haitian | Paroled | Refugee | Asylee | TPS | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| LPR | 8.406 | 2.415 | | | 0.228 | 0.160 | 0.155 | 0.315 | +| Undocumented | 2.021 | 10.059 | 0.172 | 0.104 | 0.170 | | | 0.545 | +| DACA | | 0.271 | 0.105 | | | | | | +| Cuban/Haitian | | 0.146 | | 0.435 | | | | | + +Citizens keep their label in both trees (300.244 million). The table comes from +both runs' `--labels-out` files, which stay out of the repository because they +are record-level; `scripts/xtab_labels.py` rebuilds it from them. + +## Rerunning + +The archives are Census's public-use ASEC CSV releases at the URLs +`education_assistance_source.py` names. Put the three zips in one or more +directories, then run: + +```bash +.venv/bin/python -I docs/evidence/us-immigration-statuses-767/scripts/stage_receipt_pooled.py . --archive-dir --out receipt.json +``` + +For another commit, extract `git archive packages` into a directory and +pass that directory and `--source-commit `. Add `--labels-out .pkl` +to two runs, then rebuild the row-by-row table: + +```bash +.venv/bin/python -I docs/evidence/us-immigration-statuses-767/scripts/xtab_labels.py main.pkl head.pkl +``` diff --git a/docs/evidence/us-immigration-statuses-767/receipt.head.json b/docs/evidence/us-immigration-statuses-767/receipt.head.json new file mode 100644 index 0000000000..034e05c565 --- /dev/null +++ b/docs/evidence/us-immigration-statuses-767/receipt.head.json @@ -0,0 +1,288 @@ +{ + "archives": [ + { + "archive": "asecpub24csv.zip", + "household_weighted_persons": 320890854.43000007, + "income_year": 2023, + "marsupwt_persons": 332382485.08, + "peinusyr_max": 28, + "rows": 144265, + "sha256": "cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09" + }, + { + "archive": "asecpub25csv.zip", + "household_weighted_persons": 325952369.06, + "income_year": 2024, + "marsupwt_persons": 337689641.57, + "peinusyr_max": 28, + "rows": 142125, + "sha256": "318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b" + }, + { + "archive": "asecpub26csv.zip", + "household_weighted_persons": 329518151.84999996, + "income_year": 2025, + "marsupwt_persons": 338554893.54, + "peinusyr_max": 29, + "rows": 134729, + "sha256": "fe819d0d2fc4470c282e76c247b8aa8b6054811619730307e9f2fe6186707d93" + } + ], + "composition": { + "ASYLEE": { + "by_income_year": { + "2023": 61212.7587594046, + "2024": 46805.7, + "2025": 47473.44368574968 + }, + "code_29_weighted": 0.0, + "rows": 186, + "under_19_weighted": 46328.961730005605, + "weighted": 155491.90244515426 + }, + "CITIZEN": { + "by_income_year": { + "2023": 100026523.85662985, + "2024": 99788608.02, + "2025": 100429092.3485838 + }, + "code_29_weighted": 174138.23378113355, + "rows": 388829, + "under_19_weighted": 69797402.3449544, + "weighted": 300244224.22521365 + }, + "CUBAN_HAITIAN_ENTRANT": { + "by_income_year": { + "2023": 166021.48593467462, + "2024": 204853.16333333333, + "2025": 168316.9858028873 + }, + "code_29_weighted": 23718.69405929822, + "rows": 630, + "under_19_weighted": 63696.40176895717, + "weighted": 539191.6350708952 + }, + "DACA": { + "by_income_year": { + "2023": 104140.60565893029, + "2024": 95633.99666666666, + "2025": 76955.79573372872 + }, + "code_29_weighted": 0.0, + "rows": 359, + "under_19_weighted": 4168.370981434536, + "weighted": 276730.39805932564 + }, + "LEGAL_PERMANENT_RESIDENT": { + "by_income_year": { + "2023": 3526808.2138986206, + "2024": 3555543.08, + "2025": 3344707.160514318 + }, + "code_29_weighted": 345716.8718042292, + "rows": 13141, + "under_19_weighted": 915650.1381589137, + "weighted": 10427058.454412939 + }, + "PAROLED_ONE_YEAR": { + "by_income_year": { + "2023": 99339.58014190361, + "2024": 149332.66333333333, + "2025": 149216.92251845784 + }, + "code_29_weighted": 26609.018450874144, + "rows": 438, + "under_19_weighted": 100118.51899012196, + "weighted": 397889.1659936948 + }, + "REFUGEE": { + "by_income_year": { + "2023": 47182.47778915634, + "2024": 53504.206666666665, + "2025": 59764.97282362702 + }, + "code_29_weighted": 25519.560004618885, + "rows": 176, + "under_19_weighted": 64043.9924089673, + "weighted": 160451.65727945004 + }, + "TPS": { + "by_income_year": { + "2023": 277924.2046921097, + "2024": 311935.7733333333, + "2025": 270260.13037578424 + }, + "code_29_weighted": 0.0, + "rows": 1081, + "under_19_weighted": 107278.50949508903, + "weighted": 860120.1084012273 + }, + "UNDOCUMENTED": { + "by_income_year": { + "2023": 4341636.503162008, + "2024": 4444573.083333333, + "2025": 4105001.9266283335 + }, + "code_29_weighted": 464004.9826060645, + "rows": 16279, + "under_19_weighted": 1583961.7706165907, + "weighted": 12891211.513123676 + } + }, + "draw_keys": "source_year:source_household_id(PH_SEQ):source_person_id(PERIDNUM)", + "error": null, + "gate_failures": [], + "gate_passed": true, + "humanitarian_draw_achieved": { + "asylee": { + "category": "asylee", + "origin": null, + "population": 155491.90244515426, + "relative": 1.0031735641622856, + "source": "https://ohss.dhs.gov/topics/immigration/asylees/annual-flow-report/fy24-asylees-flow-report", + "status": "ASYLEE", + "target": 155000.0 + }, + "deportation_withheld": { + "category": "deportation_withheld", + "origin": null, + "population": 0.0, + "relative": null, + "source": "https://crsreports.congress.gov/product/pdf/IN/IN12501", + "status": "DEPORTATION_WITHHELD", + "target": 0.0 + }, + "paroled_one_year:afghanistan": { + "category": "paroled_one_year", + "origin": "afghanistan", + "population": 74154.29223282929, + "relative": 1.007996795161206, + "source": "https://www.dhs.gov/sites/default/files/2023-06/PLCY%20-%20AFG%202503%20Update%202_0.pdf", + "status": "PAROLED_ONE_YEAR", + "target": 73566.0 + }, + "paroled_one_year:nicaragua": { + "category": "paroled_one_year", + "origin": "nicaragua", + "population": 93112.73319928072, + "relative": 1.0004591511688055, + "source": "https://www.cbp.gov/newsroom/national-media-release/cbp-releases-december-2024-monthly-update", + "status": "PAROLED_ONE_YEAR", + "target": 93070.0 + }, + "paroled_one_year:ukraine": { + "category": "paroled_one_year", + "origin": "ukraine", + "population": 112933.72361361884, + "relative": 0.7147704026178407, + "source": "https://www.dhs.gov/sites/default/files/2024-12/2024_1104_dmo_plcy_uniting_for_ukraine_process_overview_and_assessment.pdf", + "status": "PAROLED_ONE_YEAR", + "target": 158000.0 + }, + "paroled_one_year:venezuela": { + "category": "paroled_one_year", + "origin": "venezuela", + "population": 117688.41694796596, + "relative": 1.0030547766808655, + "source": "https://www.cbp.gov/newsroom/national-media-release/cbp-releases-december-2024-monthly-update", + "status": "PAROLED_ONE_YEAR", + "target": 117330.0 + }, + "refugee": { + "category": "refugee", + "origin": null, + "population": 160451.65727945004, + "relative": 1.0028228579965628, + "source": "https://ohss.dhs.gov/topics/immigration/refugees/annual-flow-report/fy-24-refugees-flow-report", + "status": "REFUGEE", + "target": 160000.0 + }, + "tps:el_salvador": { + "category": "tps", + "origin": "el_salvador", + "population": 170410.32409187898, + "relative": 1.0016771438170697, + "source": "https://www.everycrsreport.com/reports/RS20844.html", + "status": "TPS", + "target": 170125.0 + }, + "tps:honduras": { + "category": "tps", + "origin": "honduras", + "population": 52053.03129856825, + "relative": 1.0161645934322743, + "source": "https://www.everycrsreport.com/reports/RS20844.html", + "status": "TPS", + "target": 51225.0 + }, + "tps:nepal": { + "category": "tps", + "origin": "nepal", + "population": 7349.0149911888375, + "relative": 1.0263987417861504, + "source": "https://www.everycrsreport.com/reports/RS20844.html", + "status": "TPS", + "target": 7160.0 + }, + "tps:nicaragua": { + "category": "tps", + "origin": "nicaragua", + "population": 3192.9463618784666, + "relative": 1.0972324267623597, + "source": "https://www.everycrsreport.com/reports/RS20844.html", + "status": "TPS", + "target": 2910.0 + }, + "tps:other_designated": { + "category": "tps", + "origin": "other_designated", + "population": 21484.827706756125, + "relative": 1.0127187229203924, + "source": "https://www.everycrsreport.com/reports/RS20844.html", + "status": "TPS", + "target": 21215.0 + }, + "tps:venezuela": { + "category": "tps", + "origin": "venezuela", + "population": 605629.9639509566, + "relative": 1.0010164441393297, + "source": "https://www.everycrsreport.com/reports/RS20844.html", + "status": "TPS", + "target": 605015.0 + } + }, + "humanitarian_target_relative_band": [ + 0.35, + 1.5 + ], + "humanitarian_targets_by_status": { + "ASYLEE": 155000.0, + "DEPORTATION_WITHHELD": 0.0, + "PAROLED_ONE_YEAR": 441966.0, + "REFUGEE": 160000.0, + "TPS": 857650.0 + }, + "immigration_module_sha256": "fdb29c5cf362e1013d435222ef1b5cd0a082d6703f97e85d3b8835f2e8be1ab1", + "none_ssn_iff_undocumented": true, + "pew_labor_force": 9699442.858246539, + "pew_labor_force_control": 9700000.0, + "pew_universe": 14573640.556715064, + "pew_universe_anchor": 14000000.0, + "ratio_to_target": { + "ASYLEE": 1.0031735641622856, + "DEPORTATION_WITHHELD": null, + "PAROLED_ONE_YEAR": 0.9002709846316115, + "REFUGEE": 1.0028228579965628, + "TPS": 1.0028800890820582 + }, + "rows": 421119, + "script_sha256": "0c11b99f4c5de98e9339fe237feffe6510ee97333660006b62db07b3776c77cb", + "seed": 0, + "source_commit": "4b3b6a7a60c35f0ff137bff74308488c7ae71014", + "source_commit_from": "git", + "source_tree_clean": true, + "stage_seconds": 5.9, + "time_period": 2024, + "weights": "HSUP_WGT * 0.01 by PH_SEQ, each file rescaled to one third of income year 2024's weighted person population (asec_pool.pool_asec_sources, anchor_year=2024)" +} diff --git a/docs/evidence/us-immigration-statuses-767/receipt.main.json b/docs/evidence/us-immigration-statuses-767/receipt.main.json new file mode 100644 index 0000000000..444a5fe7a9 --- /dev/null +++ b/docs/evidence/us-immigration-statuses-767/receipt.main.json @@ -0,0 +1,101 @@ +{ + "archives": [ + { + "archive": "asecpub24csv.zip", + "household_weighted_persons": 320890854.43000007, + "income_year": 2023, + "marsupwt_persons": 332382485.08, + "peinusyr_max": 28, + "rows": 144265, + "sha256": "cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09" + }, + { + "archive": "asecpub25csv.zip", + "household_weighted_persons": 325952369.06, + "income_year": 2024, + "marsupwt_persons": 337689641.57, + "peinusyr_max": 28, + "rows": 142125, + "sha256": "318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b" + }, + { + "archive": "asecpub26csv.zip", + "household_weighted_persons": 329518151.84999996, + "income_year": 2025, + "marsupwt_persons": 338554893.54, + "peinusyr_max": 29, + "rows": 134729, + "sha256": "fe819d0d2fc4470c282e76c247b8aa8b6054811619730307e9f2fe6186707d93" + } + ], + "composition": { + "CITIZEN": { + "by_income_year": { + "2023": 100026523.85662985, + "2024": 99788608.02, + "2025": 100429092.3485838 + }, + "code_29_weighted": 174138.23378113355, + "rows": 388829, + "under_19_weighted": 69797402.3449544, + "weighted": 300244224.22521365 + }, + "CUBAN_HAITIAN_ENTRANT": { + "by_income_year": { + "2023": 188218.28306411504, + "2024": 216763.02666666667, + "2025": 176496.12279339816 + }, + "code_29_weighted": 23192.688342167814, + "rows": 676, + "under_19_weighted": 64619.12753444083, + "weighted": 581477.4325241798 + }, + "DACA": { + "by_income_year": { + "2023": 149258.51740021014, + "2024": 127492.23999999999, + "2025": 99016.39532075019 + }, + "code_29_weighted": 0.0, + "rows": 511, + "under_19_weighted": 3745.3214781583056, + "weighted": 375767.1527209604 + }, + "LEGAL_PERMANENT_RESIDENT": { + "by_income_year": { + "2023": 3915174.3808217305, + "2024": 4020124.2700000005, + "2025": 3744677.211858804 + }, + "code_29_weighted": 392355.07269730815, + "rows": 14722, + "under_19_weighted": 1122543.9672262375, + "weighted": 11679975.862680536 + }, + "UNDOCUMENTED": { + "by_income_year": { + "2023": 4371614.648750752, + "2024": 4497802.13, + "2025": 4201507.608109934 + }, + "code_29_weighted": 470021.365885609, + "rows": 16381, + "under_19_weighted": 1694338.2479112435, + "weighted": 13070924.386860687 + } + }, + "draw_keys": "source_year:source_household_id(PH_SEQ):source_person_id(PERIDNUM)", + "error": null, + "immigration_module_sha256": "8871df8b5f86f6bbf8541259a69d52b7db214560e5dacf5ad6db15268368f826", + "none_ssn_iff_undocumented": true, + "rows": 421119, + "script_sha256": "0c11b99f4c5de98e9339fe237feffe6510ee97333660006b62db07b3776c77cb", + "seed": 0, + "source_commit": "764ae3aa778cbc9030e286fabc3777e25c18f403", + "source_commit_from": "argument", + "source_tree_clean": null, + "stage_seconds": 2.9, + "time_period": 2024, + "weights": "HSUP_WGT * 0.01 by PH_SEQ, each file rescaled to one third of income year 2024's weighted person population (asec_pool.pool_asec_sources, anchor_year=2024)" +} diff --git a/docs/evidence/us-immigration-statuses-767/receipt.pre_sync_42bb166b.json b/docs/evidence/us-immigration-statuses-767/receipt.pre_sync_42bb166b.json new file mode 100644 index 0000000000..a0e256946e --- /dev/null +++ b/docs/evidence/us-immigration-statuses-767/receipt.pre_sync_42bb166b.json @@ -0,0 +1,42 @@ +{ + "archives": [ + { + "archive": "asecpub24csv.zip", + "household_weighted_persons": 320890854.43000007, + "income_year": 2023, + "marsupwt_persons": 332382485.08, + "peinusyr_max": 28, + "rows": 144265, + "sha256": "cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09" + }, + { + "archive": "asecpub25csv.zip", + "household_weighted_persons": 325952369.06, + "income_year": 2024, + "marsupwt_persons": 337689641.57, + "peinusyr_max": 28, + "rows": 142125, + "sha256": "318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b" + }, + { + "archive": "asecpub26csv.zip", + "household_weighted_persons": 329518151.84999996, + "income_year": 2025, + "marsupwt_persons": 338554893.54, + "peinusyr_max": 29, + "rows": 134729, + "sha256": "fe819d0d2fc4470c282e76c247b8aa8b6054811619730307e9f2fe6186707d93" + } + ], + "draw_keys": "source_year:source_household_id(PH_SEQ):source_person_id(PERIDNUM)", + "error": "SourceRuntimeError: PEINUSYR carries value(s) outside the 2024 ASEC domain 0..28: [29.0].", + "immigration_module_sha256": "df6699344c5fddbb5322baeb22f574be846e43c8c7600fd5dea798189c382042", + "script_sha256": "0c11b99f4c5de98e9339fe237feffe6510ee97333660006b62db07b3776c77cb", + "seed": 0, + "source_commit": "42bb166b1484df036809814434c1acf862b8cbb8", + "source_commit_from": "argument", + "source_tree_clean": null, + "stage_seconds": 0.2, + "time_period": 2024, + "weights": "HSUP_WGT * 0.01 by PH_SEQ, each file rescaled to one third of income year 2024's weighted person population (asec_pool.pool_asec_sources, anchor_year=2024)" +} diff --git a/docs/evidence/us-immigration-statuses-767/scripts/stage_receipt_pooled.py b/docs/evidence/us-immigration-statuses-767/scripts/stage_receipt_pooled.py new file mode 100644 index 0000000000..6d75669c98 --- /dev/null +++ b/docs/evidence/us-immigration-statuses-767/scripts/stage_receipt_pooled.py @@ -0,0 +1,354 @@ +"""Pooled default-pool stage receipt for the US immigration stage (#767, #779). + +Runs the US immigration source stage alone on the three Census ASEC person +files of main's default pool (income years 2023, 2024 and 2025: the 2024, 2025 +and 2026 ASEC, ``asec_sources.ASEC_DEFAULT_POOL_INCOME_YEARS``), then runs the +composition gate on its output and writes a JSON receipt of aggregates. + +- Each archive must match the SHA-256 that + ``us_runtime/education_assistance_source.py`` pins for it. +- Weights (``--weighting household``, the default) follow + ``asec_pool.pool_asec_sources`` with ``anchor_year=2024``, as + ``build_pooled_asec_unit_frame`` calls it for target year 2024: each person + carries the household's ``HSUP_WGT`` times 0.01, and each file is rescaled to + one third of income year 2024's weighted person population. + ``--weighting person`` uses ``MARSUPWT / 100 / 3`` instead. +- Draw keys use the pool's full lineage: ``source_year`` (income year), + ``source_household_id`` (``PH_SEQ``) and ``source_person_id`` (``PERIDNUM``). +- Seed 0 and period 2024, Route A's ``--target-year``. + +A refusal is recorded as the receipt's ``error`` instead of raised. Main's tree +has no humanitarian controls, Pew universe or composition gate, so its receipt +carries the composition only. + +The archives are Census's public-use ASEC CSV releases, at the URLs +``education_assistance_source.py`` names: + + https://www2.census.gov/programs-surveys/cps/datasets/2024/march/asecpub24csv.zip + https://www2.census.gov/programs-surveys/cps/datasets/2025/march/asecpub25csv.zip + https://www2.census.gov/programs-surveys/cps/datasets/2026/march/asecpub26csv.zip + +Run it with any interpreter that has the workspace's dependencies, pointing at +the tree to measure (a checkout, or ``git archive packages`` extracted): + + .venv/bin/python -I docs/evidence/us-immigration-statuses-767/scripts/stage_receipt_pooled.py \\ + --archive-dir --out receipt.json + +``--labels-out`` also writes the row-level labels (a pickle) for +cross-tabulating two trees' runs. Those rows come from the public files but are +record-level, so they stay out of the repository. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import subprocess +import sys +import time +import zipfile +from pathlib import Path + +PERIOD = 2024 +ANCHOR_INCOME_YEAR = 2024 +ARCHIVES = ( + ( + 2023, + "asecpub24csv.zip", + "pppub24.csv", + "hhpub24.csv", + "cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09", + ), + ( + 2024, + "asecpub25csv.zip", + "pppub25.csv", + "hhpub25.csv", + "318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b", + ), + ( + 2025, + "asecpub26csv.zip", + "pppub26.csv", + "hhpub26.csv", + "fe819d0d2fc4470c282e76c247b8aa8b6054811619730307e9f2fe6186707d93", + ), +) + +parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) +parser.add_argument("src_root", type=Path, help="tree whose packages/*/src to run") +parser.add_argument( + "--archive-dir", + type=Path, + action="append", + required=True, + help="directory holding ASEC zips (repeatable; each zip is found by name)", +) +parser.add_argument("--out", type=Path, required=True) +parser.add_argument("--weighting", choices=("household", "person"), default="household") +parser.add_argument( + "--source-commit", + help="commit the tree came from, when src_root is not a git checkout", +) +parser.add_argument("--labels-out", type=Path) +args = parser.parse_args() + +ROOT = args.src_root.resolve() +OUT = args.out.resolve() +for _src in sorted((ROOT / "packages").glob("*/src")): + sys.path.insert(0, str(_src)) + +import numpy as np +import pandas as pd + +import microcosm.build.us_runtime.immigration as immigration +from microcosm.build.source_runtime import SourceRuntimeConfig, run_source_stage +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights + +assert Path(immigration.__file__).resolve().is_relative_to(ROOT), immigration.__file__ + + +def _archive_path(name: str) -> Path: + found = [d / name for d in args.archive_dir if (d / name).is_file()] + if not found: + raise SystemExit(f"{name} is in none of {args.archive_dir}") + return found[0] + + +def _git(*command: str) -> str | None: + result = subprocess.run( + ["git", "-C", str(ROOT), *command], capture_output=True, text=True + ) + return result.stdout.strip() if result.returncode == 0 else None + + +git_top = _git("rev-parse", "--show-toplevel") +if git_top is not None and Path(git_top).resolve() == ROOT: + source_commit = _git("rev-parse", "HEAD") + source_commit_from = "git" + source_tree_clean = _git("status", "--porcelain", "--", "packages") == "" + if args.source_commit and args.source_commit != source_commit: + raise SystemExit( + f"--source-commit {args.source_commit} is not the checkout's " + f"HEAD {source_commit}" + ) +else: + source_commit = args.source_commit + source_commit_from = "argument" if source_commit else None + source_tree_clean = None + +columns = list( + dict.fromkeys( + [ + *immigration.US_IMMIGRATION_REQUIRED_SOURCE_COLUMNS, + "MARSUPWT", + "PERIDNUM", + "PH_SEQ", + ] + ) +) +parts = [] +archives = [] +raw_population = {} +for income_year, archive_name, member, household_member, pinned in ARCHIVES: + path = _archive_path(archive_name) + digest = hashlib.sha256(path.read_bytes()).hexdigest() + assert digest == pinned, (path, digest) + with zipfile.ZipFile(path) as archive: + names = archive.namelist() + name = next(n for n in names if n.lower().endswith(member)) + with archive.open(name) as handle: + frame = pd.read_csv(handle, usecols=columns) + name = next(n for n in names if n.lower().endswith(household_member)) + with archive.open(name) as handle: + household = pd.read_csv(handle, usecols=["H_SEQ", "HSUP_WGT"]) + household_weight = household.drop_duplicates("H_SEQ").set_index("H_SEQ")["HSUP_WGT"] + frame["household_weight"] = frame["PH_SEQ"].map(household_weight) * 0.01 + assert not frame["household_weight"].isna().any(), income_year + frame["source_year"] = income_year + frame["source_household_id"] = frame["PH_SEQ"].to_numpy() + frame["source_person_id"] = frame["PERIDNUM"].astype(str) + raw_population[income_year] = float(frame["household_weight"].sum()) + parts.append(frame) + archives.append( + { + "income_year": income_year, + "archive": archive_name, + "sha256": digest, + "rows": int(len(frame)), + "peinusyr_max": int(frame["PEINUSYR"].max()), + "household_weighted_persons": raw_population[income_year], + "marsupwt_persons": float(frame["MARSUPWT"].sum() / 100.0), + } + ) +target_population = raw_population[ANCHOR_INCOME_YEAR] +for frame in parts: + year = int(frame["source_year"].iloc[0]) + if args.weighting == "household": + scale = target_population / len(ARCHIVES) / raw_population[year] + frame["person_weight"] = frame["household_weight"] * scale + else: + frame["person_weight"] = frame["MARSUPWT"] / 100.0 / len(ARCHIVES) + frame.drop(columns=["household_weight"], inplace=True) +person = pd.concat(parts, ignore_index=True) +person["person_id"] = np.arange(1, len(person) + 1, dtype=np.int64) +person["person_household_id"] = 0 + +started = time.monotonic() +error = None +try: + output = run_source_stage( + immigration.us_immigration_stage_spec(), + tables={"person": person}, + operation_handlers={ + "derive_immigration_status": ( + immigration.derive_us_immigration_status_from_manifest + ) + }, + config=SourceRuntimeConfig(seed=0, target_year=PERIOD), + ) +except Exception as exc: # the receipt records a refusal instead of hiding it + error = f"{type(exc).__name__}: {exc}" +stage_seconds = time.monotonic() - started + +receipt = { + "source_commit": source_commit, + "source_commit_from": source_commit_from, + "source_tree_clean": source_tree_clean, + "immigration_module_sha256": hashlib.sha256( + Path(immigration.__file__).read_bytes() + ).hexdigest(), + "script_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "archives": archives, + "seed": 0, + "time_period": PERIOD, + "weights": ( + "HSUP_WGT * 0.01 by PH_SEQ, each file rescaled to one third of income " + "year 2024's weighted person population (asec_pool.pool_asec_sources, " + "anchor_year=2024)" + if args.weighting == "household" + else "MARSUPWT / 100 / 3 per file" + ), + "draw_keys": "source_year:source_household_id(PH_SEQ):source_person_id(PERIDNUM)", + "stage_seconds": round(stage_seconds, 1), + "error": error, +} +if error is None: + weights = output["person_weight"].to_numpy(dtype=np.float64) + status = output["immigration_status_str"].astype(str).to_numpy() + ssn = output["ssn_card_type"].astype(str).to_numpy() + age = output["A_AGE"].to_numpy() + code = output["PEINUSYR"].to_numpy() + year = output["source_year"].to_numpy() + composition = {} + for value in sorted(set(status)): + mask = status == value + composition[value] = { + "weighted": float(weights[mask].sum()), + "rows": int(mask.sum()), + "by_income_year": { + str(y): float(weights[mask & (year == y)].sum()) + for y in (2023, 2024, 2025) + }, + "code_29_weighted": float(weights[mask & (code == 29)].sum()), + "under_19_weighted": float(weights[mask & (age < 19)].sum()), + } + receipt.update( + { + "rows": int(len(output)), + "composition": composition, + "none_ssn_iff_undocumented": bool( + ((ssn == "NONE") == (status == "UNDOCUMENTED")).all() + ), + } + ) + # This PR's controls, Pew universe and gate; main's tree has none of them. + if hasattr(immigration, "us_immigration_controls"): + labor_force = np.isin(output["A_LFSR"].to_numpy(), [1, 2, 3, 4]) & (age >= 16) + controls = immigration.us_immigration_controls() + targets: dict[str, float] = {} + for draw in controls.humanitarian: + targets[draw.status] = targets.get(draw.status, 0.0) + float(draw.target) + pew = np.isin(status, immigration._PEW_UNAUTHORIZED_STATUS_VALUES) | ( + (status == "CUBAN_HAITIAN_ENTRANT") & (ssn == "NON_CITIZEN_VALID_EAD") + ) + gate_person = ( + output.drop(columns=["person_weight"]).reset_index(drop=True).copy() + ) + ids = np.arange(1, len(gate_person) + 1, dtype=np.int64) + for entity in US_SCHEMA.group_entities: + gate_person[US_SCHEMA.membership_column(entity)] = ids + frame = Frame( + { + "person": gate_person, + **{ + entity: pd.DataFrame({US_SCHEMA.id_column(entity): ids}) + for entity in US_SCHEMA.group_entities + }, + }, + US_SCHEMA, + {"household": Weights(weights, WeightKind.DESIGN)}, + ) + gate = immigration.us_immigration_composition_gate( + frame, time_period=PERIOD, controls=controls + ) + receipt.update( + { + "humanitarian_targets_by_status": targets, + "ratio_to_target": { + k: composition.get(k, {}).get("weighted", 0.0) / v if v else None + for k, v in targets.items() + }, + "pew_universe": float(weights[pew].sum()), + "pew_universe_anchor": float(controls.undocumented.population_anchor), + "pew_labor_force": float(weights[pew & labor_force].sum()), + "pew_labor_force_control": float(controls.undocumented.workers), + "gate_passed": bool(gate.passed), + "gate_failures": list(gate.failures), + "humanitarian_draw_achieved": gate.details.get( + "humanitarian_draw_achieved" + ), + "humanitarian_target_relative_band": list( + immigration._HUMANITARIAN_TARGET_RELATIVE_BAND + ), + } + ) +OUT.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n") +if error is None and args.labels_out is not None: + pd.DataFrame( + { + "source_year": output["source_year"].to_numpy(), + "source_person_id": output["source_person_id"].astype(str).to_numpy(), + "person_weight": output["person_weight"].to_numpy(dtype=np.float64), + "immigration_status_str": output["immigration_status_str"].astype(str), + "ssn_card_type": output["ssn_card_type"].astype(str), + } + ).to_pickle(args.labels_out) + +print("weighting:", args.weighting) +print("source commit:", source_commit, f"({source_commit_from})") +print("archives:", [(a["income_year"], a["rows"], a["peinusyr_max"]) for a in archives]) +if error is not None: + print("REFUSED:", error) + raise SystemExit(0) +print(f"stage {stage_seconds:.0f}s on {len(output)} rows") +ratios = receipt.get("ratio_to_target", {}) +for value, row in composition.items(): + ratio = ratios.get(value) + ratio_text = f" ratio {ratio:.2f}" if ratio is not None else "" + years = " ".join(f"{y}:{v / 1e6:.3f}" for y, v in row["by_income_year"].items()) + print( + f" {value:26s} {row['weighted'] / 1e6:8.3f}M{ratio_text} [{years}]" + f" code29 {row['code_29_weighted'] / 1e6:.3f}M" + f" u19 {row['under_19_weighted'] / 1e6:.3f}M" + ) +print("NONE ssn iff UNDOCUMENTED:", receipt["none_ssn_iff_undocumented"]) +if "gate_passed" in receipt: + print( + f"Pew universe {receipt['pew_universe'] / 1e6:.3f}M vs anchor " + f"{receipt['pew_universe_anchor'] / 1e6:.1f}M; labor force " + f"{receipt['pew_labor_force'] / 1e6:.3f}M vs control " + f"{receipt['pew_labor_force_control'] / 1e6:.1f}M" + ) + print("gate passed:", receipt["gate_passed"], "failures:", receipt["gate_failures"]) diff --git a/docs/evidence/us-immigration-statuses-767/scripts/xtab_labels.py b/docs/evidence/us-immigration-statuses-767/scripts/xtab_labels.py new file mode 100644 index 0000000000..47b5617881 --- /dev/null +++ b/docs/evidence/us-immigration-statuses-767/scripts/xtab_labels.py @@ -0,0 +1,86 @@ +"""Cross-tabulate two stage runs' immigration labels (millions of persons). + +Usage: python -I xtab_labels.py MAIN_LABELS.pkl HEAD_LABELS.pkl [OUT.json] + +Rows align on (source_year, source_person_id); the script refuses unequal +row sets or person weights. Prints the README's main-by-head table. +""" + +import json +import sys + +import numpy as np +import pandas as pd + +ORDER = [ + "CITIZEN", + "LEGAL_PERMANENT_RESIDENT", + "UNDOCUMENTED", + "DACA", + "CUBAN_HAITIAN_ENTRANT", + "PAROLED_ONE_YEAR", + "REFUGEE", + "ASYLEE", + "TPS", +] + + +def load(path): + frame = pd.read_pickle(path) + key = frame["source_year"].astype(str) + ":" + frame["source_person_id"].astype(str) + if key.duplicated().any(): + raise SystemExit(f"{path}: duplicate person keys") + return frame.set_index(key) + + +main_path, head_path = sys.argv[1:3] +main, head = load(main_path), load(head_path) +if set(main.index) != set(head.index): + raise SystemExit("the two runs hold different persons") +head = head.loc[main.index] +if not np.allclose(main["person_weight"], head["person_weight"], rtol=0, atol=1e-9): + raise SystemExit("the two runs weight persons differently") +weight = main["person_weight"].to_numpy() +table = ( + pd.DataFrame( + { + "main": main["immigration_status_str"].to_numpy(), + "head": head["immigration_status_str"].to_numpy(), + "w": weight, + } + ) + .groupby(["main", "head"])["w"] + .sum() + .div(1e6) + .unstack(fill_value=0.0) +) +statuses = [s for s in ORDER if s in set(table.index) | set(table.columns)] +statuses += sorted((set(table.index) | set(table.columns)) - set(statuses)) +table = table.reindex(index=statuses, columns=statuses, fill_value=0.0) +pd.set_option("display.width", 200) +print(table.round(3).to_string()) +totals = { + "main": {s: round(float(table.loc[s].sum()), 3) for s in statuses}, + "head": {s: round(float(table[s].sum()), 3) for s in statuses}, +} +print(json.dumps(totals, indent=1)) +none_main = main["ssn_card_type"].eq("NONE").to_numpy() +none_head = head["ssn_card_type"].eq("NONE").to_numpy() +und_main = main["immigration_status_str"].eq("UNDOCUMENTED").to_numpy() +und_head = head["immigration_status_str"].eq("UNDOCUMENTED").to_numpy() +print( + "NONE iff UNDOCUMENTED:", + bool((none_main == und_main).all()), + bool((none_head == und_head).all()), +) +if len(sys.argv) > 3: + out = { + "main_labels": main_path, + "head_labels": head_path, + "millions": { + r: {c: round(float(table.loc[r, c]), 6) for c in statuses} for r in statuses + }, + "totals": totals, + } + with open(sys.argv[3], "w") as stream: + json.dump(out, stream, indent=1, sort_keys=True) From e3b2c8ce9404d177a5424b2dd3b1acb6cf385e6f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 10 Nov 2026 01:51:36 -0500 Subject: [PATCH 41/49] Re-pin stacked authority after the tenth merge; document the OAW window The independent review of bad5cb667 found that the tenth merge kept main's stacked-authority pins (receipt and late-producer schedule component) in test_spec_engine_stacked_authority_semantics.py, so engine-US failed. Pin the merged tree's live values. The review also found that the Operation Allies Welcome parole window on exact-year (ACS YOEP) rows has no end year. Keep it open and say why above _PAROLE_ACS_MIN_ARRIVAL_YEAR: the stacked transfer asks the ACS arm for its own share of each national target and raises on a candidate shortfall, and the ACS 2024 1-year holds 70,123 weighted Afghan-born non-citizens who arrived in 2021-2022, fewer than the 73,566 target. The comment moves the attested kernel identities: seed protocol and map digests, the US spec (49d7b804 -> c414c985), the coverage evidence and the minimal-loader golden. Name the legacy path's second reach in the multispine design doc: post-clone source rows bind today's whole-file source_stages.json hash, which #767 changed for the fifteen source rows that bind a stage spec. The stage receipt script now records the key each draw call uses instead of a constant, and --draw-keys full|legacy forces one key so two trees' runs share their draws. Refs #767 Co-Authored-By: Claude Opus 5.5 --- docs/evidence/spec-engine/us-f0-coverage.json | 16 +-- .../scripts/stage_receipt_pooled.py | 127 +++++++++++++++++- docs/us-multispine-operator-ordering.md | 14 +- .../build/spec_engine/inventory_coverage.py | 4 +- .../microcosm/build/us_runtime/immigration.py | 7 + ...spec_engine_stacked_authority_semantics.py | 4 +- .../shared/test_spec_engine_loader.py | 2 +- .../us/test_us_multispine_pool_tool.py | 2 +- 8 files changed, 156 insertions(+), 20 deletions(-) diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index 1268ed02be..812fa53e2c 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -1655,13 +1655,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "73da1c877f8ccdc114f26a0bdcc73caf6ff10e346e5e1ad996fde17f5c366ace", - "protocol_sha256": "f4f3649532cde832cecff7139921616f60ec2947eb028e5b18b17662f4d41476" + "map_sha256": "cabadb3f2eeb49a1527c6bb4348d31e90b0a9ec8f836aa2cbe2b5d4ac6608798", + "protocol_sha256": "f0105714b03a3bef5cbc464519680e4fff3a9c2e758eece360a51cbfc656af8a" }, "failures": [], "observed": { - "map_sha256": "73da1c877f8ccdc114f26a0bdcc73caf6ff10e346e5e1ad996fde17f5c366ace", - "protocol_sha256": "f4f3649532cde832cecff7139921616f60ec2947eb028e5b18b17662f4d41476" + "map_sha256": "cabadb3f2eeb49a1527c6bb4348d31e90b0a9ec8f836aa2cbe2b5d4ac6608798", + "protocol_sha256": "f0105714b03a3bef5cbc464519680e4fff3a9c2e758eece360a51cbfc656af8a" }, "status": "covered" }, @@ -1676,7 +1676,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "f4f3649532cde832cecff7139921616f60ec2947eb028e5b18b17662f4d41476", + "implementation_sha256": "f0105714b03a3bef5cbc464519680e4fff3a9c2e758eece360a51cbfc656af8a", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1697,7 +1697,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "f4f3649532cde832cecff7139921616f60ec2947eb028e5b18b17662f4d41476", + "implementation_sha256": "f0105714b03a3bef5cbc464519680e4fff3a9c2e758eece360a51cbfc656af8a", "protocol": "legacy-v1", "streams": [ "build_model", @@ -2598,7 +2598,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "49d7b804829bdff5eb70ed16479a62e0dfb94de045f6c3c4374ea059ba634879" + "spec_sha256": "c414c985f6da4c8d623c79f15bce4c35a5fa32246fbfed941390b24fee41f429" } }, "report_schema_version": 3, @@ -2608,7 +2608,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "49d7b804829bdff5eb70ed16479a62e0dfb94de045f6c3c4374ea059ba634879" + "spec_sha256": "c414c985f6da4c8d623c79f15bce4c35a5fa32246fbfed941390b24fee41f429" }, "status": "pass" } diff --git a/docs/evidence/us-immigration-statuses-767/scripts/stage_receipt_pooled.py b/docs/evidence/us-immigration-statuses-767/scripts/stage_receipt_pooled.py index 6d75669c98..82ff98824f 100644 --- a/docs/evidence/us-immigration-statuses-767/scripts/stage_receipt_pooled.py +++ b/docs/evidence/us-immigration-statuses-767/scripts/stage_receipt_pooled.py @@ -13,8 +13,16 @@ carries the household's ``HSUP_WGT`` times 0.01, and each file is rescaled to one third of income year 2024's weighted person population. ``--weighting person`` uses ``MARSUPWT / 100 / 3`` instead. -- Draw keys use the pool's full lineage: ``source_year`` (income year), +- Each person carries the pool's full lineage: ``source_year`` (income year), ``source_household_id`` (``PH_SEQ``) and ``source_person_id`` (``PERIDNUM``). + The tree's kernel picks which of them key its draws. The script wraps + ``immigration._stable_person_draws`` and records, for each draw call, the + key that reproduces its draws: ``full`` (all three columns), ``legacy`` + (``source_year`` and ``source_person_id``) or ``person_id``. +- ``--draw-keys full`` or ``--draw-keys legacy`` hands that key to every draw + call that brings none of its own, so runs of two trees can share their + draws and differ only in method. It needs a tree whose + ``_stable_person_draws`` takes ``keys``. - Seed 0 and period 2024, Route A's ``--target-year``. A refusal is recorded as the receipt's ``error`` instead of raised. Main's tree @@ -43,6 +51,7 @@ import argparse import hashlib +import inspect import json import subprocess import sys @@ -92,6 +101,12 @@ help="commit the tree came from, when src_root is not a git checkout", ) parser.add_argument("--labels-out", type=Path) +parser.add_argument( + "--draw-keys", + choices=("native", "full", "legacy"), + default="native", + help="key the tree's draws as its kernel does (native) or force one key", +) args = parser.parse_args() ROOT = args.src_root.resolve() @@ -108,6 +123,100 @@ assert Path(immigration.__file__).resolve().is_relative_to(ROOT), immigration.__file__ +KEY_COLUMNS = { + "full": ("source_year", "source_household_id", "source_person_id"), + "legacy": ("source_year", "source_person_id"), + "person_id": ("person_id",), +} +KEY_LABELS = { + "full": "source_year:source_household_id(PH_SEQ):source_person_id(PERIDNUM)", + "legacy": "source_year:source_person_id(PERIDNUM)", + "person_id": "person_id", +} +kernel_draws = immigration._stable_person_draws +if ( + args.draw_keys != "native" + and "keys" not in inspect.signature(kernel_draws).parameters +): + raise SystemExit( + f"--draw-keys {args.draw_keys} needs a tree whose _stable_person_draws " + "takes keys" + ) +draw_calls: list[dict[str, object]] = [] + + +def _joined_keys(person: pd.DataFrame, columns: tuple[str, ...]) -> pd.Series: + keys = person[columns[0]].astype(str) + for column in columns[1:]: + keys = keys + ":" + person[column].astype(str) + return keys.reset_index(drop=True) + + +def _hashed_draws(keys: pd.Series, *, seed: int, salt: str) -> np.ndarray: + """The kernel's draw for each key: blake2b of ``seed:salt:key`` over 2**64.""" + + return np.fromiter( + ( + int.from_bytes( + hashlib.blake2b( + f"{seed}:{salt}:{key}".encode(), digest_size=8 + ).digest(), + byteorder="big", + signed=False, + ) + / float(2**64) + for key in keys + ), + dtype=np.float64, + count=len(keys), + ) + + +def _recording_draws(person: pd.DataFrame, **kwargs: object) -> np.ndarray: + """Run the kernel's draw and record which key reproduces it.""" + + caller_keys = kwargs.get("keys") + forced = args.draw_keys != "native" and caller_keys is None + if forced: + kwargs["keys"] = _joined_keys(person, KEY_COLUMNS[args.draw_keys]) + draws = np.asarray(kernel_draws(person, **kwargs)) + salt = str(kwargs["salt"]) + if len(person) == 0: + draw_calls.append({"salt": salt, "rows": 0, "key": None, "forced": forced}) + return draws + rows = np.unique( + np.r_[ + np.arange(min(1000, len(person))), + np.arange(max(0, len(person) - 1000), len(person)), + ] + ) + + def reproduces(keys: pd.Series) -> bool: + expected = _hashed_draws(keys.iloc[rows], seed=int(kwargs["seed"]), salt=salt) + return bool(np.array_equal(expected, draws[rows])) + + named = [ + name + for name, columns in KEY_COLUMNS.items() + if set(columns).issubset(person.columns) + and reproduces(_joined_keys(person, columns)) + ] + if caller_keys is not None: + if not reproduces(pd.Series(caller_keys).astype(str).reset_index(drop=True)): + raise SystemExit(f"draw call {salt!r} does not use its caller's keys") + key = "caller" + (f"={named[0]}" if len(named) == 1 else "") + elif len(named) == 1: + key = named[0] + else: + raise SystemExit(f"draw call {salt!r}: keys {named} reproduce its draws") + draw_calls.append( + {"salt": salt, "rows": int(len(person)), "key": key, "forced": forced} + ) + return draws + + +immigration._stable_person_draws = _recording_draws + def _archive_path(name: str) -> Path: found = [d / name for d in args.archive_dir if (d / name).is_file()] @@ -230,7 +339,14 @@ def _git(*command: str) -> str | None: if args.weighting == "household" else "MARSUPWT / 100 / 3 per file" ), - "draw_keys": "source_year:source_household_id(PH_SEQ):source_person_id(PERIDNUM)", + "draw_keys": { + "requested": args.draw_keys, + "used": { + name: KEY_LABELS.get(name, name) + for name in sorted({str(call["key"]) for call in draw_calls}) + }, + "calls": draw_calls, + }, "stage_seconds": round(stage_seconds, 1), "error": error, } @@ -328,6 +444,13 @@ def _git(*command: str) -> str | None: print("weighting:", args.weighting) print("source commit:", source_commit, f"({source_commit_from})") +print( + "draw keys:", + args.draw_keys, + "->", + receipt["draw_keys"]["used"], + f"over {len(draw_calls)} draw calls", +) print("archives:", [(a["income_year"], a["rows"], a["peinusyr_max"]) for a in archives]) if error is not None: print("REFUSED:", error) diff --git a/docs/us-multispine-operator-ordering.md b/docs/us-multispine-operator-ordering.md index 6da96e0515..de7a0b5be4 100644 --- a/docs/us-multispine-operator-ordering.md +++ b/docs/us-multispine-operator-ordering.md @@ -596,8 +596,8 @@ authorization tuple (`gate`, `plan_sha256`, `prompt_sha256`, not claim public-key or cryptographic signature verification. Attested legacy scoring checks a schema-9 pool against reconstructions that -follow today's late-producer registry, as main does, except where microcosm -#767 changed it. The frozen resource +follow today's late-producer registry, as main does, except for the two +producer contracts microcosm #767 changed. The frozen resource `us/legacy_schema16_late_producer_schedule.json`, which loads only if its recomputed payload SHA-256 equals `02e618cc656eb39990ed99dca2b30a52794e01e2b06a3c2df87ca4a7d85ab086`, supplies two @@ -621,8 +621,14 @@ longer hash to the sealed payload execution rows also bind today's `acs_transfer_execution_contract_identity` on this path, as on main, so #767's schema-3 transfer contract reaches it there; main's own post-transfer changes since `7c4a5ac0` already moved that identity -for all nineteen transfer groups. Restoring all of these to sealed history is -separate work. The pool's operator order must be the +for all nineteen transfer groups. Post-clone source execution rows likewise +bind today's source-stage binding, which hashes the whole packaged +`us/source_stages.json` and carries each bound stage's resolved spec. #767 +rewrote the `immigration_status` stage, so its change reaches the fifteen +source rows that bind a stage spec (direct housing assistance binds none). +Main has changed `source_stages.json` in fourteen commits since `7c4a5ac0`, so +on main too this binding refuses a pool whose rows bound an older file. +Restoring all of these to sealed history is separate work. The pool's operator order must be the literal eleven-step order ending at `by_origin_battery`, from before `us_immigration_composition_gate` existed, and the loader replays its immigration transfer targets as ordinary transfer receipts, without the paired diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 93251635db..b56aedaddb 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -396,8 +396,8 @@ # microcosm#767 moves the seed map, the protocol and the source manifest: # the attested immigration kernel gains the humanitarian status draws and # the immigration stage gains cited humanitarian stock targets. - "seed_map": "73da1c877f8ccdc114f26a0bdcc73caf6ff10e346e5e1ad996fde17f5c366ace", - "seed_protocol": "f4f3649532cde832cecff7139921616f60ec2947eb028e5b18b17662f4d41476", + "seed_map": "cabadb3f2eeb49a1527c6bb4348d31e90b0a9ec8f836aa2cbe2b5d4ac6608798", + "seed_protocol": "f0105714b03a3bef5cbc464519680e4fff3a9c2e758eece360a51cbfc656af8a", "source_manifest": "e9a70a60c3db9250066ee607d5e1fb07cd9d6a54424ab9eb9405f5e65b5a59b8", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", "tail": "565f0ceeadab69d3e96a886a00dd0305f7425fa3acde24d12ba48db2b37d5fc0", diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py b/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py index 4252187a62..dd646b627c 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py @@ -343,6 +343,13 @@ class is the better statutory label and keeps H.R.1 eligibility), as is "nicaragua": _POST_2021_ARRIVAL_CODES, "venezuela": _POST_2021_ARRIVAL_CODES, } +#: First exact (ACS YOEP) arrival year of each parole program. The windows +#: have no end year, although Operation Allies Welcome ended in 2022: the +#: stacked transfer asks the ACS arm for its own mass share of each national +#: target and refuses a candidate shortfall, and at the ACS's own weights the +#: 2024 1-year holds 70,123 Afghan-born non-citizens who arrived in 2021-2022, +#: fewer than the 73,566 OAW target. So, like the 2024 ASEC's code 28 +#: (2022-2024), the Afghan window also takes 2023-2024 arrivals. _PAROLE_ACS_MIN_ARRIVAL_YEAR: Mapping[str, int] = { "afghanistan": 2021, "ukraine": 2022, diff --git a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_stacked_authority_semantics.py b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_stacked_authority_semantics.py index a969470657..afff3a7396 100644 --- a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_stacked_authority_semantics.py +++ b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_stacked_authority_semantics.py @@ -70,7 +70,7 @@ def test_authority_projection_is_field_and_byte_identical_to_live_generation_zer assert projected == live assert stacked_identity_bytes(projected) == _canonical_bytes(live) assert projected["sha256"] == ( - "bf681aedc859fd6398ef72da68f584069b7d25b3085c549bf99f82e8324c4eef" + "7cdedd5bf056f1407910eddcd40ad0660bd7ef2d6defe224612c02b3476cbf15" ) assert { name: component["sha256"] for name, component in projected["components"].items() @@ -85,7 +85,7 @@ def test_authority_projection_is_field_and_byte_identical_to_live_generation_zer "cacc6c11e114dbae3aaa2761cc6b3fcb1191cd9b689b1c2bd096614c51ebff8b" ), "late_producer_schedule": ( - "80321bdcd6b2278da993ac6156b97b5b7532ed671e19caddc6cea19f1ace8816" + "bdc3ab40ac748822cea2712a517e6ec60b22674337b10bc0176a81809db07466" ), "metric_registry": ( "832245589a8ec5a33fc151e60662a245d411a636ef62debb884d65abcad3024a" diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py index 7bdfe818f5..00f22d1fde 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py @@ -25,7 +25,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # ssi_disability_criteria's attested bytes. # The opt-in cross-fitted gate calibration moves the fit kernels' bytes. assert first.spec_sha256 == ( - "7dd74e956915808940e675a456084981cf58369f7afd198cbd8bdec512b11285" + "b04b0843ef0926ae371c564c15450e2218b2dd3ef9ade72211fe92c7228985f6" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py index 2e9de4e544..aa63f89af1 100644 --- a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py @@ -380,7 +380,7 @@ def capture_equality(expected: object, actual: object) -> None: # through the bundle's source stages. # microcosm#767's humanitarian immigration controls, its attested # immigration kernel and its frozen legacy schedule move it. - "spec_sha256": "49d7b804829bdff5eb70ed16479a62e0dfb94de045f6c3c4374ea059ba634879", + "spec_sha256": "c414c985f6da4c8d623c79f15bce4c35a5fa32246fbfed941390b24fee41f429", }, } From 9d164367fef9497e8b1ade38b23c94fe0103156e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 10 Nov 2026 01:59:33 -0500 Subject: [PATCH 42/49] Regenerate the stage receipts with recorded draw keys; split the row changes The review of bad5cb667 found that every receipt reported this PR's draw key, although main keys its draws on the income year and PERIDNUM alone. The script now records each draw call's key. Regenerate the head, main and pre-sync receipts with it, and add receipt.main_full_keys.json: main's method run on this PR's key. The README credited the method with all the rows the two trees label differently. The key change alone moves about as many: on this PR's key, main's own method moves 2.400 million people from undocumented to LPR and 2.392 million back. Holding the key fixed, the method moves 0.319 million and 0.706 million. Rewrite the README to name both causes, with a row-by-row table for each. Refs #767 Co-Authored-By: Claude Opus 5.5 --- .../us-immigration-statuses-767/README.md | 125 +++++++++++++----- .../receipt.head.json | 101 +++++++++++++- .../receipt.main.json | 25 +++- .../receipt.main_full_keys.json | 120 +++++++++++++++++ .../receipt.pre_sync_42bb166b.json | 10 +- 5 files changed, 340 insertions(+), 41 deletions(-) create mode 100644 docs/evidence/us-immigration-statuses-767/receipt.main_full_keys.json diff --git a/docs/evidence/us-immigration-statuses-767/README.md b/docs/evidence/us-immigration-statuses-767/README.md index 1950854df2..4569d555d7 100644 --- a/docs/evidence/us-immigration-statuses-767/README.md +++ b/docs/evidence/us-immigration-statuses-767/README.md @@ -3,11 +3,11 @@ PR #779 draws humanitarian and temporary-protection statuses in the US immigration source stage (`us_runtime/immigration.py`). [`docs/us-release-build-rule.md`](../../us-release-build-rule.md) asks a pull -request that changes how the dataset is built to carry a from-scratch build from -its own tree before it merges. These receipts run that one stage alone, on the -inputs the next certified build uses, at this PR's head and at main. Max decides -whether they stand in for the from-scratch run (d2729, on the terms of -d2647). Nothing here changes a build. +request that changes how Microcosm builds the dataset to carry a from-scratch +build from its own tree before it merges. These receipts run that one stage +alone, on the inputs the next certified build uses, at this PR's head and at +main. Max decides whether they stand in for the from-scratch run (d2729, on the +terms of d2647). Nothing here changes a build. ## What the script runs @@ -22,27 +22,35 @@ the next certified build to that pool. The script: file to one third of income year 2024's weighted population, as `asec_pool.pool_asec_sources` does when `build_pooled_asec_unit_frame` pools for target year 2024; -- keys the draws by income year, `PH_SEQ` and `PERIDNUM`, the pool's lineage - columns; +- gives each person the pool's lineage columns, `source_year` (the income + year), `source_household_id` (`PH_SEQ`) and `source_person_id` (`PERIDNUM`), + and leaves the tree's kernel to choose which of them key its draws; +- records, for each draw call, the key that reproduces its draws; - runs at seed 0 and period 2024, the `--target-year` of `tools/route_a/route_a.sh`; - runs the composition gate on the output. -The receipts hold aggregates only. Each records the tree's commit and the -SHA-256 of its `immigration.py` and of the script. +`--draw-keys full` hands all three lineage columns to every draw call that +brings no key of its own, and `--draw-keys legacy` hands it `source_year` and +`source_person_id`. Two trees run on one key share their draws and differ only +in method. + +The receipts hold aggregates only. Each records the tree's commit, the SHA-256 +of its `immigration.py` and of the script, and every draw call's key. ## Receipts -| File | Tree | Result | -|---|---|---| -| `receipt.head.json` | this PR at `4b3b6a7a6`, its merge of main `764ae3aa7` | the gate passes with no failures | -| `receipt.main.json` | main at `764ae3aa7` | main draws no humanitarian status | -| `receipt.pre_sync_42bb166b.json` | this PR at `42bb166b1`, before it merged main `1acd884fe` | the stage refuses the 2026 ASEC's `PEINUSYR` code 29 | +| File | Tree | Draw keys | Result | +|---|---|---|---| +| `receipt.head.json` | this PR at `e3b2c8ce9`, which last merged main at `764ae3aa7` | its own: income year, `PH_SEQ`, `PERIDNUM` | the gate passes with no failures | +| `receipt.main.json` | main at `764ae3aa7` | its own: income year, `PERIDNUM` | main draws no humanitarian status | +| `receipt.main_full_keys.json` | main at `764ae3aa7` | this PR's, through `--draw-keys full` | main's method on this PR's draws | +| `receipt.pre_sync_42bb166b.json` | this PR at `42bb166b1`, before it merged main `1acd884fe` | none: the stage refuses before its first draw | the stage refuses the 2026 ASEC's `PEINUSYR` code 29 | -The script reran main's and the pre-sync tree from `git archive -packages`, so those two receipts take their commit from `--source-commit`. Their -`immigration_module_sha256` values equal the SHA-256 of `immigration.py` at -those commits. +The script reran main's tree and the pre-sync tree from `git archive +packages`, so those three receipts take their commit from `--source-commit`. +Their `immigration_module_sha256` values equal the SHA-256 of `immigration.py` +at those commits. ## Results @@ -72,13 +80,50 @@ Weighted persons, in millions: million control. - In both trees `ssn_card_type` is `NONE` exactly on the undocumented rows. -The totals move little, but the two trees label different rows undocumented. -Main's EAD spill counts a worker by prior-year wage or self-employment earnings -(`WSAL_VAL`, `SEMP_VAL`) and spills until the remaining undocumented rows match -the controls. This PR counts current labor-force status (`A_LFSR`, age 16 and -over) and spills until Pew's broad universe matches them. A spilled row takes an -EAD, and an EAD holder with no other mark is a lawful permanent resident. Row by -row (millions, main's label down the side, this PR's across): +## Why the two trees label different rows + +The totals move little, but the two trees label different rows. Two changes in +this PR move rows, and `receipt.main_full_keys.json` separates them: it runs +main's method on this PR's key. Comparing it with main's own run isolates the +key, and comparing it with this PR's run isolates the method. + +**The draw key.** Each draw hashes a string built from the row's lineage. Main +builds it from the income year and `PERIDNUM`. This PR adds `PH_SEQ`, so two +rows that share an income year and a `PERIDNUM` but not a household draw apart +where main's key would tie them. In these +three files the income year and `PERIDNUM` already tell all 421,119 rows apart, +so the added column neither ties nor separates any rows. It still changes every +string the draws hash, and so every row's draw. Main's EAD spill takes +candidates in draw order until it has spilled its excess, so new draws hand the +EADs to other rows. On this PR's key, main's own method moves 2.400 million +people from undocumented to LPR and 2.392 million back, and 0.220 million from +undocumented to DACA and 0.244 million back. No status total moves by more than +0.025 million. + +**The method.** This PR draws the humanitarian statuses after the legal-status +indicators and before the EAD spill. Refugee and asylee draws take only rows +that an indicator marks documented; parole and TPS draws may also take rows from +the residual pool, and those rows take an EAD. Main's spill counts a worker by +prior-year wage or self-employment earnings (`WSAL_VAL`, `SEMP_VAL`), spills +workers and then students, and stops when the workers or students left in the +residual pool match their controls. This PR's spill counts a worker by current +labor-force status (`A_LFSR` 1 to 4, age 16 and over), spills students and then +workers, and stops when the workers or students in Pew's broad unauthorized +universe match the controls. That universe still counts a DACA or Cuban/Haitian +cohort row after it takes an EAD, and every parole and TPS row. In both trees a +spilled row takes an EAD, and a documented non-citizen with no other mark is a +lawful permanent resident. With both runs on this PR's key, the method moves +0.319 million people from undocumented to LPR and 0.706 million back, and 0.031 +million from undocumented to DACA and 0.105 million back. The humanitarian +draws take 0.874 million of main's LPR rows and 0.700 million of its +undocumented rows. + +The two effects do not add, because both reshuffle one pool. Between LPR and +undocumented, the key alone moves 4.793 million people, the method alone 1.025 +million, and the two together 4.436 million. + +Row by row, in millions, with the first run's labels down the side and the +second run's across. Main against this PR, each on its own key: | Main \ this PR | LPR | Undocumented | DACA | Cuban/Haitian | Paroled | Refugee | Asylee | TPS | |---|---:|---:|---:|---:|---:|---:|---:|---:| @@ -87,9 +132,28 @@ row (millions, main's label down the side, this PR's across): | DACA | | 0.271 | 0.105 | | | | | | | Cuban/Haitian | | 0.146 | | 0.435 | | | | | -Citizens keep their label in both trees (300.244 million). The table comes from -both runs' `--labels-out` files, which stay out of the repository because they -are record-level; `scripts/xtab_labels.py` rebuilds it from them. +The key alone, main on its own key against main on this PR's key: + +| Main \ main on this PR's key | LPR | Undocumented | DACA | Cuban/Haitian | +|---|---:|---:|---:|---:| +| LPR | 9.288 | 2.392 | | | +| Undocumented | 2.400 | 10.314 | 0.220 | 0.136 | +| DACA | | 0.244 | 0.131 | | +| Cuban/Haitian | | 0.123 | | 0.458 | + +The method alone, main on this PR's key against this PR: + +| Main on this PR's key \ this PR | LPR | Undocumented | DACA | Cuban/Haitian | Paroled | Refugee | Asylee | TPS | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| LPR | 10.108 | 0.706 | | | 0.231 | 0.160 | 0.155 | 0.327 | +| Undocumented | 0.319 | 12.013 | 0.031 | 0.012 | 0.167 | | | 0.533 | +| DACA | | 0.105 | 0.246 | | | | | | +| Cuban/Haitian | | 0.067 | | 0.527 | | | | | + +Citizens keep their label in every run (300.244 million). The tables come from +the three runs' `--labels-out` files, which stay out of the repository because +they hold person records; `scripts/xtab_labels.py` rebuilds the tables from +them. ## Rerunning @@ -102,8 +166,9 @@ directories, then run: ``` For another commit, extract `git archive packages` into a directory and -pass that directory and `--source-commit `. Add `--labels-out .pkl` -to two runs, then rebuild the row-by-row table: +pass that directory and `--source-commit `; add `--draw-keys full` to +run that tree on this PR's key. Add `--labels-out .pkl` to two runs, then +rebuild a row-by-row table: ```bash .venv/bin/python -I docs/evidence/us-immigration-statuses-767/scripts/xtab_labels.py main.pkl head.pkl diff --git a/docs/evidence/us-immigration-statuses-767/receipt.head.json b/docs/evidence/us-immigration-statuses-767/receipt.head.json index 034e05c565..9f0ed9fbed 100644 --- a/docs/evidence/us-immigration-statuses-767/receipt.head.json +++ b/docs/evidence/us-immigration-statuses-767/receipt.head.json @@ -129,7 +129,98 @@ "weighted": 12891211.513123676 } }, - "draw_keys": "source_year:source_household_id(PH_SEQ):source_person_id(PERIDNUM)", + "draw_keys": { + "calls": [ + { + "forced": false, + "key": "full", + "rows": 421119, + "salt": "immigration:paroled_one_year:afghanistan" + }, + { + "forced": false, + "key": "full", + "rows": 421119, + "salt": "immigration:paroled_one_year:ukraine" + }, + { + "forced": false, + "key": "full", + "rows": 421119, + "salt": "immigration:paroled_one_year:nicaragua" + }, + { + "forced": false, + "key": "full", + "rows": 421119, + "salt": "immigration:paroled_one_year:venezuela" + }, + { + "forced": false, + "key": "full", + "rows": 421119, + "salt": "immigration:refugee" + }, + { + "forced": false, + "key": "full", + "rows": 421119, + "salt": "immigration:asylee" + }, + { + "forced": false, + "key": "full", + "rows": 421119, + "salt": "immigration:tps:venezuela" + }, + { + "forced": false, + "key": "full", + "rows": 421119, + "salt": "immigration:tps:el_salvador" + }, + { + "forced": false, + "key": "full", + "rows": 421119, + "salt": "immigration:tps:honduras" + }, + { + "forced": false, + "key": "full", + "rows": 421119, + "salt": "immigration:tps:nicaragua" + }, + { + "forced": false, + "key": "full", + "rows": 421119, + "salt": "immigration:tps:nepal" + }, + { + "forced": false, + "key": "full", + "rows": 421119, + "salt": "immigration:tps:other_designated" + }, + { + "forced": false, + "key": "full", + "rows": 421119, + "salt": "immigration:ead_students" + }, + { + "forced": false, + "key": "full", + "rows": 421119, + "salt": "immigration:ead_workers" + } + ], + "requested": "native", + "used": { + "full": "source_year:source_household_id(PH_SEQ):source_person_id(PERIDNUM)" + } + }, "error": null, "gate_failures": [], "gate_passed": true, @@ -263,7 +354,7 @@ "REFUGEE": 160000.0, "TPS": 857650.0 }, - "immigration_module_sha256": "fdb29c5cf362e1013d435222ef1b5cd0a082d6703f97e85d3b8835f2e8be1ab1", + "immigration_module_sha256": "c4d81501e63917595bae9cd976ab8068187d78cbb42d7146d5ecd1416209a4b8", "none_ssn_iff_undocumented": true, "pew_labor_force": 9699442.858246539, "pew_labor_force_control": 9700000.0, @@ -277,12 +368,12 @@ "TPS": 1.0028800890820582 }, "rows": 421119, - "script_sha256": "0c11b99f4c5de98e9339fe237feffe6510ee97333660006b62db07b3776c77cb", + "script_sha256": "6d1a8899979617dc3d53ec1d776a1825f0d55caf00abfe037e806aebbdc8d8eb", "seed": 0, - "source_commit": "4b3b6a7a60c35f0ff137bff74308488c7ae71014", + "source_commit": "e3b2c8ce9404d177a5424b2dd3b1acb6cf385e6f", "source_commit_from": "git", "source_tree_clean": true, - "stage_seconds": 5.9, + "stage_seconds": 8.3, "time_period": 2024, "weights": "HSUP_WGT * 0.01 by PH_SEQ, each file rescaled to one third of income year 2024's weighted person population (asec_pool.pool_asec_sources, anchor_year=2024)" } diff --git a/docs/evidence/us-immigration-statuses-767/receipt.main.json b/docs/evidence/us-immigration-statuses-767/receipt.main.json index 444a5fe7a9..e36fb6fee6 100644 --- a/docs/evidence/us-immigration-statuses-767/receipt.main.json +++ b/docs/evidence/us-immigration-statuses-767/receipt.main.json @@ -85,17 +85,36 @@ "weighted": 13070924.386860687 } }, - "draw_keys": "source_year:source_household_id(PH_SEQ):source_person_id(PERIDNUM)", + "draw_keys": { + "calls": [ + { + "forced": false, + "key": "legacy", + "rows": 421119, + "salt": "immigration:ead_workers" + }, + { + "forced": false, + "key": "legacy", + "rows": 421119, + "salt": "immigration:ead_students" + } + ], + "requested": "native", + "used": { + "legacy": "source_year:source_person_id(PERIDNUM)" + } + }, "error": null, "immigration_module_sha256": "8871df8b5f86f6bbf8541259a69d52b7db214560e5dacf5ad6db15268368f826", "none_ssn_iff_undocumented": true, "rows": 421119, - "script_sha256": "0c11b99f4c5de98e9339fe237feffe6510ee97333660006b62db07b3776c77cb", + "script_sha256": "6d1a8899979617dc3d53ec1d776a1825f0d55caf00abfe037e806aebbdc8d8eb", "seed": 0, "source_commit": "764ae3aa778cbc9030e286fabc3777e25c18f403", "source_commit_from": "argument", "source_tree_clean": null, - "stage_seconds": 2.9, + "stage_seconds": 1.1, "time_period": 2024, "weights": "HSUP_WGT * 0.01 by PH_SEQ, each file rescaled to one third of income year 2024's weighted person population (asec_pool.pool_asec_sources, anchor_year=2024)" } diff --git a/docs/evidence/us-immigration-statuses-767/receipt.main_full_keys.json b/docs/evidence/us-immigration-statuses-767/receipt.main_full_keys.json new file mode 100644 index 0000000000..8cf263c5c7 --- /dev/null +++ b/docs/evidence/us-immigration-statuses-767/receipt.main_full_keys.json @@ -0,0 +1,120 @@ +{ + "archives": [ + { + "archive": "asecpub24csv.zip", + "household_weighted_persons": 320890854.43000007, + "income_year": 2023, + "marsupwt_persons": 332382485.08, + "peinusyr_max": 28, + "rows": 144265, + "sha256": "cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09" + }, + { + "archive": "asecpub25csv.zip", + "household_weighted_persons": 325952369.06, + "income_year": 2024, + "marsupwt_persons": 337689641.57, + "peinusyr_max": 28, + "rows": 142125, + "sha256": "318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b" + }, + { + "archive": "asecpub26csv.zip", + "household_weighted_persons": 329518151.84999996, + "income_year": 2025, + "marsupwt_persons": 338554893.54, + "peinusyr_max": 29, + "rows": 134729, + "sha256": "fe819d0d2fc4470c282e76c247b8aa8b6054811619730307e9f2fe6186707d93" + } + ], + "composition": { + "CITIZEN": { + "by_income_year": { + "2023": 100026523.85662985, + "2024": 99788608.02, + "2025": 100429092.3485838 + }, + "code_29_weighted": 174138.23378113355, + "rows": 388829, + "under_19_weighted": 69797402.3449544, + "weighted": 300244224.22521365 + }, + "CUBAN_HAITIAN_ENTRANT": { + "by_income_year": { + "2023": 183475.60695258938, + "2024": 227568.58, + "2025": 183211.21581351623 + }, + "code_29_weighted": 24247.812392376745, + "rows": 688, + "under_19_weighted": 64589.79838310768, + "weighted": 594255.4027661056 + }, + "DACA": { + "by_income_year": { + "2023": 141697.3615097301, + "2024": 108160.53333333333, + "2025": 101408.83964565907 + }, + "code_29_weighted": 0.0, + "rows": 455, + "under_19_weighted": 3805.3054248002936, + "weighted": 351266.7344887225 + }, + "LEGAL_PERMANENT_RESIDENT": { + "by_income_year": { + "2023": 3925049.9863831, + "2024": 3972096.273333333, + "2025": 3790862.9623701023 + }, + "code_29_weighted": 387197.4977466433, + "rows": 14680, + "under_19_weighted": 1107430.6397713334, + "weighted": 11688009.222086536 + }, + "UNDOCUMENTED": { + "by_income_year": { + "2023": 4374042.875191389, + "2024": 4554356.279999999, + "2025": 4146214.3202536087 + }, + "code_29_weighted": 474123.816786065, + "rows": 16467, + "under_19_weighted": 1709420.9205708387, + "weighted": 13074613.475444999 + } + }, + "draw_keys": { + "calls": [ + { + "forced": true, + "key": "full", + "rows": 421119, + "salt": "immigration:ead_workers" + }, + { + "forced": true, + "key": "full", + "rows": 421119, + "salt": "immigration:ead_students" + } + ], + "requested": "full", + "used": { + "full": "source_year:source_household_id(PH_SEQ):source_person_id(PERIDNUM)" + } + }, + "error": null, + "immigration_module_sha256": "8871df8b5f86f6bbf8541259a69d52b7db214560e5dacf5ad6db15268368f826", + "none_ssn_iff_undocumented": true, + "rows": 421119, + "script_sha256": "6d1a8899979617dc3d53ec1d776a1825f0d55caf00abfe037e806aebbdc8d8eb", + "seed": 0, + "source_commit": "764ae3aa778cbc9030e286fabc3777e25c18f403", + "source_commit_from": "argument", + "source_tree_clean": null, + "stage_seconds": 1.2, + "time_period": 2024, + "weights": "HSUP_WGT * 0.01 by PH_SEQ, each file rescaled to one third of income year 2024's weighted person population (asec_pool.pool_asec_sources, anchor_year=2024)" +} diff --git a/docs/evidence/us-immigration-statuses-767/receipt.pre_sync_42bb166b.json b/docs/evidence/us-immigration-statuses-767/receipt.pre_sync_42bb166b.json index a0e256946e..984299b72d 100644 --- a/docs/evidence/us-immigration-statuses-767/receipt.pre_sync_42bb166b.json +++ b/docs/evidence/us-immigration-statuses-767/receipt.pre_sync_42bb166b.json @@ -28,15 +28,19 @@ "sha256": "fe819d0d2fc4470c282e76c247b8aa8b6054811619730307e9f2fe6186707d93" } ], - "draw_keys": "source_year:source_household_id(PH_SEQ):source_person_id(PERIDNUM)", + "draw_keys": { + "calls": [], + "requested": "native", + "used": {} + }, "error": "SourceRuntimeError: PEINUSYR carries value(s) outside the 2024 ASEC domain 0..28: [29.0].", "immigration_module_sha256": "df6699344c5fddbb5322baeb22f574be846e43c8c7600fd5dea798189c382042", - "script_sha256": "0c11b99f4c5de98e9339fe237feffe6510ee97333660006b62db07b3776c77cb", + "script_sha256": "6d1a8899979617dc3d53ec1d776a1825f0d55caf00abfe037e806aebbdc8d8eb", "seed": 0, "source_commit": "42bb166b1484df036809814434c1acf862b8cbb8", "source_commit_from": "argument", "source_tree_clean": null, - "stage_seconds": 0.2, + "stage_seconds": 0.1, "time_period": 2024, "weights": "HSUP_WGT * 0.01 by PH_SEQ, each file rescaled to one third of income year 2024's weighted person population (asec_pool.pool_asec_sources, anchor_year=2024)" } From 4f94e489aba5de4c33444742ac8333e736acba03 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 10 Nov 2026 09:12:43 -0500 Subject: [PATCH 43/49] Score the Medicaid probe in people; let short stacked draws take every candidate The delta review of e3b2c8ce9 and 9d164367f (wf_5bb29ab5-2eb) found that the post-export scorer cannot score the H.R.1 Medicaid probe. PolicyEngine-US prices each Medicaid enrollee through medicaid_slcsp_state_denominator, an aggregate over the whole file, which the household-batched scorer refuses and a reform engine refuses even in one batch (#956). Engine-US failed, and a release build would have aborted at the reform-coverage smoke. The probe now counts weighted people in medicaid_enrolled, with a 10,000-person floor. Probes carry a measure_unit, usd by default or persons; the smoke gate states effects and floors in it, and the manifest check refuses a budget measure that is not a live variable of that unit. medicaid_reporters.py counts the people in the restored statuses who report Medicaid at interview, from the head run's labels. The review also found that the receipt script said main's tree has no composition gate. Main has one, so the script now runs each tree's own gate. The changelog fragment no longer says schema-9 pools authenticate against the contract they sealed. The review found that the OAW kernel comment misstated the stacked transfer's rule: the reconciliation asks the ACS rows for each draw's target less the ASEC rows' labelled mass. Under that rule, U4U, Nicaraguan parole and Honduran TPS hold too few ACS candidates at the stacked ACS arm's weights, and the reconciliation refused any shortfall, so every stacked build would have stopped at U4U. A draw short of candidates now takes every one, as the ASEC stage's draws do, and its receipt records candidates_exhausted and candidate_shortfall_population. The reconciliation refuses only a draw that ends below 0.35 of its target, the composition band's floor, and the stacked validator replays the rule. The transfer contract's candidate_shortfall value changes, which moves its three identities. The Operation Allies Welcome window on exact-year rows now ends in 2022. stacked_supply.py projects each draw's ACS supply against its residual. The kernel edits move the attested identities: seed protocol and map digests, producer resource semantics, the stacked checkpoint base identity, the US spec (c414c985 -> 783ff9a4), the coverage evidence and the minimal-loader golden. The coverage manifest's ACA probe reasons now cite the default-pool receipt's TPS and humanitarian totals. Refs #767 Co-Authored-By: Claude Opus 5.5 --- ...nitarian-immigration-statuses-767.added.md | 2 +- docs/evidence/spec-engine/us-f0-coverage.json | 26 +- .../scripts/medicaid_reporters.py | 136 ++++++++ .../scripts/stacked_supply.py | 306 ++++++++++++++++++ .../scripts/stage_receipt_pooled.py | 58 ++-- docs/us-release-build-rule.md | 2 +- .../build/spec_engine/inventory_coverage.py | 8 +- .../us/release_input_coverage_manifest.json | 11 +- .../microcosm/build/us/spec/imputation.yaml | 2 +- .../build/us_runtime/acs_transfer.py | 2 +- .../microcosm/build/us_runtime/immigration.py | 95 ++++-- .../build/us_runtime/reform_coverage_smoke.py | 12 +- .../us_runtime/release_input_coverage.py | 63 +++- .../build/us_runtime/stacked_spine.py | 85 +++-- .../us/test_release_input_coverage.py | 69 ++++ .../shared/test_release_input_coverage.py | 83 +++++ .../shared/test_spec_engine_loader.py | 2 +- .../engine_free/us/test_us_acs_transfer.py | 103 ++++-- .../engine_free/us/test_us_immigration.py | 38 +++ .../us/test_us_immigration_properties.py | 167 ++++++++-- .../us/test_us_post_export_scoring.py | 5 +- .../engine_free/us/test_us_stacked_spine.py | 220 +++++++++++-- .../us/test_us_hr1_immigration_channels.py | 19 +- .../us/test_us_multispine_pool_tool.py | 2 +- .../us/test_us_post_export_scoring.py | 9 +- ...uild_us_release_input_coverage_manifest.py | 50 ++- 26 files changed, 1366 insertions(+), 209 deletions(-) create mode 100644 docs/evidence/us-immigration-statuses-767/scripts/medicaid_reporters.py create mode 100644 docs/evidence/us-immigration-statuses-767/scripts/stacked_supply.py diff --git a/changelog.d/us-humanitarian-immigration-statuses-767.added.md b/changelog.d/us-humanitarian-immigration-statuses-767.added.md index 2765828feb..d2717d8472 100644 --- a/changelog.d/us-humanitarian-immigration-statuses-767.added.md +++ b/changelog.d/us-humanitarian-immigration-statuses-767.added.md @@ -1 +1 @@ -Impute humanitarian and temporary-protection immigration statuses (REFUGEE, ASYLEE, PAROLED_ONE_YEAR per program origin, TPS per country) in the US immigration stage, drawn to cited DHS/OHSS/CBP/CRS stocks, so the enacted H.R.1 §71109/§71301/§71302 and SNAP §10108 eligibility channels stop computing $0; gate each category against its target and add four release-blocking reform-coverage probes. Attested schema-9 pools keep authenticating for scoring against the frozen contract they sealed. +Impute humanitarian and temporary-protection immigration statuses (REFUGEE, ASYLEE, PAROLED_ONE_YEAR per program origin, TPS per country) in the US immigration stage, drawn to cited DHS/OHSS/CBP/CRS stocks, so the enacted H.R.1 §71109/§71301/§71302 and SNAP §10108 eligibility channels stop computing $0; gate each category against its target and add four release-blocking reform-coverage probes. In a stacked build, a draw whose ACS candidates cannot cover its residual takes every candidate, as the ASEC stage's draws do, and its receipt records the shortfall; the transfer refuses only a draw that ends below 0.35 of its target; Operation Allies Welcome parole takes exact-year ACS arrivals from 2021 and 2022 only. Attested schema-9 legacy scoring takes the immigration stage's and its transfer group's contracts from a frozen schema-16 schedule; its other contracts follow today's registry, as on main. diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index 812fa53e2c..933c68d56a 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -1347,12 +1347,12 @@ ], "expected": { "producer_count": 38, - "sha256": "f794523ccebcbfa98197078a5bc9e7fdf4b1483d88decfab9363b00af3ff809e" + "sha256": "5653131d24487b1779e7c0a60deaed10d3a2a0b5a876c65dda40a6df0ef4e436" }, "failures": [], "observed": { "producer_count": 38, - "sha256": "f794523ccebcbfa98197078a5bc9e7fdf4b1483d88decfab9363b00af3ff809e" + "sha256": "5653131d24487b1779e7c0a60deaed10d3a2a0b5a876c65dda40a6df0ef4e436" }, "status": "covered" }, @@ -1655,13 +1655,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "cabadb3f2eeb49a1527c6bb4348d31e90b0a9ec8f836aa2cbe2b5d4ac6608798", - "protocol_sha256": "f0105714b03a3bef5cbc464519680e4fff3a9c2e758eece360a51cbfc656af8a" + "map_sha256": "4556389f3383a2d2600857cf7117b9306f8f45a9a6231ce2a8fe62bb6a5e388c", + "protocol_sha256": "0f4550f80efd0273589b533a4a453e74b06aa71b9ed1e08fdc016ef3d9d27fef" }, "failures": [], "observed": { - "map_sha256": "cabadb3f2eeb49a1527c6bb4348d31e90b0a9ec8f836aa2cbe2b5d4ac6608798", - "protocol_sha256": "f0105714b03a3bef5cbc464519680e4fff3a9c2e758eece360a51cbfc656af8a" + "map_sha256": "4556389f3383a2d2600857cf7117b9306f8f45a9a6231ce2a8fe62bb6a5e388c", + "protocol_sha256": "0f4550f80efd0273589b533a4a453e74b06aa71b9ed1e08fdc016ef3d9d27fef" }, "status": "covered" }, @@ -1676,7 +1676,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "f0105714b03a3bef5cbc464519680e4fff3a9c2e758eece360a51cbfc656af8a", + "implementation_sha256": "0f4550f80efd0273589b533a4a453e74b06aa71b9ed1e08fdc016ef3d9d27fef", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1697,7 +1697,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "f0105714b03a3bef5cbc464519680e4fff3a9c2e758eece360a51cbfc656af8a", + "implementation_sha256": "0f4550f80efd0273589b533a4a453e74b06aa71b9ed1e08fdc016ef3d9d27fef", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1886,7 +1886,7 @@ "alpha", "zeta" ], - "sha256": "d78292684ed4507dcc924e52e4eaff2b0bd826c1ddd2bc053696b0673fc9d8ce" + "sha256": "6aefc9bd351e4ca81998012de0eceacab9c802334d1ae2582f29cde4a6cd5036" }, "failures": [], "observed": { @@ -1910,7 +1910,7 @@ "alpha", "zeta" ], - "sha256": "d78292684ed4507dcc924e52e4eaff2b0bd826c1ddd2bc053696b0673fc9d8ce" + "sha256": "6aefc9bd351e4ca81998012de0eceacab9c802334d1ae2582f29cde4a6cd5036" }, "status": "covered" }, @@ -1974,7 +1974,7 @@ "take_up_contract", "us_qbi_reconciliation_contract" ], - "sha256": "a7dcbd23396f4d32e62ebef94e9968cf17856c8cf06d14c7e45fa903729864b8" + "sha256": "6e015f560238e801f4ef3263c05069ce4981078b620d260a2d37170e621e3beb" }, "status": "covered" }, @@ -2598,7 +2598,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "c414c985f6da4c8d623c79f15bce4c35a5fa32246fbfed941390b24fee41f429" + "spec_sha256": "783ff9a423a3de4db335c5886e37139ac94b0ae619528433e08a273c17bb111f" } }, "report_schema_version": 3, @@ -2608,7 +2608,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "c414c985f6da4c8d623c79f15bce4c35a5fa32246fbfed941390b24fee41f429" + "spec_sha256": "783ff9a423a3de4db335c5886e37139ac94b0ae619528433e08a273c17bb111f" }, "status": "pass" } diff --git a/docs/evidence/us-immigration-statuses-767/scripts/medicaid_reporters.py b/docs/evidence/us-immigration-statuses-767/scripts/medicaid_reporters.py new file mode 100644 index 0000000000..df7e952ba6 --- /dev/null +++ b/docs/evidence/us-immigration-statuses-767/scripts/medicaid_reporters.py @@ -0,0 +1,136 @@ +"""Count the people in the restored Medicaid statuses who report Medicaid. + +Usage: + python -I medicaid_reporters.py LABELS.pkl --archive-dir [--out OUT.json] + +LABELS.pkl is a ``--labels-out`` file from ``stage_receipt_pooled.py``. The +script reads ``NOW_MCAID`` (Medicaid at interview) from the three ASEC person +files that script runs on, after checking each archive against the same +SHA-256, and joins them to the labels on ``(source_year, PERIDNUM)`` as that +script keys them. It then sums the run's person weights, by status, over the +statuses that the reform-coverage probe +``hr1_medicaid_humanitarian_eligibility_restoration`` restores. +``microcosm.build.us_runtime.cps_carried`` maps ``NOW_MCAID`` to +``has_medicaid_health_coverage_at_interview``, the column the Medicaid take-up +stage anchors on. The output holds aggregates only, in millions of people. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import zipfile +from pathlib import Path + +import numpy as np +import pandas as pd + +# The archives and SHA-256 values of stage_receipt_pooled.py. +ARCHIVES = ( + ( + 2023, + "asecpub24csv.zip", + "pppub24.csv", + "cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09", + ), + ( + 2024, + "asecpub25csv.zip", + "pppub25.csv", + "318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b", + ), + ( + 2025, + "asecpub26csv.zip", + "pppub26.csv", + "fe819d0d2fc4470c282e76c247b8aa8b6054811619730307e9f2fe6186707d93", + ), +) +# The statuses the probe adds to the 2027 eligible list. +RESTORED = ( + "REFUGEE", + "ASYLEE", + "DEPORTATION_WITHHELD", + "PAROLED_ONE_YEAR", + "CONDITIONAL_ENTRANT", +) + +parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) +parser.add_argument("labels", type=Path) +parser.add_argument( + "--archive-dir", + type=Path, + action="append", + required=True, + help="directory holding ASEC zips (repeatable; each zip is found by name)", +) +parser.add_argument("--out", type=Path) +args = parser.parse_args() + + +def _archive_path(name: str) -> Path: + found = [d / name for d in args.archive_dir if (d / name).is_file()] + if not found: + raise SystemExit(f"{name} is in none of {args.archive_dir}") + return found[0] + + +labels = pd.read_pickle(args.labels) +key = labels["source_year"].astype(str) + ":" + labels["source_person_id"].astype(str) +if key.duplicated().any(): + raise SystemExit(f"{args.labels}: duplicate person keys") + +parts = [] +archives = [] +for income_year, archive_name, member, pinned in ARCHIVES: + path = _archive_path(archive_name) + digest = hashlib.sha256(path.read_bytes()).hexdigest() + if digest != pinned: + raise SystemExit(f"{path}: SHA-256 {digest} is not the pinned {pinned}") + with zipfile.ZipFile(path) as archive: + name = next(n for n in archive.namelist() if n.lower().endswith(member)) + with archive.open(name) as handle: + frame = pd.read_csv(handle, usecols=["PERIDNUM", "NOW_MCAID"]) + # stage_receipt_pooled.py reads PERIDNUM as an integer, then as text. + frame.index = str(income_year) + ":" + frame["PERIDNUM"].astype(str) + parts.append(frame["NOW_MCAID"]) + archives.append( + {"income_year": income_year, "archive": archive_name, "sha256": digest} + ) +now_mcaid = pd.concat(parts) +if now_mcaid.index.duplicated().any(): + raise SystemExit("the ASEC files repeat a (source_year, PERIDNUM) key") +if set(now_mcaid.index) != set(key): + raise SystemExit("the labels and the ASEC files hold different persons") + +reports = now_mcaid.reindex(key).to_numpy() == 1 +weight = labels["person_weight"].to_numpy(dtype=np.float64) +status = labels["immigration_status_str"].to_numpy() +rows = {} +for name in (*RESTORED, "all restored"): + mask = np.isin(status, RESTORED) if name == "all restored" else status == name + rows[name] = { + "rows": int(mask.sum()), + "persons_millions": round(float(weight[mask].sum()) / 1e6, 6), + "now_mcaid_persons_millions": round( + float(weight[mask & reports].sum()) / 1e6, 6 + ), + } + print( + f"{name:22s} {rows[name]['persons_millions']:7.3f}M people, " + f"{rows[name]['now_mcaid_persons_millions']:6.3f}M report Medicaid at " + f"interview ({rows[name]['rows']} rows)" + ) + +if args.out is not None: + receipt = { + "labels_file": args.labels.name, + "labels_sha256": hashlib.sha256(args.labels.read_bytes()).hexdigest(), + "labels_rows": int(len(labels)), + "script_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "archives": archives, + "column": "NOW_MCAID == 1", + "statuses": rows, + } + args.out.write_text(json.dumps(receipt, indent=1, sort_keys=True) + "\n") diff --git a/docs/evidence/us-immigration-statuses-767/scripts/stacked_supply.py b/docs/evidence/us-immigration-statuses-767/scripts/stacked_supply.py new file mode 100644 index 0000000000..d793025a6f --- /dev/null +++ b/docs/evidence/us-immigration-statuses-767/scripts/stacked_supply.py @@ -0,0 +1,306 @@ +"""Project each humanitarian draw's stacked ACS supply against its residual. + +Usage: + python -I stacked_supply.py RECEIPT.json --acs-dir --archive-dir + [--out OUT.json] + +A stacked (multispine) pool joins the ASEC pool and the ACS 2024 1-year, each +arm carrying half the ASEC pool's household mass +(``stacked_spine.DEFAULT_STACKED_HOUSEHOLD_MASS_SHARES``, anchored on the ASEC +channel by ``spine_assembly.assemble_spines``). The immigration stage labels +the ASEC rows at the full pool's person mass (``multispine_pool``'s +``person_weight_scale``); ``reconcile_us_immigration_humanitarian_transfer`` +then asks the ACS rows for each draw's residual: its target less the ASEC +arm's labelled mass. + +This script does not build a stacked pool. It projects one from aggregates: + +- the ASEC arm's labelled mass from RECEIPT.json's + ``humanitarian_draw_achieved`` (a ``stage_receipt_pooled.py`` run on the + default pool), scaled to the stacked weights; +- the ACS arm's supply from the ACS 2024 1-year PUMS (``csv_hus.zip`` and + ``csv_pus.zip`` in ``--acs-dir``, checked against the SHA-256 values in + ``us_runtime/acs_2024_1yr_sources.json``), weighted as ``acs_pums`` weights the + frame (WGTP of each occupied unit, PWGTP for a group-quarters person), with + each draw's candidates chosen by the kernel's own + ``_humanitarian_draw_candidates``. + +Every non-citizen counts as SSN-eligible, and no draw removes another's +candidates, so each supply is an upper bound. Refugee, asylee and +withholding draws take only the indicator-documented code the transfer +assigns, so their bounds are loose; parole and TPS draws take every +non-citizen code, so theirs are tight up to the order of the draws. + +Two lower bounds bracket each draw's share of its target: + +- ``asec_arm_share_of_target``: the ASEC arm's labelled mass alone. The + reconciliation never relabels those rows, so every draw reaches at least + this share. +- ``worst_case_share_of_target``: the ASEC arm plus the ACS supply left after + every earlier draw in manifest order takes as much of the draw's candidates + as it can, up to its own residual (a draw can pass its residual by the + weight of the one tie group that crosses it, which this bound leaves out; + ACS draws key on single persons). The bound holds for parole and TPS + draws, whose SSN rule this script reads exactly. A refugee, asylee or + withholding draw takes only rows the transfer imputes as + ``OTHER_NON_CITIZEN``, which PUMS cannot show, so its worst case is its + ASEC arm share. + +The output holds aggregates only. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import zipfile +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.build.us_runtime import immigration + +# The archives and SHA-256 values of stage_receipt_pooled.py. +ARCHIVES = ( + ( + 2023, + "asecpub24csv.zip", + "pppub24.csv", + "hhpub24.csv", + "cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09", + ), + ( + 2024, + "asecpub25csv.zip", + "pppub25.csv", + "hhpub25.csv", + "318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b", + ), + ( + 2025, + "asecpub26csv.zip", + "pppub26.csv", + "hhpub26.csv", + "fe819d0d2fc4470c282e76c247b8aa8b6054811619730307e9f2fe6186707d93", + ), +) +ANCHOR_INCOME_YEAR = 2024 +TIME_PERIOD = 2024 +ARM_SHARE = 0.5 + +parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) +parser.add_argument("receipt", type=Path) +parser.add_argument("--acs-dir", type=Path, required=True) +parser.add_argument( + "--archive-dir", + type=Path, + action="append", + required=True, + help="directory holding ASEC zips (repeatable; each zip is found by name)", +) +parser.add_argument("--out", type=Path) +args = parser.parse_args() + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _archive_path(name: str) -> Path: + found = [d / name for d in args.archive_dir if (d / name).is_file()] + if not found: + raise SystemExit(f"{name} is in none of {args.archive_dir}") + return found[0] + + +def _read_pums(path: Path, token: str, usecols: list[str]) -> pd.DataFrame: + parts = [] + with zipfile.ZipFile(path) as archive: + for name in sorted(archive.namelist()): + low = name.lower() + if low.endswith(".csv") and token in low: + with archive.open(name) as handle: + parts.append( + pd.read_csv(handle, usecols=usecols, dtype={"SERIALNO": str}) + ) + return pd.concat(parts, ignore_index=True) + + +# The ASEC pool's household and person masses, rescaled as the pool rescales. +asec_files = [] +for income_year, archive_name, member, household_member, pinned in ARCHIVES: + path = _archive_path(archive_name) + digest = _sha256(path) + if digest != pinned: + raise SystemExit(f"{path}: SHA-256 {digest} is not the pinned {pinned}") + with zipfile.ZipFile(path) as archive: + names = archive.namelist() + with archive.open(next(n for n in names if n.lower().endswith(member))) as f: + persons = pd.read_csv(f, usecols=["PH_SEQ"]) + with archive.open( + next(n for n in names if n.lower().endswith(household_member)) + ) as f: + households = pd.read_csv(f, usecols=["H_SEQ", "HSUP_WGT"]) + weight = households.drop_duplicates("H_SEQ").set_index("H_SEQ")["HSUP_WGT"] * 0.01 + person_mass = float(persons["PH_SEQ"].map(weight).sum()) + household_mass = float(weight.loc[persons["PH_SEQ"].unique()].sum()) + asec_files.append( + { + "income_year": income_year, + "archive": archive_name, + "sha256": digest, + "person_mass": person_mass, + "household_mass": household_mass, + } + ) +anchor_person_mass = next( + f["person_mass"] for f in asec_files if f["income_year"] == ANCHOR_INCOME_YEAR +) +pool_person_mass = 0.0 +pool_household_mass = 0.0 +for entry in asec_files: + scale = anchor_person_mass / len(ARCHIVES) / entry["person_mass"] + pool_person_mass += entry["person_mass"] * scale + pool_household_mass += entry["household_mass"] * scale + +# The ACS frame's weights: WGTP per occupied unit, PWGTP for a GQ person. +hus_path = args.acs_dir / "csv_hus.zip" +pus_path = args.acs_dir / "csv_pus.zip" +# The archives must be the ones the stacked pool pins. +acs_pins = json.loads( + (Path(immigration.__file__).parent / "acs_2024_1yr_sources.json").read_text() +) +acs_digests = {} +for artifact in acs_pins["artifacts"]: + path = args.acs_dir / artifact["filename"] + digest = _sha256(path) + if digest != artifact["sha256"]: + raise SystemExit(f"{path}: SHA-256 {digest} is not the pinned one") + acs_digests[artifact["filename"]] = digest +hus = _read_pums(hus_path, "hus", ["SERIALNO", "WGTP"]) +pus = _read_pums(pus_path, "pus", ["SERIALNO", "CIT", "POBP", "YOEP", "AGEP", "PWGTP"]) +unit_weight = pd.Series(hus["WGTP"].to_numpy(dtype=np.float64), index=hus["SERIALNO"]) +person_unit_weight = pus["SERIALNO"].map(unit_weight).to_numpy(dtype=np.float64) +gq = person_unit_weight == 0 +if pus.loc[gq, "SERIALNO"].duplicated().any(): + raise SystemExit("an ACS WGTP=0 unit holds more than one person") +acs_weight = np.where(gq, pus["PWGTP"].to_numpy(dtype=np.float64), person_unit_weight) +occupied = pus["SERIALNO"].drop_duplicates() +acs_household_mass = float(unit_weight.loc[occupied].clip(lower=0).sum()) + float( + acs_weight[gq].sum() +) +acs_person_mass = float(acs_weight.sum()) + +kappa = ARM_SHARE * pool_household_mass / acs_household_mass +asec_arm_person_mass = ARM_SHARE * pool_person_mass +acs_arm_person_mass = kappa * acs_person_mass +full_person_mass = asec_arm_person_mass + acs_arm_person_mass +# The stage labels ASEC rows at the full person mass; stacked ASEC rows carry +# asec_arm_person_mass of it. Candidate masses in the stage scale by rho. +rho = full_person_mass / pool_person_mass +to_stacked = asec_arm_person_mass / full_person_mass + +person = pd.DataFrame( + { + "CIT": pus["CIT"].to_numpy(dtype=np.float64), + "POBP": pus["POBP"].to_numpy(dtype=np.float64), + "YOEP": pus["YOEP"].to_numpy(dtype=np.float64), + "age": pus["AGEP"].to_numpy(dtype=np.float64), + } +) +profile = immigration._source_aware_immigration_profile(person, time_period=TIME_PERIOD) +ssn_codes = np.where(profile.is_citizen, 1, 3) + +receipt = json.loads(args.receipt.read_text()) +achieved = receipt["humanitarian_draw_achieved"] +draws = {} +# Each earlier draw's candidates and residual, in manifest order. +earlier: list[tuple[np.ndarray, float]] = [] +for draw in immigration.us_immigration_controls().humanitarian: + target = float(draw.target) + stage = float(achieved[draw.label]["population"]) + stage_exhausted = stage < target * (1 - 1e-9) + stage_mass = min(target, rho * stage) if stage_exhausted else min(target, stage) + immutable = stage_mass * to_stacked + residual = max(0.0, target - immutable) + candidates = immigration._humanitarian_draw_candidates( + draw, ssn_codes=ssn_codes, profile=profile + ) + acs_candidates = float(acs_weight[candidates].sum()) + supply = kappa * acs_candidates + # An earlier draw takes at most its residual, and at most its overlap with + # this draw's candidates. + removed = sum( + min(prior_residual, kappa * float(acs_weight[candidates & prior].sum())) + for prior, prior_residual in earlier + ) + earlier.append((candidates, residual)) + worst_supply = max(0.0, supply - removed) + exact_ssn_rule = draw.category in {"paroled_one_year", "tps"} + worst_reach = immutable + (min(worst_supply, residual) if exact_ssn_rule else 0.0) + draws[draw.label] = { + "target": target, + "asec_stage_population": stage, + "asec_arm_population": immutable, + "acs_residual": residual, + "acs_candidates_at_acs_weights": acs_candidates, + "acs_arm_supply_bound": supply, + "earlier_draws_worst_case_removal": removed, + "shortfall_at_bound": max(0.0, residual - supply), + "projected_share_of_target": ( + (immutable + min(supply, residual)) / target if target > 0 else None + ), + "asec_arm_share_of_target": immutable / target if target > 0 else None, + "worst_case_share_of_target": worst_reach / target if target > 0 else None, + } + print( + f"{draw.label:32s} target {target:9,.0f} ASEC arm {immutable:9,.0f} " + f"residual {residual:9,.0f} ACS bound {supply:9,.0f} " + f"short {max(0.0, residual - supply):9,.0f} " + f"worst-case share {worst_reach / target if target > 0 else float('nan'):.3f}" + ) +positive = [entry for entry in draws.values() if entry["target"] > 0] +lowest_worst_case = min(entry["worst_case_share_of_target"] for entry in positive) +print(f"lowest worst-case share of target: {lowest_worst_case:.4f}") + +# OAW before this change took exact-year Afghan arrivals from 2021 on. +afghan = ( + (~profile.is_citizen) + & (profile.birth_country == 200) + & profile.exact_arrival + & (profile.arrival_year >= 2021) +) +oaw_years = { + "2021-2022": float(acs_weight[afghan & (profile.arrival_year <= 2022)].sum()), + "2021-2024": float(acs_weight[afghan].sum()), +} +print( + f"kappa {kappa:.4f}; ACS Afghan non-citizen arrivals at ACS weights: " + + ", ".join(f"{k} {v:,.0f}" for k, v in oaw_years.items()) +) + +if args.out is not None: + out = { + "receipt_file": args.receipt.name, + "receipt_sha256": _sha256(args.receipt), + "script_sha256": _sha256(Path(__file__)), + "acs": { + **acs_digests, + "household_mass": acs_household_mass, + "person_mass": acs_person_mass, + }, + "asec_archives": asec_files, + "asec_pool_household_mass": pool_household_mass, + "asec_pool_person_mass": pool_person_mass, + "arm_household_share": ARM_SHARE, + "acs_arm_weight_scale": kappa, + "asec_arm_person_mass": asec_arm_person_mass, + "acs_arm_person_mass": acs_arm_person_mass, + "afghan_noncitizen_arrivals_at_acs_weights": oaw_years, + "composition_band_floor": immigration._HUMANITARIAN_TARGET_RELATIVE_BAND[0], + "lowest_worst_case_share_of_target": lowest_worst_case, + "draws": draws, + } + args.out.write_text(json.dumps(out, indent=1, sort_keys=True) + "\n") diff --git a/docs/evidence/us-immigration-statuses-767/scripts/stage_receipt_pooled.py b/docs/evidence/us-immigration-statuses-767/scripts/stage_receipt_pooled.py index 82ff98824f..78cc02be21 100644 --- a/docs/evidence/us-immigration-statuses-767/scripts/stage_receipt_pooled.py +++ b/docs/evidence/us-immigration-statuses-767/scripts/stage_receipt_pooled.py @@ -26,8 +26,8 @@ - Seed 0 and period 2024, Route A's ``--target-year``. A refusal is recorded as the receipt's ``error`` instead of raised. Main's tree -has no humanitarian controls, Pew universe or composition gate, so its receipt -carries the composition only. +has no humanitarian controls or Pew universe, so its receipt carries the +composition and its own composition gate's verdict only. The archives are Census's public-use ASEC CSV releases, at the URLs ``education_assistance_source.py`` names: @@ -379,8 +379,35 @@ def _git(*command: str) -> str | None: ), } ) - # This PR's controls, Pew universe and gate; main's tree has none of them. - if hasattr(immigration, "us_immigration_controls"): + # Every tree runs its own composition gate. This PR's tree also defines the + # humanitarian controls and the Pew universe; main's has neither. + gate_person = output.drop(columns=["person_weight"]).reset_index(drop=True).copy() + ids = np.arange(1, len(gate_person) + 1, dtype=np.int64) + for entity in US_SCHEMA.group_entities: + gate_person[US_SCHEMA.membership_column(entity)] = ids + frame = Frame( + { + "person": gate_person, + **{ + entity: pd.DataFrame({US_SCHEMA.id_column(entity): ids}) + for entity in US_SCHEMA.group_entities + }, + }, + US_SCHEMA, + {"household": Weights(weights, WeightKind.DESIGN)}, + ) + gate_parameters = inspect.signature( + immigration.us_immigration_composition_gate + ).parameters + gate_kwargs: dict[str, object] = {} + if "time_period" in gate_parameters: + gate_kwargs["time_period"] = PERIOD + if not hasattr(immigration, "us_immigration_controls"): + gate = immigration.us_immigration_composition_gate(frame, **gate_kwargs) + receipt.update( + {"gate_passed": bool(gate.passed), "gate_failures": list(gate.failures)} + ) + else: labor_force = np.isin(output["A_LFSR"].to_numpy(), [1, 2, 3, 4]) & (age >= 16) controls = immigration.us_immigration_controls() targets: dict[str, float] = {} @@ -389,25 +416,8 @@ def _git(*command: str) -> str | None: pew = np.isin(status, immigration._PEW_UNAUTHORIZED_STATUS_VALUES) | ( (status == "CUBAN_HAITIAN_ENTRANT") & (ssn == "NON_CITIZEN_VALID_EAD") ) - gate_person = ( - output.drop(columns=["person_weight"]).reset_index(drop=True).copy() - ) - ids = np.arange(1, len(gate_person) + 1, dtype=np.int64) - for entity in US_SCHEMA.group_entities: - gate_person[US_SCHEMA.membership_column(entity)] = ids - frame = Frame( - { - "person": gate_person, - **{ - entity: pd.DataFrame({US_SCHEMA.id_column(entity): ids}) - for entity in US_SCHEMA.group_entities - }, - }, - US_SCHEMA, - {"household": Weights(weights, WeightKind.DESIGN)}, - ) gate = immigration.us_immigration_composition_gate( - frame, time_period=PERIOD, controls=controls + frame, controls=controls, **gate_kwargs ) receipt.update( { @@ -467,11 +477,11 @@ def _git(*command: str) -> str | None: f" u19 {row['under_19_weighted'] / 1e6:.3f}M" ) print("NONE ssn iff UNDOCUMENTED:", receipt["none_ssn_iff_undocumented"]) -if "gate_passed" in receipt: +if "pew_universe" in receipt: print( f"Pew universe {receipt['pew_universe'] / 1e6:.3f}M vs anchor " f"{receipt['pew_universe_anchor'] / 1e6:.1f}M; labor force " f"{receipt['pew_labor_force'] / 1e6:.3f}M vs control " f"{receipt['pew_labor_force_control'] / 1e6:.1f}M" ) - print("gate passed:", receipt["gate_passed"], "failures:", receipt["gate_failures"]) +print("gate passed:", receipt["gate_passed"], "failures:", receipt["gate_failures"]) diff --git a/docs/us-release-build-rule.md b/docs/us-release-build-rule.md index 111d41d8d9..29e26b7c8f 100644 --- a/docs/us-release-build-rule.md +++ b/docs/us-release-build-rule.md @@ -254,7 +254,7 @@ known violations at run time: Tests pin the watched lists against the installed engine's formula sources, exercise each guard on a real engine, and compare the shipped baseline plans -(18 smoke keys, 75 validation keys and 1 demographics key) against whole-file +(23 smoke keys, 75 validation keys and 1 demographics key) against whole-file values on a small written H5 in three batches. On 2026-09-24, the sweep below completed on a written fixture containing eight diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index b56aedaddb..185ec9006a 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -363,12 +363,12 @@ "acs_person_predictors": "878c788a6f037d7aca12b3586ea034eff04f3034ffa11935a736493042551f25", "authority": "7cdedd5bf056f1407910eddcd40ad0660bd7ef2d6defe224612c02b3476cbf15", "early_families": "4aa9f736fd76e83955477ad1667e58f48f264783f05bdc7f0102cd32d61323bd", - "full_checkpoint": "d78292684ed4507dcc924e52e4eaff2b0bd826c1ddd2bc053696b0673fc9d8ce", + "full_checkpoint": "6aefc9bd351e4ca81998012de0eceacab9c802334d1ae2582f29cde4a6cd5036", "gap_fill_schedule": "1c31f9868f7884347cc19cf1ff65da43f950b9114941a715bab168246db414a7", "graph_nodes": "2606b852c3ecdc7ae0f7ff5b30c24a5ce6834b218db0898b44b18b906daa9558", "geography_assignment": "44e7f5f6cd7ceeef85d532c3d46306d03de4f4ee811376e6d29920742746b65c", "late_families": "7857518a5327a816acae80d55a04f43e7445a5e47be822d3ccb747bcb375e199", - "late_resource_semantics": "f794523ccebcbfa98197078a5bc9e7fdf4b1483d88decfab9363b00af3ff809e", + "late_resource_semantics": "5653131d24487b1779e7c0a60deaed10d3a2a0b5a876c65dda40a6df0ef4e436", "late_schedule": "46cea046d2fb8223d32772bd6a7807c52791640943949154982dcc52d93fa9eb", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "6655e37f15befe053f8de2583c216f97098e4c2bde526f315299c2858c97f055", @@ -396,8 +396,8 @@ # microcosm#767 moves the seed map, the protocol and the source manifest: # the attested immigration kernel gains the humanitarian status draws and # the immigration stage gains cited humanitarian stock targets. - "seed_map": "cabadb3f2eeb49a1527c6bb4348d31e90b0a9ec8f836aa2cbe2b5d4ac6608798", - "seed_protocol": "f0105714b03a3bef5cbc464519680e4fff3a9c2e758eece360a51cbfc656af8a", + "seed_map": "4556389f3383a2d2600857cf7117b9306f8f45a9a6231ce2a8fe62bb6a5e388c", + "seed_protocol": "0f4550f80efd0273589b533a4a453e74b06aa71b9ed1e08fdc016ef3d9d27fef", "source_manifest": "e9a70a60c3db9250066ee607d5e1fb07cd9d6a54424ab9eb9405f5e65b5a59b8", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", "tail": "565f0ceeadab69d3e96a886a00dd0305f7425fa3acde24d12ba48db2b37d5fc0", diff --git a/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json index 8b00da7688..99b4070aef 100644 --- a/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json @@ -1498,12 +1498,13 @@ "binding_inputs": [ "immigration_status_str" ], - "budget_measure": "medicaid", + "budget_measure": "medicaid_enrolled", "effect_direction": "reform_minus_baseline", "expected_sign": "positive", "id": "hr1_medicaid_humanitarian_eligibility_restoration", "issue": "PolicyEngine/microcosm#767", - "min_abs_effect": 50000000.0, + "measure_unit": "persons", + "min_abs_effect": 10000.0, "name": "H.R.1 SS71109 Medicaid humanitarian-status restoration", "parameter_changes": { "gov.hhs.medicaid.eligibility.eligible_immigration_statuses": { @@ -1520,7 +1521,7 @@ } }, "period": 2027, - "reason": "H.R.1 SS71109 narrows the Medicaid qualified-immigrant list to citizens, LPRs, and Cuban/Haitian entrants from 2026-10-01; the probe restores the pre-narrowing list at 2027 law, which binds only through the REFUGEE/ASYLEE/DEPORTATION_WITHHELD/PAROLED_ONE_YEAR values of immigration_status_str. The source stage draws roughly 160k refugees (74% reporting Medicaid on the 2024 ASEC), 155k asylees, and 405k parolees to cited DHS/OHSS stocks, so restoring their eligibility must re-enroll anchored takers and move person-level medicaid dollars by far more than the floor. A ~$0 score means the humanitarian statuses regressed to zero records (microcosm #767's silent-zero failure) or the engine channel broke." + "reason": "H.R.1 SS71109 narrows the Medicaid qualified-immigrant list to citizens, LPRs, and Cuban/Haitian entrants from 2026-10-01. The probe restores the pre-narrowing list at 2027 law, adding REFUGEE, ASYLEE, DEPORTATION_WITHHELD, PAROLED_ONE_YEAR and CONDITIONAL_ENTRANT; the source stage draws no conditional entrants, so the reform binds only through the first four values of immigration_status_str. The probe counts weighted people in medicaid_enrolled rather than summing medicaid dollars, because PolicyEngine-US prices each enrollee through medicaid_slcsp_state_denominator, an aggregate over the whole file that the household-batched post-export scorer refuses (PolicyEngine/microcosm#956). On the default-pool stage receipt (docs/evidence/us-immigration-statuses-767), 0.391 million of the 0.714 million weighted people in the restored statuses report Medicaid at interview, and the Medicaid take-up stage sets takes_up_medicaid_if_eligible for every reporter. The reform only widens the status list, so every reporter whom the narrowed list alone kept out enrolls. The 10,000-person floor is 2.6% of the reporters: a structural zero fails it, and the effect clears it unless Medicaid's other rules keep out more than 97% of the reporters. A ~0 count means the humanitarian statuses regressed to zero records (microcosm #767's silent-zero failure) or the engine channel broke." }, { "binding_inputs": [ @@ -1539,7 +1540,7 @@ } }, "period": 2026, - "reason": "H.R.1 SS71302 repeals the 26 USC 36B(c)(1)(B) below-FPL lawfully-present exception from 2026. Restoring it for 2026 binds, at 2026 law, only through tax units containing a non-citizen who is ACA-lawfully-present yet Medicaid-ineligible by status - on this file exactly the TPS population the source stage draws to the CRS RS20844 per-country stocks (865k weighted persons; the other humanitarian classes remain Medicaid-qualified at 2026 annual parameters). Below-poverty TPS units with marketplace take-up must regain premium tax credits above the floor; ~$0 means the TPS records vanished or the exception channel broke." + "reason": "H.R.1 SS71302 repeals the 26 USC 36B(c)(1)(B) below-FPL lawfully-present exception from 2026. Restoring it for 2026 binds, at 2026 law, only through tax units containing a non-citizen who is ACA-lawfully-present yet Medicaid-ineligible by status - on this file exactly the TPS population the source stage draws to the CRS RS20844 per-country stocks (0.860 million weighted persons on the default-pool stage receipt, docs/evidence/us-immigration-statuses-767; the other humanitarian classes remain Medicaid-qualified at 2026 annual parameters). Below-poverty TPS units with marketplace take-up must regain premium tax credits above the floor; ~$0 means the TPS records vanished or the exception channel broke." }, { "binding_inputs": [ @@ -1561,7 +1562,7 @@ } }, "period": 2027, - "reason": "H.R.1 SS71301 adds TPS/REFUGEE/ASYLEE/DEPORTATION_WITHHELD/PAROLED_ONE_YEAR to the ACA ineligible-status list from 2027-01-01. The probe restores the pre-H.R.1 list (DACA and UNDOCUMENTED only) at 2027, re-qualifying every humanitarian status the source stage draws (~1.6M weighted persons across parole, refugee, asylee, and TPS). Units among them with marketplace take-up must regain premium tax credits above the floor; ~$0 means the humanitarian statuses regressed or the lawful-presence channel broke." + "reason": "H.R.1 SS71301 adds TPS/REFUGEE/ASYLEE/DEPORTATION_WITHHELD/PAROLED_ONE_YEAR to the ACA ineligible-status list from 2027-01-01. The probe restores the pre-H.R.1 list (DACA and UNDOCUMENTED only) at 2027, re-qualifying every humanitarian status the source stage draws (1.574 million weighted persons across parole, refugee, asylee and TPS on the default-pool stage receipt). Units among them with marketplace take-up must regain premium tax credits above the floor; ~$0 means the humanitarian statuses regressed or the lawful-presence channel broke." }, { "binding_inputs": [ diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/imputation.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/imputation.yaml index a570727836..fecc5fe9d6 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/imputation.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/imputation.yaml @@ -474,7 +474,7 @@ transfer_execution: mutable_rows: newly_imputed_paired_cells_only target_basis: manifest_residual_after_immutable_asec_mass selection: stable_person_hash_in_manifest_draw_order - candidate_shortfall: error + candidate_shortfall: take_every_candidate_error_below_composition_band_floor schedule_d_capital_gain_distributions: activation: derive_schedule_d: true diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_transfer.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_transfer.py index 58e3326c98..d7ffca31e9 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_transfer.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_transfer.py @@ -526,7 +526,7 @@ def acs_transfer_execution_contract_identity( "mutable_rows": "newly_imputed_paired_cells_only", "target_basis": "manifest_residual_after_immutable_asec_mass", "selection": "stable_person_hash_in_manifest_draw_order", - "candidate_shortfall": "error", + "candidate_shortfall": "take_every_candidate_error_below_composition_band_floor", }, "passive_partnership_s_corp": { "enabled": passive_enabled, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py b/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py index dd646b627c..62d806e833 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/immigration.py @@ -343,19 +343,21 @@ class is the better statutory label and keeps H.R.1 eligibility), as is "nicaragua": _POST_2021_ARRIVAL_CODES, "venezuela": _POST_2021_ARRIVAL_CODES, } -#: First exact (ACS YOEP) arrival year of each parole program. The windows -#: have no end year, although Operation Allies Welcome ended in 2022: the -#: stacked transfer asks the ACS arm for its own mass share of each national -#: target and refuses a candidate shortfall, and at the ACS's own weights the -#: 2024 1-year holds 70,123 Afghan-born non-citizens who arrived in 2021-2022, -#: fewer than the 73,566 OAW target. So, like the 2024 ASEC's code 28 -#: (2022-2024), the Afghan window also takes 2023-2024 arrivals. +#: First exact (ACS YOEP) arrival year of each parole program. _PAROLE_ACS_MIN_ARRIVAL_YEAR: Mapping[str, int] = { "afghanistan": 2021, "ukraine": 2022, "nicaragua": 2023, "venezuela": 2022, } +#: Last exact (ACS YOEP) arrival year of each parole program that ended +#: before the period. Operation Allies Welcome paroled Afghans who arrived in +#: 2021 and 2022, so exact-year rows take only those arrivals. ASEC rows keep +#: their coarser codes: the 2024 ASEC's code 28 spans 2022-2024. A program +#: missing here has no last year. +_PAROLE_ACS_MAX_ARRIVAL_YEAR: Mapping[str, int] = { + "afghanistan": 2022, +} #: Asylum grants lag arrival by filing queues plus court backlog; grantees #: in 2022-2024 overwhelmingly arrived 2016+ (OHSS asylee flow reports). The #: 2026 ASEC's code 29 (2024-2026) is left out: its arrivals come after all @@ -484,7 +486,8 @@ class is the better statutory label and keeps H.R.1 eligibility), as is #: The draw forces the target when candidates suffice, so the band only #: bites on pool exhaustion (the ASEC undercovers 2022-24 arrivals — the #: Ukraine parole pool saturates below its admin count) or collapse. A -#: category with an explicit zero target must emit exactly zero. +#: category with an explicit zero target must emit exactly zero. The stacked +#: transfer's reconciliation refuses an exhausted draw below the band's floor. _HUMANITARIAN_TARGET_RELATIVE_BAND = (0.35, 1.5) @@ -1453,6 +1456,10 @@ def _humanitarian_draw_candidates( acs_window = profile.exact_arrival & ( profile.arrival_year >= _PAROLE_ACS_MIN_ARRIVAL_YEAR[origin] ) + if origin in _PAROLE_ACS_MAX_ARRIVAL_YEAR: + acs_window &= ( + profile.arrival_year <= _PAROLE_ACS_MAX_ARRIVAL_YEAR[origin] + ) cohort |= origin_match & (asec_window | acs_window) return noncitizen & np.isin(ssn_codes, (0, 3)) & cohort & ~excluded if draw.category == "refugee": @@ -1695,11 +1702,16 @@ def reconcile_us_immigration_humanitarian_transfer( ) -> tuple[pd.DataFrame, dict[str, object]]: """Reconcile a transferred immigration pair to manifest draw targets. - Immutable rows are the ASEC source contribution. Only rows whose paired - target cells were imputed may be repaired or selected. Each manifest draw - receives the residual of its national target after its compatible, - immutable contribution, in manifest order; candidate exhaustion is a hard - error rather than permission to ship a wrong-origin label. + Immutable rows carry the ASEC source contribution; the reconciliation + repairs or selects only rows whose paired target cells the transfer + imputed. In manifest order, each draw asks those rows for the residual of + its national target after its compatible, immutable contribution, and + only rows its origin, arrival and SSN rules admit can take its label. + When the admitted rows hold less weight than the residual, the draw takes + every one of them, as the ASEC stage's own draws do, and its receipt + records the shortfall. The reconciliation refuses only a draw that then + reaches less than the floor of the composition gate's band + (``_HUMANITARIAN_TARGET_RELATIVE_BAND``) of its target. """ result = person.copy() @@ -1817,6 +1829,7 @@ def reconcile_us_immigration_humanitarian_transfer( draw_receipts: dict[str, object] = {} largest_target = max((draw.target for draw in controls.humanitarian), default=0.0) tolerance = max(1e-6, largest_target * 1e-12) + band_floor = _HUMANITARIAN_TARGET_RELATIVE_BAND[0] for draw in controls.humanitarian: immutable_draw = ( _humanitarian_emitted_mask( @@ -1848,13 +1861,8 @@ def reconcile_us_immigration_humanitarian_transfer( & ~selected_once ) available_population = float(weights[candidates].sum()) - if available_population + tolerance < residual_target: - raise ValueError( - "Humanitarian transfer candidate shortfall for " - f"{draw.label!r}: residual_target={residual_target:,.6f}, " - f"eligible_recipient_population={available_population:,.6f}, " - f"immutable_population={immutable_population:,.6f}." - ) + # Short of candidates, the selection below takes every one of them. + exhausted = available_population + tolerance < residual_target stable_draws = _stable_person_draws( result, seed=seed, @@ -1866,8 +1874,13 @@ def reconcile_us_immigration_humanitarian_transfer( stable_draws, residual_target, ) + if exhausted and not np.array_equal(selected, candidates): + raise RuntimeError( + f"Humanitarian transfer selection for {draw.label!r} left " + "candidates unselected although they hold less than its residual." + ) selected_population = float(weights[selected].sum()) - if selected_population + tolerance < residual_target: + if not exhausted and selected_population + tolerance < residual_target: raise RuntimeError( f"Humanitarian transfer selection underfilled {draw.label!r}." ) @@ -1884,21 +1897,46 @@ def reconcile_us_immigration_humanitarian_transfer( ) compatible_humanitarian |= achieved_mask achieved_population = float(weights[achieved_mask].sum()) + # An exhausted draw crosses no threshold: it takes every candidate. threshold_tie_population = 0.0 - if selected.any(): + if selected.any() and not exhausted: threshold = float(np.max(stable_draws[selected])) threshold_tie_population = float( weights[candidates & (stable_draws == threshold)].sum() ) residual_selection_error = abs(selected_population - residual_target) - if residual_selection_error > threshold_tie_population + tolerance: + if ( + not exhausted + and residual_selection_error > threshold_tie_population + tolerance + ): raise RuntimeError( f"Humanitarian transfer residual selection for {draw.label!r} " "missed its target by more than the threshold tie mass." ) + candidate_shortfall = ( + residual_target - available_population if exhausted else 0.0 + ) + if exhausted and achieved_population + tolerance < band_floor * float( + draw.target + ): + raise ValueError( + "Humanitarian transfer candidate shortfall for " + f"{draw.label!r} leaves it below {band_floor} of its target, " + "the floor of the composition gate's band: " + f"target={float(draw.target):,.6f}, " + f"achieved_population={achieved_population:,.6f}, " + f"residual_target={residual_target:,.6f}, " + f"eligible_recipient_population={available_population:,.6f}, " + f"immutable_population={immutable_population:,.6f}." + ) absolute_error = abs(achieved_population - float(draw.target)) immutable_overshoot = max(0.0, immutable_population - float(draw.target)) discrete_bound = immutable_overshoot + threshold_tie_population + tolerance + if not exhausted and absolute_error > discrete_bound: + raise RuntimeError( + f"Humanitarian transfer reconciliation for {draw.label!r} " + "missed its target by more than the discrete selection bound." + ) draw_receipts[draw.label] = { "target": float(draw.target), "immutable_population": immutable_population, @@ -1909,15 +1947,12 @@ def reconcile_us_immigration_humanitarian_transfer( "absolute_error": absolute_error, "selection_threshold_tie_population": threshold_tie_population, "residual_selection_error": residual_selection_error, - "within_residual_discrete_weight_bound": True, + "within_residual_discrete_weight_bound": not exhausted, "immutable_overshoot": immutable_overshoot, - "within_discrete_weight_bound": absolute_error <= discrete_bound, + "within_discrete_weight_bound": not exhausted, + "candidates_exhausted": exhausted, + "candidate_shortfall_population": candidate_shortfall, } - if absolute_error > discrete_bound: - raise RuntimeError( - f"Humanitarian transfer reconciliation for {draw.label!r} " - "missed its target by more than the discrete selection bound." - ) result["ssn_card_type"] = ssn result["immigration_status_str"] = status diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/reform_coverage_smoke.py b/packages/microcosm-build/src/microcosm/build/us_runtime/reform_coverage_smoke.py index 8c8efce063..f377554f43 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/reform_coverage_smoke.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/reform_coverage_smoke.py @@ -132,6 +132,7 @@ def us_reform_coverage_smoke_gate( "effect": effect, "expected_sign": probe.expected_sign, "min_abs_effect": probe.min_abs_effect, + "measure_unit": probe.measure_unit, "binding_inputs": list(probe.binding_inputs), "issue": probe.issue, "passed": passed, @@ -142,11 +143,16 @@ def us_reform_coverage_smoke_gate( if probe.expected_sign == "either" else f"a {probe.expected_sign} effect" ) + if probe.measure_unit == "persons": + effect_text = f"{effect:+,.0f} persons" + floor_text = f"{probe.min_abs_effect:,.0f} persons" + else: + effect_text = f"{effect:+,.0f}" + floor_text = f"${probe.min_abs_effect:,.0f}" scored = ( - f"{probe.id}: '{probe.name}' scores {effect:+,.0f} on " + f"{probe.id}: '{probe.name}' scores {effect_text} on " f"{probe.budget_measure} for {probe_period}; expected " - f"{expectation} with magnitude at least " - f"${probe.min_abs_effect:,.0f}." + f"{expectation} with magnitude at least {floor_text}." ) # A wrong-signed effect that clears the floor proves the reform # binds, so the inputs are carried; what disagrees is the probe's diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/release_input_coverage.py b/packages/microcosm-build/src/microcosm/build/us_runtime/release_input_coverage.py index 00f88d9a7a..be1455a073 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/release_input_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/release_input_coverage.py @@ -339,9 +339,16 @@ def __post_init__(self) -> None: ) +#: What a reform-coverage probe's weighted budget-measure total counts. +#: ``usd`` measures carry PolicyEngine-US's ``currency-USD`` unit; +#: ``persons`` measures are person-level booleans, whose weighted sum +#: counts people. +REFORM_COVERAGE_MEASURE_UNITS = frozenset({"usd", "persons"}) + + @dataclass(frozen=True) class ReformCoverageProbe: - """A pinned reform whose $0 score on the export signals a coverage hole. + """A pinned reform whose zero score on the export signals a coverage hole. A reform that mechanically binds through named input leaves must move its budget measure. If it scores ~$0 on the release, the leaves it binds through @@ -357,6 +364,12 @@ class ReformCoverageProbe: neutralized_variable: Input leaf neutralized by a structural reform when no parameter change isolates the leaf. budget_measure: The variable whose weighted total change is scored. + measure_unit: What the budget measure's weighted total counts: + ``"usd"`` (dollars, the default) or ``"persons"`` (people, + for a person-level boolean measure). The smoke gate states + ``min_abs_effect`` in this unit, and the engine half of + :func:`assert_release_input_coverage_manifest_current` checks + it against the measure's PolicyEngine-US metadata. effect_direction: ``"reform_minus_baseline"`` or ``"baseline_minus_reform"``. period: Optional probe-specific scoring year. OBBBA probes use 2026 @@ -384,6 +397,7 @@ class ReformCoverageProbe: period: int | None = None expected_sign: str = "positive" neutralized_variable: str | None = None + measure_unit: str = "usd" def __post_init__(self) -> None: if not self.id: @@ -420,6 +434,11 @@ def __post_init__(self) -> None: raise ValueError( f"{self.id}: expected_sign must be 'positive', 'negative', or 'either'." ) + if self.measure_unit not in REFORM_COVERAGE_MEASURE_UNITS: + raise ValueError( + f"{self.id}: measure_unit must be one of " + f"{sorted(REFORM_COVERAGE_MEASURE_UNITS)}." + ) @dataclass(frozen=True) @@ -564,6 +583,7 @@ def load_release_input_coverage_manifest( if raw_probe.get("neutralized_variable") is None else str(raw_probe["neutralized_variable"]) ), + measure_unit=str(raw_probe.get("measure_unit", "usd")), ) ) @@ -718,6 +738,37 @@ def _coverage_engine() -> Any | None: return None +def _probe_measure_drift(engine: Any, probe: ReformCoverageProbe) -> list[str]: + """Why ``probe``'s budget measure disagrees with its ``measure_unit``. + + Empty when the measure is a live PolicyEngine-US variable of the declared + unit: a ``currency-USD`` variable for ``usd``, a person-level boolean for + ``persons``. + """ + try: + metadata = engine.variable_metadata(probe.budget_measure) + unit = engine.variable_unit(probe.budget_measure) + except ValueError: + return [ + f"probe {probe.id!r}: budget_measure {probe.budget_measure!r} is " + "not a PolicyEngine-US variable." + ] + if probe.measure_unit == "persons": + if (metadata.entity, metadata.dtype) != ("person", "bool"): + return [ + f"probe {probe.id!r}: measure_unit 'persons' counts people, so " + "its budget_measure must be a person-level boolean; " + f"{probe.budget_measure!r} is a {metadata.dtype} on " + f"{metadata.entity}." + ] + elif unit != "currency-USD": + return [ + f"probe {probe.id!r}: measure_unit 'usd' needs a dollar " + f"budget_measure; {probe.budget_measure!r} declares unit {unit!r}." + ] + return [] + + def _ecps_populated_layers() -> frozenset[str]: payload = _resource_payload(_ECPS_PARITY_REFERENCE_RESOURCE) shares = payload.get("nonzero_shares") @@ -803,9 +854,12 @@ def assert_release_input_coverage_manifest_current( given) under either its live or its historical spelling: the parity gate honours both, so a restored input filed as a gap would be exempted there even while this manifest still requires it. - - Every declared column must be a real PolicyEngine-US input leaf, and every - probe's ``binding_inputs`` / ``budget_measure`` must resolve on the live - engine, so the contract cannot guard names the engine no longer has. + - Every declared column must be a real PolicyEngine-US input leaf, every + probe's ``binding_inputs`` must be live input leaves, and every probe's + ``budget_measure`` must be a live variable in its ``measure_unit`` + (``currency-USD`` for ``usd``, a person-level boolean for + ``persons``), so the contract cannot guard names the engine no longer + has or state a floor in the wrong unit. A no-op for the engine-graph half when no engine is available (the workspace test environment); the checked-in-facts half always runs. @@ -921,6 +975,7 @@ def assert_release_input_coverage_manifest_current( f"probe {probe.id!r}: binding_inputs are not " f"PolicyEngine-US input leaves: {bad_inputs}." ) + failures.extend(_probe_measure_drift(engine, probe)) if failures: raise ValueError( diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py b/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py index 156e0d7b84..fe7968c3fb 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py @@ -101,6 +101,7 @@ transfer_acs_inputs, ) from microcosm.build.us_runtime.immigration import ( + _HUMANITARIAN_TARGET_RELATIVE_BAND, us_immigration_controls, us_immigration_humanitarian_draw_mask, us_immigration_humanitarian_transfer_selection_masks, @@ -4203,6 +4204,15 @@ def _validate_post_transfer_live_output_binding( "within_residual_discrete_weight_bound", "immutable_overshoot", "within_discrete_weight_bound", + "candidates_exhausted", + "candidate_shortfall_population", + } +) +_IMMIGRATION_RECONCILIATION_BOOLEAN_DRAW_FIELDS = frozenset( + { + "within_residual_discrete_weight_bound", + "within_discrete_weight_bound", + "candidates_exhausted", } ) @@ -4428,10 +4438,11 @@ def _validate_immigration_post_transfer_reconciliation( raise ValueError( f"{boundary}: immigration reconciliation draw order is non-canonical." ) - numeric_fields = _IMMIGRATION_RECONCILIATION_DRAW_FIELDS - { - "within_residual_discrete_weight_bound", - "within_discrete_weight_bound", - } + numeric_fields = ( + _IMMIGRATION_RECONCILIATION_DRAW_FIELDS + - _IMMIGRATION_RECONCILIATION_BOOLEAN_DRAW_FIELDS + ) + band_floor = _HUMANITARIAN_TARGET_RELATIVE_BAND[0] for label in expected_order: draw = draws[label] if not isinstance(draw, Mapping) or set(draw) != ( @@ -4441,17 +4452,27 @@ def _validate_immigration_post_transfer_reconciliation( f"{boundary}: immigration reconciliation draw {label!r} schema " "is invalid." ) - if any( - not isinstance(draw.get(field), (int, float)) - or isinstance(draw[field], bool) - or not np.isfinite(float(draw[field])) - or draw[field] < 0 - for field in numeric_fields - ) or any( - draw.get(field) is not True - for field in ( - "within_residual_discrete_weight_bound", - "within_discrete_weight_bound", + # A draw meets both selection bounds exactly when its candidates hold + # its residual; an exhausted draw takes every candidate instead. + exhausted_flag = draw.get("candidates_exhausted") + if ( + any( + not isinstance(draw.get(field), (int, float)) + or isinstance(draw[field], bool) + or not np.isfinite(float(draw[field])) + or draw[field] < 0 + for field in numeric_fields + ) + or any( + not isinstance(draw.get(field), bool) + for field in _IMMIGRATION_RECONCILIATION_BOOLEAN_DRAW_FIELDS + ) + or any( + draw[field] is exhausted_flag + for field in ( + "within_residual_discrete_weight_bound", + "within_discrete_weight_bound", + ) ) ): raise ValueError( @@ -4494,12 +4515,31 @@ def _validate_immigration_post_transfer_reconciliation( f"{boundary}: immigration reconciliation draw {label!r} has " "immutable mass against an explicit-zero target." ) - if eligible + tolerance < residual: + exhausted = eligible + tolerance < residual + if exhausted and not exhausted_flag: raise ValueError( f"{boundary}: immigration reconciliation draw {label!r} " "conceals a recipient candidate shortfall." ) - if selected + tolerance < residual: + if exhausted_flag and not exhausted: + raise ValueError( + f"{boundary}: immigration reconciliation draw {label!r} " + "claims a candidate shortfall its eligible population covers." + ) + shortfall = residual - eligible if exhausted else 0.0 + if exhausted: + # Every candidate is taken; the draw keeps the gate's floor. + if abs(selected - eligible) > tolerance or tie > tolerance: + raise ValueError( + f"{boundary}: immigration reconciliation draw {label!r} " + "leaves candidates unselected after exhausting them." + ) + if achieved + tolerance < band_floor * target: + raise ValueError( + f"{boundary}: immigration reconciliation draw {label!r} " + "falls below the floor of the composition gate's band." + ) + elif selected + tolerance < residual: raise ValueError( f"{boundary}: immigration reconciliation draw {label!r} " "underfills its residual target." @@ -4511,10 +4551,17 @@ def _validate_immigration_post_transfer_reconciliation( or tie > eligible + tolerance or abs(achieved - (immutable + selected)) > tolerance or abs(float(draw["residual_selection_error"]) - residual_error) > tolerance - or residual_error > tie + tolerance + or abs(float(draw["candidate_shortfall_population"]) - shortfall) + > tolerance or abs(float(draw["absolute_error"]) - absolute_error) > tolerance or abs(float(draw["immutable_overshoot"]) - immutable_overshoot) > tolerance - or absolute_error > immutable_overshoot + tie + tolerance + or ( + not exhausted + and ( + residual_error > tie + tolerance + or absolute_error > immutable_overshoot + tie + tolerance + ) + ) ): raise ValueError( f"{boundary}: immigration reconciliation draw {label!r} " diff --git a/packages/microcosm-build/tests/engine_contract/us/test_release_input_coverage.py b/packages/microcosm-build/tests/engine_contract/us/test_release_input_coverage.py index 29591bbc3a..e1431a05e5 100644 --- a/packages/microcosm-build/tests/engine_contract/us/test_release_input_coverage.py +++ b/packages/microcosm-build/tests/engine_contract/us/test_release_input_coverage.py @@ -1,6 +1,8 @@ """Tests split from packages/microcosm-build/tests/test_release_input_coverage.py.""" # ruff: noqa: F403, F405 +import dataclasses + from test_support.microcosm_build.release_input_coverage import * @@ -440,3 +442,70 @@ def test_real_engine_treats_receipt_inputs_as_false_default_input_leaves( ) assert not failing.passed assert failing.details["degenerate_required"] == sorted(names) + + +_HR1_MEDICAID_PROBE = "hr1_medicaid_humanitarian_eligibility_restoration" + + +@pytest.fixture(scope="module") +def live_engine(): + from microcosm.frame.adapters.policyengine_us import PolicyEngineUSEngine + + return PolicyEngineUSEngine() + + +def _with_probe_change(probe_id: str, **changes) -> ReleaseInputCoverageManifest: + manifest = load_release_input_coverage_manifest() + return ReleaseInputCoverageManifest( + reference=manifest.reference, + columns=manifest.columns, + probes=tuple( + dataclasses.replace(probe, **changes) if probe.id == probe_id else probe + for probe in manifest.probes + ), + schema_version=manifest.schema_version, + ) + + +class TestProbeMeasureUnits: + """The engine half checks each probe's measure against its declared unit.""" + + def test_shipped_probes_match_their_measures(self, live_engine) -> None: + assert_release_input_coverage_manifest_current(engine=live_engine) + + def test_people_count_declared_as_dollars_is_refused(self, live_engine) -> None: + with pytest.raises( + ValueError, + match=( + "measure_unit 'usd' needs a dollar budget_measure; " + "'medicaid_enrolled' declares unit None" + ), + ): + assert_release_input_coverage_manifest_current( + engine=live_engine, + manifest=_with_probe_change(_HR1_MEDICAID_PROBE, measure_unit="usd"), + ) + + def test_dollars_declared_as_people_are_refused(self, live_engine) -> None: + with pytest.raises( + ValueError, + match="must be a person-level boolean; 'medicaid' is a float on person", + ): + assert_release_input_coverage_manifest_current( + engine=live_engine, + manifest=_with_probe_change( + _HR1_MEDICAID_PROBE, budget_measure="medicaid" + ), + ) + + def test_unknown_measure_is_refused(self, live_engine) -> None: + with pytest.raises( + ValueError, + match="budget_measure 'not_a_variable' is not a PolicyEngine-US variable", + ): + assert_release_input_coverage_manifest_current( + engine=live_engine, + manifest=_with_probe_change( + _HR1_MEDICAID_PROBE, budget_measure="not_a_variable" + ), + ) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_release_input_coverage.py b/packages/microcosm-build/tests/engine_free/shared/test_release_input_coverage.py index 7e5838e1ab..9e205bc218 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_release_input_coverage.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_release_input_coverage.py @@ -184,6 +184,72 @@ def test_either_sign_probe_passes_both_directions_and_keeps_floor_teeth( assert not structural_zero.passed assert "an effect in either direction" in structural_zero.failures[0] + def test_dollar_probe_states_its_floor_in_dollars(self, monkeypatch) -> None: + monkeypatch.setattr(smoke_module, "_build_reform", lambda changes: "REFORM") + result = us_reform_coverage_smoke_gate( + simulate=lambda reform: _Sim(10.0e9), probes=[_probe()] + ) + assert not result.passed + assert ( + "scores +0 on ssi for 2024; expected a positive effect with " + "magnitude at least $1,000,000,000." in result.failures[0] + ) + assert result.details["results"]["ssi_probe"]["measure_unit"] == "usd" + + def test_people_count_probe_states_effect_and_floor_in_people( + self, monkeypatch + ) -> None: + # A person-level boolean measure (medicaid_enrolled) sums weighted + # people, so the gate must state the effect and floor as people, not + # print a dollar floor (microcosm#767). + monkeypatch.setattr(smoke_module, "_build_reform", lambda changes: "REFORM") + probe = ReformCoverageProbe( + id="enrollment_probe", + name="Status-list restoration", + parameter_changes={ + "gov.example.statuses": {"2027-01-01.2100-12-31": ["REFUGEE"]} + }, + budget_measure="medicaid_enrolled", + binding_inputs=("immigration_status_str",), + min_abs_effect=10_000.0, + reason="The restoration binds through the status leaf.", + issue="PolicyEngine/microcosm#767", + period=2027, + measure_unit="persons", + ) + + undersized = us_reform_coverage_smoke_gate( + simulate=lambda reform: _Sim(2_000.0 if reform else 0.0), + probes=[probe], + ) + assert not undersized.passed + failure = undersized.failures[0] + assert "scores +2,000 persons on medicaid_enrolled for 2027" in failure + assert "magnitude at least 10,000 persons." in failure + assert "$" not in failure + result = undersized.details["results"]["enrollment_probe"] + assert result["measure_unit"] == "persons" + + passing = us_reform_coverage_smoke_gate( + simulate=lambda reform: _Sim(250_000.0 if reform else 0.0), + probes=[probe], + ) + assert passing.passed + + def test_probe_rejects_unknown_measure_unit(self) -> None: + with pytest.raises(ValueError, match="measure_unit must be one of"): + ReformCoverageProbe( + id="bad_unit", + name="Bad unit", + parameter_changes={"gov.example.switch": {"2026": 0}}, + budget_measure="medicaid_enrolled", + binding_inputs=("x",), + min_abs_effect=1.0, + reason="r.", + issue="PolicyEngine/microcosm#1", + measure_unit="households", + ) + def test_probe_rejects_unknown_expected_sign(self) -> None: with pytest.raises(ValueError, match="expected_sign must be"): ReformCoverageProbe( @@ -546,6 +612,23 @@ def test_form_4952_election_is_promoted_with_unique_probe(self) -> None: assert probe.neutralized_variable == output assert probe.binding_inputs == (output,) + def test_only_the_hr1_medicaid_probe_counts_people(self) -> None: + # The household-batched post-export scorer refuses Medicaid dollars, + # which PolicyEngine-US prices through a file-wide state denominator + # (microcosm#956), so the H.R.1 Medicaid probe counts enrollees. Every + # other probe sums dollars, the loader's default unit. + counted = { + probe.id: (probe.budget_measure, probe.min_abs_effect) + for probe in us_release_reform_coverage_probes() + if probe.measure_unit != "usd" + } + assert counted == { + "hr1_medicaid_humanitarian_eligibility_restoration": ( + "medicaid_enrolled", + 10_000.0, + ) + } + def test_shipped_ssi_probe_binds_through_the_assets(self) -> None: probes = us_release_reform_coverage_probes() assert probes, "the shipped manifest must pin at least one reform probe" diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py index 00f22d1fde..7092899401 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py @@ -25,7 +25,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # ssi_disability_criteria's attested bytes. # The opt-in cross-fitted gate calibration moves the fit kernels' bytes. assert first.spec_sha256 == ( - "b04b0843ef0926ae371c564c15450e2218b2dd3ef9ade72211fe92c7228985f6" + "97d33501596875471497481f9ca40b84b9b4c3aff2362c9625ea336f1273c613" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_acs_transfer.py b/packages/microcosm-build/tests/engine_free/us/test_us_acs_transfer.py index a5b76be3c8..f1588835f2 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_acs_transfer.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_acs_transfer.py @@ -1219,6 +1219,8 @@ def test_humanitarian_reconciliation_uses_residual_targets_and_records_receipt( "within_residual_discrete_weight_bound": True, "immutable_overshoot": 0.0, "within_discrete_weight_bound": True, + "candidates_exhausted": False, + "candidate_shortfall_population": 0.0, } assert draws["paroled_one_year:ukraine"]["residual_target"] == 60.0 assert draws["paroled_one_year:ukraine"]["achieved_population"] == 60.0 @@ -1226,9 +1228,7 @@ def test_humanitarian_reconciliation_uses_residual_targets_and_records_receipt( assert all(draw["within_discrete_weight_bound"] for draw in draws.values()) -def test_humanitarian_reconciliation_fails_on_candidate_shortfall( - monkeypatch: pytest.MonkeyPatch, -) -> None: +def _transfer_mixed_immigration(ukraine_target: float, monkeypatch) -> object: donor = _with_columns( _donor_frame(), "person", @@ -1241,28 +1241,85 @@ def test_humanitarian_reconciliation_fails_on_candidate_shortfall( monkeypatch.setattr( immigration_module, "us_immigration_controls", - lambda: _small_immigration_controls(ukraine_target=61.0), + lambda: _small_immigration_controls(ukraine_target=ukraine_target), ) + return transfer_acs_inputs( + _mixed_immigration_recipient(), + donor, + target_families={ + "person": { + "source_operator_immigration": ( + "ssn_card_type", + "immigration_status_str", + ) + } + }, + donor_channel=None, + seed=19, + n_estimators=1, + ) + + +def test_humanitarian_reconciliation_takes_every_candidate_when_they_fall_short( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # The Ukrainian candidates hold 60 weighted persons against a residual of + # 61, as a stacked pool's ACS arm can hold fewer recent arrivals than a + # draw's residual: the draw takes them all and records the gap. + result = _transfer_mixed_immigration(61.0, monkeypatch) + person = result.frame.person + assert person["immigration_status_str"].tolist() == [ + "PAROLED_ONE_YEAR", + "CITIZEN", + "PAROLED_ONE_YEAR", + "CITIZEN", + "CITIZEN", + "TPS", + ] + receipt = next( + item.reconciliation + for item in result.imputed_inputs + if item.column == "immigration_status_str" + ) + assert receipt is not None + ukraine = receipt["draws"]["paroled_one_year:ukraine"] + assert ukraine == { + "target": 61.0, + "immutable_population": 0.0, + "residual_target": 61.0, + "eligible_recipient_population": 60.0, + "selected_recipient_population": 60.0, + "achieved_population": 60.0, + "absolute_error": 1.0, + "selection_threshold_tie_population": 0.0, + "residual_selection_error": 1.0, + "within_residual_discrete_weight_bound": False, + "immutable_overshoot": 0.0, + "within_discrete_weight_bound": False, + "candidates_exhausted": True, + "candidate_shortfall_population": 1.0, + } + others = [ + draw + for label, draw in receipt["draws"].items() + if label != "paroled_one_year:ukraine" + ] + assert all(not draw["candidates_exhausted"] for draw in others) + assert all(draw["within_discrete_weight_bound"] for draw in others) + + +def test_humanitarian_reconciliation_refuses_a_shortfall_below_the_band_floor( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # 60 candidates reach 0.30 of a 200 target, under the composition gate's + # floor of 0.35. with pytest.raises( ValueError, - match="candidate shortfall.*paroled_one_year:ukraine", + match=r"candidate shortfall for 'paroled_one_year:ukraine' leaves it " + r"below 0\.35 of its target", ): - transfer_acs_inputs( - _mixed_immigration_recipient(), - donor, - target_families={ - "person": { - "source_operator_immigration": ( - "ssn_card_type", - "immigration_status_str", - ) - } - }, - donor_channel=None, - seed=19, - n_estimators=1, - ) + _transfer_mixed_immigration(200.0, monkeypatch) def test_large_target_family_is_split_to_bound_retained_qrf_forests( @@ -3387,13 +3444,13 @@ def test_adult_care_qualifying_rows_fails_closed_on_missing_structure( #: (schema 3, #767). Every caller that passes none (the shared plan, the #: pool lane, banked checkpoints and the spec projection) must keep these. _DEFAULT_CONTRACT_SHA256 = ( - "4403aefd1de03d28b68fa9b653d5a7e9e6df9a133b4afde22f7f1673b58db63e" + "30eb194ad1ce1f2f76ed06c299195e6e2d556fca3ec1fe95b8b6cd70f752fe6f" ) _DECLARED_PLAN_CONTRACT_SHA256 = ( - "ce27d25196cd8bfbd108cb6beb7e1d9f48334df874e32173ea73ef4d1ce18a0c" + "cce290dc5454df6818aa25eb087642049b4574446e1c9a61a8aef74aa8df1d69" ) _PREGNANCY_CONTRACT_SHA256 = ( - "061d8f1000065e43b2a5bac09a9d4e1b8ec81d123944222d99c42a5f3597045e" + "e5b4d7c3187e640e8a30d2f938e4d0e7fc179664b801b1f618d8e882bb052e0c" ) _EXTENSION_FEATURE = "__acs_transfer_fixture_interest_total" _EXTENSION_FAMILIES = { diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_immigration.py b/packages/microcosm-build/tests/engine_free/us/test_us_immigration.py index 34d4b97727..4255d01144 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_immigration.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_immigration.py @@ -1774,3 +1774,41 @@ def test_summary_reports_weighted_composition(self) -> None: assert ssn["population"]["CITIZEN"] == 3.0 assert ssn["population"]["NONE"] == 1.0 assert summary["person_population"] == 5.0 + + +def test_exact_year_parole_windows_close_when_their_program_closed() -> None: + # Operation Allies Welcome paroled Afghans who arrived in 2021 and 2022, so + # an exact-year (ACS YOEP) row is its candidate only in those years. + # Uniting for Ukraine began in 2022 and has no last year. + years = [2020.0, 2021.0, 2022.0, 2023.0, 2024.0] + person = pd.DataFrame( + { + "CIT": [5.0] * 10, + "POBP": [200.0] * 5 + [164.0] * 5, + "YOEP": years * 2, + "age": [30.0] * 10, + } + ) + profile = immigration_module._source_aware_immigration_profile( + person, time_period=2024 + ) + draws = { + draw.label: draw + for draw in immigration_module.us_immigration_controls().humanitarian + } + + def candidates(label: str) -> list[bool]: + return immigration_module._humanitarian_draw_candidates( + draws[label], + ssn_codes=np.full(len(person), 3), + profile=profile, + ).tolist() + + assert candidates("paroled_one_year:afghanistan") == [ + *(False, True, True, False, False), + *([False] * 5), + ] + assert candidates("paroled_one_year:ukraine") == [ + *([False] * 5), + *(False, False, True, True, True), + ] diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_immigration_properties.py b/packages/microcosm-build/tests/engine_free/us/test_us_immigration_properties.py index fb6e974057..c9b79f4314 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_immigration_properties.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_immigration_properties.py @@ -45,7 +45,10 @@ exactly the rows the gate counts as Pew-unauthorized. - ``reconcile_us_immigration_humanitarian_transfer``: its output satisfies the gate's structural contract; each receipt's achieved population is its - immutable plus selected population; and + immutable plus selected population; a draw whose candidates hold less than + its residual takes every candidate no earlier draw took, and its receipt + records the shortfall; a draw that then reaches less than the composition + band's floor of its target is refused; and ``us_immigration_humanitarian_transfer_selection_masks`` replays the reconciliation's mutable-row selections exactly (differential). @@ -1825,21 +1828,34 @@ class _TransferCase: mutable: np.ndarray controls: immigration.ImmigrationControls seed: int + #: Each draw's recipient candidates before earlier draws remove theirs. + pools: Mapping[str, np.ndarray] + #: The draw whose target no selection can bring to the band floor. + floor_breaker: str | None = None + + +#: Exhausting cases keep each draw at this share of its target or more, above +#: the composition band's floor, so only the floor test reaches the refusal. +_FLOOR_SAFE_SHARE = immigration._HUMANITARIAN_TARGET_RELATIVE_BAND[0] + 0.01 @st.composite -def _transfer_cases(draw) -> _TransferCase: +def _transfer_cases(draw, *, floor_breaker: bool = False) -> _TransferCase: """ASEC rows labeled by the stage (immutable) stacked with ACS recipients carrying unconstrained donor labels (mutable), as in the ACS transfer. - Reconciliation targets are feasible by construction, so the documented - refusal ("candidate exhaustion is a hard error") never applies and any - error is a finding. Each draw's target is at least half its immutable ASEC - mass (so a draw with immutable rows never has an explicit-zero target) - plus a share of the recipient mass that this draw, and no earlier draw, - can claim; ``selected_once`` can only remove rows an earlier draw claims. - Some recipients are ACS twins of ASEC rows (same origin, age and the - ASEC bin's midpoint year), so a draw often has both an immutable + Each draw's target is at least half its immutable ASEC mass (so a draw + with immutable rows never has an explicit-zero target) plus a share of + the recipient mass that this draw, and no earlier draw, can claim; + ``selected_once`` can only remove rows an earlier draw claims. Some + targets then add an excess the candidates cannot fill, so those draws + exhaust them. The excess keeps the immutable mass plus the exclusive + recipient mass at ``_FLOOR_SAFE_SHARE`` of the target or more, so the + refusal below the composition band's floor never applies and any error is + a finding. With ``floor_breaker``, one draw instead takes a target that + even its immutable mass plus every one of its candidates leaves below the + floor. Some recipients are ACS twins of ASEC rows (same origin, age and + the ASEC bin's midpoint year), so a draw often has both an immutable contribution and recipients to select. """ @@ -1894,6 +1910,15 @@ def _transfer_cases(draw) -> _TransferCase: codes = np.where(~profile.is_citizen & (codes == 1), 3, codes) claimed = np.zeros(len(person), dtype=bool) targets: dict[str, float] = {} + pools: dict[str, np.ndarray] = {} + breaker = ( + draw( + st.sampled_from([c.label for c in stage.controls.humanitarian]), + label="floor breaker", + ) + if floor_breaker + else None + ) for control in stage.controls.humanitarian: immutable_mass = float( weights[ @@ -1913,6 +1938,16 @@ def _transfer_cases(draw) -> _TransferCase: ) exclusive = float(weights[pool & ~claimed].sum()) claimed |= pool + pools[control.label] = pool + if control.label == breaker: + reachable = immutable_mass + float(weights[pool].sum()) + floor = immigration._HUMANITARIAN_TARGET_RELATIVE_BAND[0] + targets[control.label] = ( + (reachable + 1.0) + / floor + * draw(st.floats(1.05, 3.0), label="floor breach") + ) + continue immutable_share = draw( st.one_of(st.just(1.0), st.floats(0.5, 1.0)), label="immutable share" ) @@ -1920,9 +1955,15 @@ def _transfer_cases(draw) -> _TransferCase: st.one_of(st.just(0.0), st.just(1.0), st.floats(0.0, 1.0)), label="exclusive share", ) - targets[control.label] = ( - immutable_share * immutable_mass + exclusive_share * exclusive + base_target = immutable_share * immutable_mass + exclusive_share * exclusive + headroom = max( + 0.0, (immutable_mass + exclusive) / _FLOOR_SAFE_SHARE - base_target + ) + excess_share = draw( + st.one_of(st.just(0.0), st.just(1.0), st.floats(0.0, 1.0)), + label="excess share", ) + targets[control.label] = base_target + excess_share * headroom return _TransferCase( person=person, weights=weights, @@ -1936,6 +1977,8 @@ def _transfer_cases(draw) -> _TransferCase: ) ), seed=draw(st.integers(0, 2**31 - 1)), + pools=pools, + floor_breaker=breaker, ) @@ -1997,12 +2040,17 @@ def test_reconciliation_receipts_obey_their_accounting_identities(case) -> None: masks, and must equal immutable + selected; a zero target selects nothing; the selection never exceeds the eligible pool. + A draw whose eligible candidates hold less than its residual "takes every + one of them and its receipt records the shortfall": its selection equals + its eligible pool, it crosses no threshold, its shortfall is the residual + less the pool, and it keeps the composition band's floor. + Receipt consistency only (not independent evidence): the function raises - ``RuntimeError`` before returning a receipt whose selection misses its - residual by more than its threshold tie mass or whose achieved error - exceeds the discrete bound, and it hard-codes - ``within_residual_discrete_weight_bound``. Those assertions confirm the - receipt records the quantities it checked.""" + ``RuntimeError`` before returning a receipt whose unexhausted selection + misses its residual by more than its threshold tie mass or whose achieved + error exceeds the discrete bound, and it sets both bound flags to the + negation of ``candidates_exhausted``. Those assertions confirm the receipt + records the quantities it checked.""" _, receipt = _reconcile(case) tolerance = float(receipt["floating_tolerance"]) @@ -2028,15 +2076,26 @@ def test_reconciliation_receipts_obey_their_accounting_identities(case) -> None: assert selected == 0.0 # Receipt consistency with the function's own guards. assert entry["target"] == target - assert entry["residual_target"] == pytest.approx( - max(0.0, target - immutable), abs=tolerance - ) - assert abs(selected - entry["residual_target"]) <= tie + tolerance - assert abs(achieved - target) <= ( - max(0.0, immutable - target) + tie + tolerance - ) - assert entry["within_residual_discrete_weight_bound"] is True - assert entry["within_discrete_weight_bound"] is True + residual = entry["residual_target"] + eligible = entry["eligible_recipient_population"] + assert residual == pytest.approx(max(0.0, target - immutable), abs=tolerance) + exhausted = eligible + tolerance < residual + assert entry["candidates_exhausted"] is exhausted + assert entry["within_residual_discrete_weight_bound"] is not exhausted + assert entry["within_discrete_weight_bound"] is not exhausted + if exhausted: + event("a draw exhausted its candidates") + assert selected == eligible + assert tie == 0.0 + assert entry["candidate_shortfall_population"] == residual - eligible + floor = immigration._HUMANITARIAN_TARGET_RELATIVE_BAND[0] + assert achieved + tolerance >= floor * target + else: + assert entry["candidate_shortfall_population"] == 0.0 + assert abs(selected - residual) <= tie + tolerance + assert abs(achieved - target) <= ( + max(0.0, immutable - target) + tie + tolerance + ) @_TRANSFER @@ -2068,3 +2127,59 @@ def test_selection_replay_reproduces_the_reconciled_mutable_rows(case) -> None: np.testing.assert_array_equal( replay[draw.label], emitted & case.mutable, err_msg=draw.label ) + + +@_TRANSFER +@given(_transfer_cases()) +def test_exhausted_draws_take_every_candidate_no_earlier_draw_took(case) -> None: + """A draw whose candidates hold less than its residual "takes every one of + them". Independent evidence: each draw's candidate pool comes from the + strategy, computed before any selection, and the rows earlier draws took + come from the replay that ``test_selection_replay_reproduces_the_reconciled_ + mutable_rows`` checks against the emitted labels. An exhausted draw's + selection is exactly its pool less those rows, and its receipt's eligible + population is their weight.""" + + result, receipt = _reconcile(case) + replay = immigration.us_immigration_humanitarian_transfer_selection_masks( + _frame(result, case.weights), + mutable_rows=case.mutable, + seed=case.seed, + time_period=_TIME_PERIOD, + controls=case.controls, + ) + taken = np.zeros(len(case.person), dtype=bool) + for draw in case.controls.humanitarian: + remaining = case.pools[draw.label] & ~taken + entry = receipt["draws"][draw.label] + assert entry["eligible_recipient_population"] == pytest.approx( + float(case.weights[remaining].sum()), + abs=float(receipt["floating_tolerance"]), + ) + if entry["candidates_exhausted"]: + event("an exhausted draw took its remaining candidates") + np.testing.assert_array_equal( + replay[draw.label], remaining, err_msg=draw.label + ) + assert ( + result.loc[remaining, "immigration_status_str"] == draw.status + ).all() + taken |= replay[draw.label] + + +@_TRANSFER +@given(_transfer_cases(floor_breaker=True)) +def test_reconciliation_refuses_a_draw_exhausted_below_the_band_floor(case) -> None: + """The reconciliation "refuses only a draw that then reaches less than the + floor of the composition gate's band of its target". The strategy gives + one draw a target that its immutable mass plus every one of its + candidates leaves below that floor, and keeps every other draw above it: + the reconciliation must refuse that draw, by name.""" + + floor = immigration._HUMANITARIAN_TARGET_RELATIVE_BAND[0] + with pytest.raises(ValueError) as refusal: + _reconcile(case) + assert str(refusal.value).startswith( + "Humanitarian transfer candidate shortfall for " + f"{case.floor_breaker!r} leaves it below {floor} of its target" + ) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_post_export_scoring.py b/packages/microcosm-build/tests/engine_free/us/test_us_post_export_scoring.py index eeb0fabb2a..c93a0e35e5 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_post_export_scoring.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_post_export_scoring.py @@ -1548,8 +1548,9 @@ def test_guard_sweep_records_and_scores_shipped_reform_passes( == sum(spec.in_sample for spec in specs) == 12 ) - # The microcosm#767 H.R.1 probes add aca_ptc 2026 and 2027 and medicaid - # 2027; their snap 2026 key is shared with the Texas vehicle probe. + # The microcosm#767 H.R.1 probes add aca_ptc 2026 and 2027 and + # medicaid_enrolled 2027; their snap 2026 key is shared with the Texas + # vehicle probe. assert counts["baseline_keys"] == { "reform_coverage_smoke": 23, "reform_validation": 75, diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_stacked_spine.py b/packages/microcosm-build/tests/engine_free/us/test_us_stacked_spine.py index d2617a0e1b..e3804f8123 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_stacked_spine.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_stacked_spine.py @@ -5680,9 +5680,11 @@ def test_immigration_reconciliation_replays_receipt_and_downstream_live_frame( ) -def test_immigration_live_replay_rejects_equal_weight_selection_identity_swap( - monkeypatch: pytest.MonkeyPatch, -) -> None: +def _ukrainian_parole_stack() -> tuple[ + Frame, pd.DataFrame, np.ndarray, np.ndarray, float +]: + """One ASEC citizen household stacked on two equal-weight ACS U4U candidates.""" + asec = _source_frame( household_ids=[11], weights=[2.0], @@ -5726,14 +5728,11 @@ def test_immigration_live_replay_rejects_equal_weight_selection_identity_swap( weights = np.asarray(base.resolve_weights("person").values, dtype=np.float64) mutable_weight = float(weights[mutable][0]) assert weights[mutable].tolist() == [mutable_weight, mutable_weight] - draw = immigration_module.HumanitarianDraw( - category="paroled_one_year", - origin="ukraine", - status="PAROLED_ONE_YEAR", - target=mutable_weight, - source="https://example.com/u4u", - ) - controls = immigration_module.ImmigrationControls( + return base, person, weights, mutable, mutable_weight + + +def _ukrainian_parole_controls(target: float) -> immigration_module.ImmigrationControls: + return immigration_module.ImmigrationControls( undocumented=immigration_module.UndocumentedControls( workers=1.0, students=1.0, @@ -5744,8 +5743,37 @@ def test_immigration_live_replay_rejects_equal_weight_selection_identity_swap( "undocumented_population_anchor": "https://example.com/population", }, ), - humanitarian=(draw,), + humanitarian=( + immigration_module.HumanitarianDraw( + category="paroled_one_year", + origin="ukraine", + status="PAROLED_ONE_YEAR", + target=target, + source="https://example.com/u4u", + ), + ), ) + + +def _with_person(frame: Frame, replacement: pd.DataFrame) -> Frame: + tables = {entity: frame.table(entity) for entity in frame.entities} + tables["person"] = replacement + return Frame( + tables, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +def test_immigration_live_replay_rejects_equal_weight_selection_identity_swap( + monkeypatch: pytest.MonkeyPatch, +) -> None: + base, person, weights, mutable, mutable_weight = _ukrainian_parole_stack() + immutable = ~mutable + controls = _ukrainian_parole_controls(mutable_weight) reconciled_person, reconciliation = ( immigration_module.reconcile_us_immigration_humanitarian_transfer( person, @@ -5757,19 +5785,7 @@ def test_immigration_live_replay_rejects_equal_weight_selection_identity_swap( ) ) - def with_person(frame: Frame, replacement: pd.DataFrame) -> Frame: - tables = {entity: frame.table(entity) for entity in frame.entities} - tables["person"] = replacement - return Frame( - tables, - frame.schema, - {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, - frame.strata, - mass_log=frame.mass_log, - metadata=frame.metadata, - ) - - reconciled = with_person(base, reconciled_person) + reconciled = _with_person(base, reconciled_person) target_receipt = { "producer_rows": int(immutable.sum()), "authorized_null_rows": int(mutable.sum()), @@ -5804,7 +5820,7 @@ def with_person(frame: Frame, replacement: pd.DataFrame) -> Frame: "LEGAL_PERMANENT_RESIDENT" ) swapped_person.loc[unselected_index, "immigration_status_str"] = "PAROLED_ONE_YEAR" - swapped = with_person(reconciled, swapped_person) + swapped = _with_person(reconciled, swapped_person) assert float( swapped.resolve_weights("person") .values[ @@ -5826,6 +5842,158 @@ def with_person(frame: Frame, replacement: pd.DataFrame) -> Frame: ) +def _exhausted_ukrainian_parole( + monkeypatch: pytest.MonkeyPatch, + target_multiple: float, +) -> tuple[Frame, dict[str, object], float]: + """Reconcile the two-candidate stack to ``target_multiple`` candidate weights.""" + + base, person, weights, mutable, mutable_weight = _ukrainian_parole_stack() + controls = _ukrainian_parole_controls(target_multiple * mutable_weight) + reconciled_person, reconciliation = ( + immigration_module.reconcile_us_immigration_humanitarian_transfer( + person, + weights=weights, + mutable_rows=mutable, + seed=0, + time_period=2024, + controls=controls, + ) + ) + monkeypatch.setattr( + stacked_spine_module, + "us_immigration_controls", + lambda: controls, + ) + target_receipt = { + "producer_rows": int((~mutable).sum()), + "authorized_null_rows": int(mutable.sum()), + "imputed_rows": int(mutable.sum()), + "unmodeled_rows": 0, + "residual_null_rows": 0, + "post_transfer_reconciliation": reconciliation, + } + return _with_person(base, reconciled_person), target_receipt, mutable_weight + + +def test_immigration_reconciliation_takes_every_candidate_of_an_exhausted_draw( + monkeypatch: pytest.MonkeyPatch, +) -> None: + frame, target_receipt, weight = _exhausted_ukrainian_parole(monkeypatch, 2.5) + draw = target_receipt["post_transfer_reconciliation"]["draws"][ + "paroled_one_year:ukraine" + ] + assert draw["candidates_exhausted"] is True + assert draw["within_residual_discrete_weight_bound"] is False + assert draw["within_discrete_weight_bound"] is False + assert draw["eligible_recipient_population"] == pytest.approx(2 * weight) + assert draw["selected_recipient_population"] == pytest.approx(2 * weight) + assert draw["selection_threshold_tie_population"] == 0.0 + assert draw["candidate_shortfall_population"] == pytest.approx(0.5 * weight) + person = frame.table("person") + channel = person[support_channel_column("person")].astype(str) + assert person.loc[channel.eq("acs"), "immigration_status_str"].tolist() == [ + "PAROLED_ONE_YEAR", + "PAROLED_ONE_YEAR", + ] + for live in (frame, None): + stacked_spine_module._validate_immigration_post_transfer_reconciliation( + target_receipt, + boundary="exhausted draw", + frame=live, + ) + + +def _forged(target_receipt: dict[str, object], **fields: object) -> dict[str, object]: + forged = deepcopy(target_receipt) + forged["post_transfer_reconciliation"]["draws"]["paroled_one_year:ukraine"].update( + fields + ) + return forged + + +def test_immigration_reconciliation_rejects_forged_candidate_shortfalls( + monkeypatch: pytest.MonkeyPatch, +) -> None: + frame, target_receipt, weight = _exhausted_ukrainian_parole(monkeypatch, 2.5) + validate = stacked_spine_module._validate_immigration_post_transfer_reconciliation + + concealed = _forged( + target_receipt, + candidates_exhausted=False, + within_residual_discrete_weight_bound=True, + within_discrete_weight_bound=True, + ) + with pytest.raises(ValueError, match="conceals a recipient candidate shortfall"): + validate(concealed, boundary="concealed shortfall", frame=None) + + covered = _forged(target_receipt, eligible_recipient_population=2.5 * weight) + with pytest.raises( + ValueError, + match="claims a candidate shortfall its eligible population covers", + ): + validate(covered, boundary="covered shortfall", frame=None) + + inconsistent = _forged(target_receipt, within_discrete_weight_bound=True) + with pytest.raises(ValueError, match="values are invalid"): + validate(inconsistent, boundary="inconsistent bound flag", frame=None) + + unselected = _forged(target_receipt, selected_recipient_population=weight) + with pytest.raises(ValueError, match="leaves candidates unselected"): + validate(unselected, boundary="unselected candidate", frame=None) + + person = frame.table("person").copy() + channel = person[support_channel_column("person")].astype(str) + dropped = person.index[channel.eq("acs").to_numpy()][0] + person.loc[dropped, "immigration_status_str"] = "LEGAL_PERMANENT_RESIDENT" + with pytest.raises( + ValueError, + match="differs from the live ASEC/ACS weighted population", + ): + validate( + target_receipt, + boundary="live candidate left unselected", + frame=_with_person(frame, person), + ) + + +def test_immigration_reconciliation_rejects_exhaustion_below_the_band_floor( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Two candidates reach 0.2 of a ten-candidate target, under the + # composition gate's floor of 0.35: the reconciliation refuses it. + with pytest.raises( + ValueError, + match=r"leaves it below 0\.35 of its target", + ): + _exhausted_ukrainian_parole(monkeypatch, 10.0) + + # A receipt that claims such a draw is refused too. + _frame, target_receipt, weight = _exhausted_ukrainian_parole(monkeypatch, 2.5) + monkeypatch.setattr( + stacked_spine_module, + "us_immigration_controls", + lambda: _ukrainian_parole_controls(10 * weight), + ) + below_floor = _forged( + target_receipt, + target=10 * weight, + residual_target=10 * weight, + residual_selection_error=8 * weight, + absolute_error=8 * weight, + candidate_shortfall_population=8 * weight, + ) + with pytest.raises( + ValueError, + match="falls below the floor of the composition gate's band", + ): + stacked_spine_module._validate_immigration_post_transfer_reconciliation( + below_floor, + boundary="exhausted below the band floor", + frame=None, + ) + + def test_late_executor_signature_rejects_qrf_regime_evidence_tampering( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/packages/microcosm-build/tests/engine_scenario/us/test_us_hr1_immigration_channels.py b/packages/microcosm-build/tests/engine_scenario/us/test_us_hr1_immigration_channels.py index 85c60e4e71..19b3cf3c3b 100644 --- a/packages/microcosm-build/tests/engine_scenario/us/test_us_hr1_immigration_channels.py +++ b/packages/microcosm-build/tests/engine_scenario/us/test_us_hr1_immigration_channels.py @@ -4,7 +4,7 @@ reforms on the full export; these tests prove the same channels move, nonzero and sign-correct, on synthetic households whose ``immigration_status_str`` carries the humanitarian values the source stage -now imputes. A probe whose channel dies here would score $0 on every +now imputes. A probe whose channel dies here would score zero on every release, so this is the PR-CI early warning for the engine side of the silent-zero failure — without needing restricted microdata. @@ -86,8 +86,13 @@ def _effect(probe, situation: dict, year: int) -> tuple[float, float]: def test_medicaid_restoration_reenrolls_refugee_family(hr1_probes) -> None: # At 2027 law refugees are outside the §71109-narrowed qualified list; - # restoring the pre-H.R.1 list must re-enroll the anchored taker. + # restoring the pre-H.R.1 list must enroll the anchored taker. The probe + # counts enrollees (medicaid_enrolled), so the totals here are people. probe = hr1_probes["hr1_medicaid_humanitarian_eligibility_restoration"] + assert (probe.budget_measure, probe.measure_unit) == ( + "medicaid_enrolled", + "persons", + ) situation = _household( "REFUGEE", 25_000.0, @@ -152,9 +157,10 @@ def test_snap_restoration_reincludes_refugee_members(hr1_probes) -> None: def test_probe_magnitude_floors_sit_far_below_plausible_scale(hr1_probes) -> None: # The floors are release-scale guards: far above numerical noise, far # below the plausible aggregate effect of the imputed stocks (about - # 1.6M weighted persons across the humanitarian categories). + # 1.6M weighted persons across the humanitarian categories). The + # Medicaid floor counts people; the others are dollars. floors = { - "hr1_medicaid_humanitarian_eligibility_restoration": 50_000_000.0, + "hr1_medicaid_humanitarian_eligibility_restoration": 10_000.0, "hr1_aca_below_fpl_exception_restoration": 5_000_000.0, "hr1_aca_lawful_presence_restoration": 10_000_000.0, "hr1_snap_humanitarian_eligibility_restoration": 25_000_000.0, @@ -162,6 +168,11 @@ def test_probe_magnitude_floors_sit_far_below_plausible_scale(hr1_probes) -> Non for probe_id, floor in floors.items(): probe = hr1_probes[probe_id] assert probe.min_abs_effect == floor + assert probe.measure_unit == ( + "persons" + if probe_id == "hr1_medicaid_humanitarian_eligibility_restoration" + else "usd" + ) assert probe.expected_sign == "positive" assert probe.binding_inputs == ("immigration_status_str",) assert probe.issue == "PolicyEngine/microcosm#767" diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py index aa63f89af1..fa730ab99e 100644 --- a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py @@ -380,7 +380,7 @@ def capture_equality(expected: object, actual: object) -> None: # through the bundle's source stages. # microcosm#767's humanitarian immigration controls, its attested # immigration kernel and its frozen legacy schedule move it. - "spec_sha256": "c414c985f6da4c8d623c79f15bce4c35a5fa32246fbfed941390b24fee41f429", + "spec_sha256": "783ff9a423a3de4db335c5886e37139ac94b0ae619528433e08a273c17bb111f", }, } diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_us_post_export_scoring.py b/packages/microcosm-build/tests/engine_workflow/us/test_us_post_export_scoring.py index e4d27c450f..62b4d758b8 100644 --- a/packages/microcosm-build/tests/engine_workflow/us/test_us_post_export_scoring.py +++ b/packages/microcosm-build/tests/engine_workflow/us/test_us_post_export_scoring.py @@ -277,9 +277,12 @@ def test_shipped_baseline_plans_match_whole_file_policyengine_us( path = _write_engine_h5(builder, tmp_path, _GUARD_HOUSEHOLDS) plans, _, counts = sweep._record_requests(builder) # The four H.R.1 humanitarian-eligibility probes (#767) add three new - # baseline keys: aca_ptc 2026 and 2027 and medicaid 2027 (their snap 2026 - # key is shared with the Texas vehicle probe). microcosm#1251's passive - # partnership/S-corp probe adds one smoke baseline key. + # baseline keys: aca_ptc 2026 and 2027 and medicaid_enrolled 2027 (their + # snap 2026 key is shared with the Texas vehicle probe). The Medicaid + # probe counts enrollees because Medicaid dollars divide by a file-wide + # state denominator, which the household-batched scorer refuses + # (microcosm#956). microcosm#1251's passive partnership/S-corp probe + # adds one smoke baseline key. assert counts["baseline_keys"] == { "reform_coverage_smoke": 23, "reform_validation": 75, diff --git a/tools/build_us_release_input_coverage_manifest.py b/tools/build_us_release_input_coverage_manifest.py index cb5f91180a..326a094f5a 100644 --- a/tools/build_us_release_input_coverage_manifest.py +++ b/tools/build_us_release_input_coverage_manifest.py @@ -1614,25 +1614,38 @@ ] } }, - "budget_measure": "medicaid", + "budget_measure": "medicaid_enrolled", + "measure_unit": "persons", "period": 2027, "effect_direction": "reform_minus_baseline", "expected_sign": "positive", "binding_inputs": ["immigration_status_str"], - "min_abs_effect": 50_000_000.0, + "min_abs_effect": 10_000.0, "reason": ( "H.R.1 SS71109 narrows the Medicaid qualified-immigrant list to " - "citizens, LPRs, and Cuban/Haitian entrants from 2026-10-01; the " - "probe restores the pre-narrowing list at 2027 law, which binds " - "only through the REFUGEE/ASYLEE/DEPORTATION_WITHHELD/" - "PAROLED_ONE_YEAR values of immigration_status_str. The source " - "stage draws roughly 160k refugees (74% reporting Medicaid on the " - "2024 ASEC), 155k asylees, and 405k parolees to cited DHS/OHSS " - "stocks, so restoring their eligibility must re-enroll anchored " - "takers and move person-level medicaid dollars by far more than " - "the floor. A ~$0 score means the humanitarian statuses regressed " - "to zero records (microcosm #767's silent-zero failure) or the " - "engine channel broke." + "citizens, LPRs, and Cuban/Haitian entrants from 2026-10-01. The " + "probe restores the pre-narrowing list at 2027 law, adding " + "REFUGEE, ASYLEE, DEPORTATION_WITHHELD, PAROLED_ONE_YEAR and " + "CONDITIONAL_ENTRANT; the source stage draws no conditional " + "entrants, so the reform binds only through the first four values " + "of immigration_status_str. The probe counts weighted people in " + "medicaid_enrolled rather than summing medicaid dollars, because " + "PolicyEngine-US prices each enrollee through " + "medicaid_slcsp_state_denominator, an aggregate over the whole " + "file that the household-batched post-export scorer refuses " + "(PolicyEngine/microcosm#956). On the default-pool stage receipt " + "(docs/evidence/us-immigration-statuses-767), 0.391 million of " + "the 0.714 million weighted people in the restored statuses " + "report Medicaid at interview, and the Medicaid take-up stage " + "sets takes_up_medicaid_if_eligible for every reporter. The " + "reform only widens the status list, so every reporter whom the " + "narrowed list alone kept out enrolls. The 10,000-person floor is " + "2.6% of the reporters: a structural zero fails it, and the effect " + "clears it unless Medicaid's other rules keep out more than 97% of " + "the reporters. A ~0 count " + "means the humanitarian statuses regressed to zero records " + "(microcosm #767's silent-zero failure) or the engine channel " + "broke." ), "issue": "PolicyEngine/microcosm#767", }, @@ -1657,8 +1670,10 @@ "non-citizen who is ACA-lawfully-present yet " "Medicaid-ineligible by status - on this file exactly the TPS " "population the source stage draws to the CRS RS20844 per-country " - "stocks (865k weighted persons; the other humanitarian classes " - "remain Medicaid-qualified at 2026 annual parameters). " + "stocks (0.860 million weighted persons on the default-pool stage " + "receipt, docs/evidence/us-immigration-statuses-767; the other " + "humanitarian classes remain Medicaid-qualified at 2026 annual " + "parameters). " "Below-poverty TPS units with marketplace take-up must regain " "premium tax credits above the floor; ~$0 means the TPS records " "vanished or the exception channel broke." @@ -1684,8 +1699,9 @@ "PAROLED_ONE_YEAR to the ACA ineligible-status list from " "2027-01-01. The probe restores the pre-H.R.1 list (DACA and " "UNDOCUMENTED only) at 2027, re-qualifying every humanitarian " - "status the source stage draws (~1.6M weighted persons across " - "parole, refugee, asylee, and TPS). Units among them with " + "status the source stage draws (1.574 million weighted persons " + "across parole, refugee, asylee and TPS on the default-pool stage " + "receipt). Units among them with " "marketplace take-up must regain premium tax credits above the " "floor; ~$0 means the humanitarian statuses regressed or the " "lawful-presence channel broke." From 2f70d5c3ce6d310bfab5a5a4f505b74f9f104917 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 10 Nov 2026 09:23:50 -0500 Subject: [PATCH 44/49] Regenerate the #767 stage receipts at 4f94e489a; add the stacked-supply receipt The receipt script reran each tree at the code commit. The head run labels every row as before and its composition, gate and draw results equal the previous head receipt's; only provenance moves. Main's two receipts now carry the result of main's own composition gate, which passes. The Medicaid-reporter receipt names the new head's labels file. receipt.stacked_supply.json projects each humanitarian draw's ACS supply in a stacked pool from receipt.head.json and the ACS 2024 1-year PUMS. Four draws run short of ACS candidates and end at 0.718 to 0.971 of their targets; in the worst case none ends below half its target. The README's receipts table, stacked-pool section and rerun instructions describe it. Refs #767 Co-Authored-By: Claude Opus 5.5 --- .../us-immigration-statuses-767/README.md | 105 ++++++++- .../receipt.head.json | 8 +- .../receipt.head_medicaid_reporters.json | 56 +++++ .../receipt.main.json | 6 +- .../receipt.main_full_keys.json | 6 +- .../receipt.pre_sync_42bb166b.json | 2 +- .../receipt.stacked_supply.json | 217 ++++++++++++++++++ 7 files changed, 387 insertions(+), 13 deletions(-) create mode 100644 docs/evidence/us-immigration-statuses-767/receipt.head_medicaid_reporters.json create mode 100644 docs/evidence/us-immigration-statuses-767/receipt.stacked_supply.json diff --git a/docs/evidence/us-immigration-statuses-767/README.md b/docs/evidence/us-immigration-statuses-767/README.md index 4569d555d7..b7c5e4df71 100644 --- a/docs/evidence/us-immigration-statuses-767/README.md +++ b/docs/evidence/us-immigration-statuses-767/README.md @@ -28,7 +28,7 @@ the next certified build to that pool. The script: - records, for each draw call, the key that reproduces its draws; - runs at seed 0 and period 2024, the `--target-year` of `tools/route_a/route_a.sh`; -- runs the composition gate on the output. +- runs that tree's own composition gate on the output. `--draw-keys full` hands all three lineage columns to every draw call that brings no key of its own, and `--draw-keys legacy` hands it `source_year` and @@ -42,10 +42,12 @@ of its `immigration.py` and of the script, and every draw call's key. | File | Tree | Draw keys | Result | |---|---|---|---| -| `receipt.head.json` | this PR at `e3b2c8ce9`, which last merged main at `764ae3aa7` | its own: income year, `PH_SEQ`, `PERIDNUM` | the gate passes with no failures | -| `receipt.main.json` | main at `764ae3aa7` | its own: income year, `PERIDNUM` | main draws no humanitarian status | -| `receipt.main_full_keys.json` | main at `764ae3aa7` | this PR's, through `--draw-keys full` | main's method on this PR's draws | +| `receipt.head.json` | this PR at `4f94e489a`, which last merged main at `764ae3aa7` | its own: income year, `PH_SEQ`, `PERIDNUM` | its gate passes with no failures | +| `receipt.main.json` | main at `764ae3aa7` | its own: income year, `PERIDNUM` | main draws no humanitarian status; its gate passes | +| `receipt.main_full_keys.json` | main at `764ae3aa7` | this PR's, through `--draw-keys full` | main's method on this PR's draws; its gate passes | | `receipt.pre_sync_42bb166b.json` | this PR at `42bb166b1`, before it merged main `1acd884fe` | none: the stage refuses before its first draw | the stage refuses the 2026 ASEC's `PEINUSYR` code 29 | +| `receipt.head_medicaid_reporters.json` | this PR at `4f94e489a`, from the head run's labels | its own | 0.391 million of the 0.714 million people in the restored Medicaid statuses report Medicaid at interview | +| `receipt.stacked_supply.json` | this PR at `4f94e489a`, from `receipt.head.json` and the ACS 2024 1-year PUMS | none: it draws no rows | four draws run short of ACS candidates, and none ends below half its target in the worst case | The script reran main's tree and the pre-sync tree from `git archive packages`, so those three receipts take their commit from `--source-commit`. @@ -80,6 +82,86 @@ Weighted persons, in millions: million control. - In both trees `ssn_card_type` is `NONE` exactly on the undocumented rows. +## Medicaid at interview in the restored statuses + +The reform-coverage probe `hr1_medicaid_humanitarian_eligibility_restoration` +restores the pre-H.R.1 Medicaid status list at 2027 law and counts the people +it enrolls. `scripts/medicaid_reporters.py` joins the head run's labels to the +ASEC's `NOW_MCAID` (Medicaid at interview), which `us_runtime/cps_carried.py` +maps to `has_medicaid_health_coverage_at_interview`, the column the Medicaid +take-up stage anchors on. `receipt.head_medicaid_reporters.json` holds the +aggregates, in millions of weighted people: + +| Status | People | Report Medicaid at interview | +|---|---:|---:| +| Refugee | 0.160 | 0.121 | +| Asylee | 0.155 | 0.107 | +| Paroled, one year | 0.398 | 0.163 | +| All restored | 0.714 | 0.391 | + +The stage draws no deportation-withheld or conditional-entrant rows. The +probe's 10,000-person floor is 2.6% of the 0.391 million reporters. + +## The stacked pool's ACS residuals + +The `--exact-k` arm of `tools/build_us_fiscal_refresh_release.py`, the only arm +that runs the improvement gate, takes a pool from +`tools/build_us_multispine_pool.py`, whose default pipeline stacks the ASEC pool +and the ACS 2024 1-year. Each arm carries half the ASEC pool's household mass +(`stacked_spine.DEFAULT_STACKED_HOUSEHOLD_MASS_SHARES`), and the immigration +stage labels the ASEC rows at the full pool's person mass, so those rows carry +about half of each draw's target. The ACS transfer's +`reconcile_us_immigration_humanitarian_transfer` then asks the ACS rows for the +rest, each draw's residual. + +The reconciliation used to refuse any draw whose ACS candidates held less +weight than its residual. Now such a draw takes every candidate, as the ASEC +stage's draws do, and its receipt records the shortfall. The reconciliation +refuses only a draw that ends below 0.35 of its target, the floor of the +composition gate's band, and the stacked validator replays the same rule. +Operation Allies Welcome (OAW) parole now takes exact-year ACS arrivals from +2021 and 2022 only, the program's arrival years; its window used to stay open so +that the old rule would not refuse it. + +`scripts/stacked_supply.py` projects each draw's ACS supply without building a +stacked pool. It takes the ASEC arm's labelled mass from `receipt.head.json`, +scaled to the stacked weights, and each draw's candidates from the ACS 2024 +1-year PUMS through the kernel's own candidate rule, weighted as the frame +weights them and scaled by the ACS arm's weight scale, 0.478. +`receipt.stacked_supply.json` holds the aggregates, in weighted persons: + +| Draw | Target | ASEC arm | ACS residual | ACS supply | Projected share | Worst case | +|---|---:|---:|---:|---:|---:|---:| +| Parole, Afghanistan (OAW) | 73,566 | 37,128 | 36,438 | 34,292 | 0.971 | 0.971 | +| Parole, Ukraine (U4U) | 158,000 | 56,467 | 101,533 | 57,046 | 0.718 | 0.718 | +| Parole, Nicaragua | 93,070 | 46,971 | 46,099 | 27,188 | 0.797 | 0.797 | +| Parole, Venezuela | 117,330 | 59,215 | 58,115 | 152,918 | 1.000 | 1.000 | +| Refugee | 160,000 | 80,750 | 79,250 | 363,809 | 1.000 | 0.505 | +| Asylee | 155,000 | 78,227 | 76,773 | 2,292,045 | 1.000 | 0.505 | +| TPS, Venezuela | 605,015 | 305,344 | 299,671 | 338,594 | 1.000 | 0.710 | +| TPS, El Salvador | 170,125 | 85,860 | 84,265 | 114,363 | 1.000 | 1.000 | +| TPS, Honduras | 51,225 | 25,853 | 25,372 | 22,197 | 0.938 | 0.938 | +| TPS, Nicaragua | 2,910 | 1,469 | 1,441 | 6,960 | 1.000 | 1.000 | +| TPS, Nepal | 7,160 | 3,614 | 3,546 | 16,986 | 1.000 | 1.000 | +| TPS, other designated | 21,215 | 10,707 | 10,508 | 141,413 | 1.000 | 1.000 | + +- Four draws run short of candidates: U4U, Nicaraguan parole, Honduran TPS and + OAW. Under the old rule, U4U, the second draw in manifest order, would have + stopped every stacked build on these inputs. Under the new rule the four end + at 0.718 to 0.971 of their targets. +- The worst case lets every earlier draw take as much of a draw's candidates as + it can, up to its own residual. It holds for parole and TPS draws, which take + any non-citizen SSN code. Refugee and asylee draws take only rows the transfer + imputes as `OTHER_NON_CITIZEN`, which PUMS cannot show, so their supply column + overstates and their worst case counts the ASEC arm alone. +- The ASEC arm alone carries 0.505 of each target the ASEC stage met, and 0.357 + of U4U's. No draw falls below the 0.35 floor in the worst case: the lowest is + 0.505. +- The projection scales each arm by one factor. A build stands or falls on the + reconciliation itself and on the pool's terminal gates, which run the + composition gate on the stacked pool (`tools/build_us_multispine_pool.py`), + not on this projection. + ## Why the two trees label different rows The totals move little, but the two trees label different rows. Two changes in @@ -173,3 +255,18 @@ rebuild a row-by-row table: ```bash .venv/bin/python -I docs/evidence/us-immigration-statuses-767/scripts/xtab_labels.py main.pkl head.pkl ``` + +Count the head run's restored-status Medicaid reporters from its labels: + +```bash +.venv/bin/python -I docs/evidence/us-immigration-statuses-767/scripts/medicaid_reporters.py head.pkl --archive-dir --out receipt.head_medicaid_reporters.json +``` + +Project the stacked pool's ACS supply from the head receipt. Put the ACS 2024 +1-year PUMS files `csv_hus.zip` and `csv_pus.zip`, at the URLs +`us_runtime/acs_2024_1yr_sources.json` pins, in one directory; the script also +reads the three ASEC zips for the pool's masses: + +```bash +.venv/bin/python -I docs/evidence/us-immigration-statuses-767/scripts/stacked_supply.py receipt.head.json --acs-dir --archive-dir --out receipt.stacked_supply.json +``` diff --git a/docs/evidence/us-immigration-statuses-767/receipt.head.json b/docs/evidence/us-immigration-statuses-767/receipt.head.json index 9f0ed9fbed..d7b45f6eb3 100644 --- a/docs/evidence/us-immigration-statuses-767/receipt.head.json +++ b/docs/evidence/us-immigration-statuses-767/receipt.head.json @@ -354,7 +354,7 @@ "REFUGEE": 160000.0, "TPS": 857650.0 }, - "immigration_module_sha256": "c4d81501e63917595bae9cd976ab8068187d78cbb42d7146d5ecd1416209a4b8", + "immigration_module_sha256": "ecac0e264521639af5e240fe33edf8f7c3568437497d4eb27dee1862ba7e5b6c", "none_ssn_iff_undocumented": true, "pew_labor_force": 9699442.858246539, "pew_labor_force_control": 9700000.0, @@ -368,12 +368,12 @@ "TPS": 1.0028800890820582 }, "rows": 421119, - "script_sha256": "6d1a8899979617dc3d53ec1d776a1825f0d55caf00abfe037e806aebbdc8d8eb", + "script_sha256": "7c28e5000282137583839624f97d731b43637d8f2a49cd8bbfb5e25bcb6e47dc", "seed": 0, - "source_commit": "e3b2c8ce9404d177a5424b2dd3b1acb6cf385e6f", + "source_commit": "4f94e489aba5de4c33444742ac8333e736acba03", "source_commit_from": "git", "source_tree_clean": true, - "stage_seconds": 8.3, + "stage_seconds": 10.8, "time_period": 2024, "weights": "HSUP_WGT * 0.01 by PH_SEQ, each file rescaled to one third of income year 2024's weighted person population (asec_pool.pool_asec_sources, anchor_year=2024)" } diff --git a/docs/evidence/us-immigration-statuses-767/receipt.head_medicaid_reporters.json b/docs/evidence/us-immigration-statuses-767/receipt.head_medicaid_reporters.json new file mode 100644 index 0000000000..e5b84d3c78 --- /dev/null +++ b/docs/evidence/us-immigration-statuses-767/receipt.head_medicaid_reporters.json @@ -0,0 +1,56 @@ +{ + "archives": [ + { + "archive": "asecpub24csv.zip", + "income_year": 2023, + "sha256": "cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09" + }, + { + "archive": "asecpub25csv.zip", + "income_year": 2024, + "sha256": "318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b" + }, + { + "archive": "asecpub26csv.zip", + "income_year": 2025, + "sha256": "fe819d0d2fc4470c282e76c247b8aa8b6054811619730307e9f2fe6186707d93" + } + ], + "column": "NOW_MCAID == 1", + "labels_file": "head_4f94e489a.native.labels.pkl", + "labels_rows": 421119, + "labels_sha256": "363451875c6d7842c59a79e037880d9bcf29f266676a9e1a1586310b0779ff0f", + "script_sha256": "ee1efc1a55a511bec5e668f7032e71de68acf58d29e8e996b44609376ae35a72", + "statuses": { + "ASYLEE": { + "now_mcaid_persons_millions": 0.106635, + "persons_millions": 0.155492, + "rows": 186 + }, + "CONDITIONAL_ENTRANT": { + "now_mcaid_persons_millions": 0.0, + "persons_millions": 0.0, + "rows": 0 + }, + "DEPORTATION_WITHHELD": { + "now_mcaid_persons_millions": 0.0, + "persons_millions": 0.0, + "rows": 0 + }, + "PAROLED_ONE_YEAR": { + "now_mcaid_persons_millions": 0.163247, + "persons_millions": 0.397889, + "rows": 438 + }, + "REFUGEE": { + "now_mcaid_persons_millions": 0.121047, + "persons_millions": 0.160452, + "rows": 176 + }, + "all restored": { + "now_mcaid_persons_millions": 0.39093, + "persons_millions": 0.713833, + "rows": 800 + } + } +} diff --git a/docs/evidence/us-immigration-statuses-767/receipt.main.json b/docs/evidence/us-immigration-statuses-767/receipt.main.json index e36fb6fee6..c03cce907a 100644 --- a/docs/evidence/us-immigration-statuses-767/receipt.main.json +++ b/docs/evidence/us-immigration-statuses-767/receipt.main.json @@ -106,15 +106,17 @@ } }, "error": null, + "gate_failures": [], + "gate_passed": true, "immigration_module_sha256": "8871df8b5f86f6bbf8541259a69d52b7db214560e5dacf5ad6db15268368f826", "none_ssn_iff_undocumented": true, "rows": 421119, - "script_sha256": "6d1a8899979617dc3d53ec1d776a1825f0d55caf00abfe037e806aebbdc8d8eb", + "script_sha256": "7c28e5000282137583839624f97d731b43637d8f2a49cd8bbfb5e25bcb6e47dc", "seed": 0, "source_commit": "764ae3aa778cbc9030e286fabc3777e25c18f403", "source_commit_from": "argument", "source_tree_clean": null, - "stage_seconds": 1.1, + "stage_seconds": 1.9, "time_period": 2024, "weights": "HSUP_WGT * 0.01 by PH_SEQ, each file rescaled to one third of income year 2024's weighted person population (asec_pool.pool_asec_sources, anchor_year=2024)" } diff --git a/docs/evidence/us-immigration-statuses-767/receipt.main_full_keys.json b/docs/evidence/us-immigration-statuses-767/receipt.main_full_keys.json index 8cf263c5c7..3d6b9c1740 100644 --- a/docs/evidence/us-immigration-statuses-767/receipt.main_full_keys.json +++ b/docs/evidence/us-immigration-statuses-767/receipt.main_full_keys.json @@ -106,15 +106,17 @@ } }, "error": null, + "gate_failures": [], + "gate_passed": true, "immigration_module_sha256": "8871df8b5f86f6bbf8541259a69d52b7db214560e5dacf5ad6db15268368f826", "none_ssn_iff_undocumented": true, "rows": 421119, - "script_sha256": "6d1a8899979617dc3d53ec1d776a1825f0d55caf00abfe037e806aebbdc8d8eb", + "script_sha256": "7c28e5000282137583839624f97d731b43637d8f2a49cd8bbfb5e25bcb6e47dc", "seed": 0, "source_commit": "764ae3aa778cbc9030e286fabc3777e25c18f403", "source_commit_from": "argument", "source_tree_clean": null, - "stage_seconds": 1.2, + "stage_seconds": 2.2, "time_period": 2024, "weights": "HSUP_WGT * 0.01 by PH_SEQ, each file rescaled to one third of income year 2024's weighted person population (asec_pool.pool_asec_sources, anchor_year=2024)" } diff --git a/docs/evidence/us-immigration-statuses-767/receipt.pre_sync_42bb166b.json b/docs/evidence/us-immigration-statuses-767/receipt.pre_sync_42bb166b.json index 984299b72d..1f9c346e6d 100644 --- a/docs/evidence/us-immigration-statuses-767/receipt.pre_sync_42bb166b.json +++ b/docs/evidence/us-immigration-statuses-767/receipt.pre_sync_42bb166b.json @@ -35,7 +35,7 @@ }, "error": "SourceRuntimeError: PEINUSYR carries value(s) outside the 2024 ASEC domain 0..28: [29.0].", "immigration_module_sha256": "df6699344c5fddbb5322baeb22f574be846e43c8c7600fd5dea798189c382042", - "script_sha256": "6d1a8899979617dc3d53ec1d776a1825f0d55caf00abfe037e806aebbdc8d8eb", + "script_sha256": "7c28e5000282137583839624f97d731b43637d8f2a49cd8bbfb5e25bcb6e47dc", "seed": 0, "source_commit": "42bb166b1484df036809814434c1acf862b8cbb8", "source_commit_from": "argument", diff --git a/docs/evidence/us-immigration-statuses-767/receipt.stacked_supply.json b/docs/evidence/us-immigration-statuses-767/receipt.stacked_supply.json new file mode 100644 index 0000000000..19b07a97a9 --- /dev/null +++ b/docs/evidence/us-immigration-statuses-767/receipt.stacked_supply.json @@ -0,0 +1,217 @@ +{ + "acs": { + "csv_hus.zip": "8281008e53de98f0ef81e7a2ee5a8725991dda1ecfd2713ead73246425e515d0", + "csv_pus.zip": "afdc6d90c6e2f0bab365ed32d95ba4c4d8ac651162f46ac7861295b2dc469894", + "household_mass": 141125706.0, + "person_mass": 334342004.0 + }, + "acs_arm_person_mass": 159947952.85979113, + "acs_arm_weight_scale": 0.4783962258591688, + "afghan_noncitizen_arrivals_at_acs_weights": { + "2021-2022": 71681.0, + "2021-2024": 110913.0 + }, + "arm_household_share": 0.5, + "asec_archives": [ + { + "archive": "asecpub24csv.zip", + "household_mass": 132391925.44999999, + "income_year": 2023, + "person_mass": 320890854.43000007, + "sha256": "cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09" + }, + { + "archive": "asecpub25csv.zip", + "household_mass": 134963892.8, + "income_year": 2024, + "person_mass": 325952369.06, + "sha256": "318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b" + }, + { + "archive": "asecpub26csv.zip", + "household_mass": 137123796.5, + "income_year": 2025, + "person_mass": 329518151.84999996, + "sha256": "fe819d0d2fc4470c282e76c247b8aa8b6054811619730307e9f2fe6186707d93" + } + ], + "asec_arm_person_mass": 162976184.53, + "asec_pool_household_mass": 135028010.2442213, + "asec_pool_person_mass": 325952369.06, + "composition_band_floor": 0.35, + "draws": { + "asylee": { + "acs_arm_supply_bound": 2292044.6361100897, + "acs_candidates_at_acs_weights": 4791101.0, + "acs_residual": 76773.24121281803, + "asec_arm_population": 78226.75878718197, + "asec_arm_share_of_target": 0.5046887663689159, + "asec_stage_population": 155491.90244515426, + "earlier_draws_worst_case_removal": 198844.79859674792, + "projected_share_of_target": 1.0, + "shortfall_at_bound": 0.0, + "target": 155000.0, + "worst_case_share_of_target": 0.5046887663689159 + }, + "deportation_withheld": { + "acs_arm_supply_bound": 6685225.110438908, + "acs_candidates_at_acs_weights": 13974243.0, + "acs_residual": 0.0, + "asec_arm_population": 0.0, + "asec_arm_share_of_target": null, + "asec_stage_population": 0.0, + "earlier_draws_worst_case_removal": 76773.24121281803, + "projected_share_of_target": null, + "shortfall_at_bound": 0.0, + "target": 0.0, + "worst_case_share_of_target": null + }, + "paroled_one_year:afghanistan": { + "acs_arm_supply_bound": 34291.919865811076, + "acs_candidates_at_acs_weights": 71681.0, + "acs_residual": 36438.06621330433, + "asec_arm_population": 37127.93378669567, + "asec_arm_share_of_target": 0.5046887663689159, + "asec_stage_population": 74154.29223282929, + "earlier_draws_worst_case_removal": 0, + "projected_share_of_target": 0.9708269261956168, + "shortfall_at_bound": 2146.1463474932534, + "target": 73566.0, + "worst_case_share_of_target": 0.9708269261956168 + }, + "paroled_one_year:nicaragua": { + "acs_arm_supply_bound": 27188.21430802828, + "acs_candidates_at_acs_weights": 56832.0, + "acs_residual": 46098.616514044996, + "asec_arm_population": 46971.383485955004, + "asec_arm_share_of_target": 0.5046887663689159, + "asec_stage_population": 93112.73319928072, + "earlier_draws_worst_case_removal": 0.0, + "projected_share_of_target": 0.7968152766088243, + "shortfall_at_bound": 18910.402206016715, + "target": 93070.0, + "worst_case_share_of_target": 0.7968152766088243 + }, + "paroled_one_year:ukraine": { + "acs_arm_supply_bound": 57045.87955635072, + "acs_candidates_at_acs_weights": 119244.0, + "acs_residual": 101533.13819319058, + "asec_arm_population": 56466.86180680942, + "asec_arm_share_of_target": 0.3573852013089204, + "asec_stage_population": 112933.72361361884, + "earlier_draws_worst_case_removal": 0.0, + "projected_share_of_target": 0.718435071918735, + "shortfall_at_bound": 44487.258636839855, + "target": 158000.0, + "worst_case_share_of_target": 0.718435071918735 + }, + "paroled_one_year:venezuela": { + "acs_arm_supply_bound": 152917.91840720573, + "acs_candidates_at_acs_weights": 319647.0, + "acs_residual": 58114.8670419351, + "asec_arm_population": 59215.1329580649, + "asec_arm_share_of_target": 0.5046887663689159, + "asec_stage_population": 117688.41694796596, + "earlier_draws_worst_case_removal": 0.0, + "projected_share_of_target": 1.0, + "shortfall_at_bound": 0.0, + "target": 117330.0, + "worst_case_share_of_target": 1.0 + }, + "refugee": { + "acs_arm_supply_bound": 363808.8482564773, + "acs_candidates_at_acs_weights": 760476.0, + "acs_residual": 79249.79738097345, + "asec_arm_population": 80750.20261902655, + "asec_arm_share_of_target": 0.5046887663689159, + "asec_stage_population": 160451.65727945004, + "earlier_draws_worst_case_removal": 123713.9927204219, + "projected_share_of_target": 1.0, + "shortfall_at_bound": 0.0, + "target": 160000.0, + "worst_case_share_of_target": 0.5046887663689159 + }, + "tps:el_salvador": { + "acs_arm_supply_bound": 114363.48816898945, + "acs_candidates_at_acs_weights": 239056.0, + "acs_residual": 84264.82362148818, + "asec_arm_population": 85860.17637851182, + "asec_arm_share_of_target": 0.5046887663689159, + "asec_stage_population": 170410.32409187898, + "earlier_draws_worst_case_removal": 0.0, + "projected_share_of_target": 1.0, + "shortfall_at_bound": 0.0, + "target": 170125.0, + "worst_case_share_of_target": 1.0 + }, + "tps:honduras": { + "acs_arm_supply_bound": 22196.628087413716, + "acs_candidates_at_acs_weights": 46398.0, + "acs_residual": 25372.31794275228, + "asec_arm_population": 25852.68205724772, + "asec_arm_share_of_target": 0.5046887663689159, + "asec_stage_population": 52053.03129856825, + "earlier_draws_worst_case_removal": 0.0, + "projected_share_of_target": 0.9380050784706966, + "shortfall_at_bound": 3175.6898553385654, + "target": 51225.0, + "worst_case_share_of_target": 0.9380050784706966 + }, + "tps:nepal": { + "acs_arm_supply_bound": 16985.936395355646, + "acs_candidates_at_acs_weights": 35506.0, + "acs_residual": 3546.428432798562, + "asec_arm_population": 3613.571567201438, + "asec_arm_share_of_target": 0.5046887663689159, + "asec_stage_population": 7349.0149911888375, + "earlier_draws_worst_case_removal": 0.0, + "projected_share_of_target": 1.0, + "shortfall_at_bound": 0.0, + "target": 7160.0, + "worst_case_share_of_target": 1.0 + }, + "tps:nicaragua": { + "acs_arm_supply_bound": 6960.186690025047, + "acs_candidates_at_acs_weights": 14549.0, + "acs_residual": 1441.3556898664547, + "asec_arm_population": 1468.6443101335453, + "asec_arm_share_of_target": 0.5046887663689159, + "asec_stage_population": 3192.9463618784666, + "earlier_draws_worst_case_removal": 0.0, + "projected_share_of_target": 1.0, + "shortfall_at_bound": 0.0, + "target": 2910.0, + "worst_case_share_of_target": 1.0 + }, + "tps:other_designated": { + "acs_arm_supply_bound": 141412.9675715186, + "acs_candidates_at_acs_weights": 295598.0, + "acs_residual": 10508.027821483449, + "asec_arm_population": 10706.972178516551, + "asec_arm_share_of_target": 0.5046887663689159, + "asec_stage_population": 21484.827706756125, + "earlier_draws_worst_case_removal": 49690.537583766, + "projected_share_of_target": 1.0, + "shortfall_at_bound": 0.0, + "target": 21215.0, + "worst_case_share_of_target": 1.0 + }, + "tps:venezuela": { + "acs_arm_supply_bound": 338593.5399838922, + "acs_candidates_at_acs_weights": 707768.0, + "acs_residual": 299670.7260153103, + "asec_arm_population": 305344.2739846897, + "asec_arm_share_of_target": 0.5046887663689159, + "asec_stage_population": 605629.9639509566, + "earlier_draws_worst_case_removal": 214137.90563572658, + "projected_share_of_target": 1.0, + "shortfall_at_bound": 0.0, + "target": 605015.0, + "worst_case_share_of_target": 0.7103954585140125 + } + }, + "lowest_worst_case_share_of_target": 0.5046887663689159, + "receipt_file": "receipt.head.json", + "receipt_sha256": "d9bdd7e626acaf9638c8c696f65ce6b9ba377263eff752dc128498c0abe30a2f", + "script_sha256": "7b3490eef29ce1207fd03c08f7d191bf11a6c2b7821aa181679edb9e553aec3a" +} From 5610bde70df07c30e873fbd906667766f731508b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 10 Nov 2026 17:06:59 -0500 Subject: [PATCH 45/49] State which stacked draws may run short in the #767 evidence README The stacked-supply section listed four draws that run short of ACS candidates without saying the projected share assumes no earlier draw takes a draw's candidates. Define the projected share, note that none of the four shares a candidate with an earlier draw, and add TPS Venezuela: its supply exceeds its residual by 38,923, but earlier draws could take up to 214,138 of its candidates, which would leave it at 0.710 of its target (receipt.stacked_supply.json, worst_case_share_of_target). Refs #767 Co-Authored-By: Claude Opus 5.5 --- .../us-immigration-statuses-767/README.md | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/docs/evidence/us-immigration-statuses-767/README.md b/docs/evidence/us-immigration-statuses-767/README.md index b7c5e4df71..9f5225b2c7 100644 --- a/docs/evidence/us-immigration-statuses-767/README.md +++ b/docs/evidence/us-immigration-statuses-767/README.md @@ -47,7 +47,7 @@ of its `immigration.py` and of the script, and every draw call's key. | `receipt.main_full_keys.json` | main at `764ae3aa7` | this PR's, through `--draw-keys full` | main's method on this PR's draws; its gate passes | | `receipt.pre_sync_42bb166b.json` | this PR at `42bb166b1`, before it merged main `1acd884fe` | none: the stage refuses before its first draw | the stage refuses the 2026 ASEC's `PEINUSYR` code 29 | | `receipt.head_medicaid_reporters.json` | this PR at `4f94e489a`, from the head run's labels | its own | 0.391 million of the 0.714 million people in the restored Medicaid statuses report Medicaid at interview | -| `receipt.stacked_supply.json` | this PR at `4f94e489a`, from `receipt.head.json` and the ACS 2024 1-year PUMS | none: it draws no rows | four draws run short of ACS candidates, and none ends below half its target in the worst case | +| `receipt.stacked_supply.json` | this PR at `4f94e489a`, from `receipt.head.json` and the ACS 2024 1-year PUMS | none: it draws no rows | four draws run short of ACS candidates and a fifth may; none ends below half its target in the worst case | The script reran main's tree and the pre-sync tree from `git archive packages`, so those three receipts take their commit from `--source-commit`. @@ -128,7 +128,10 @@ stacked pool. It takes the ASEC arm's labelled mass from `receipt.head.json`, scaled to the stacked weights, and each draw's candidates from the ACS 2024 1-year PUMS through the kernel's own candidate rule, weighted as the frame weights them and scaled by the ACS arm's weight scale, 0.478. -`receipt.stacked_supply.json` holds the aggregates, in weighted persons: +`receipt.stacked_supply.json` holds the aggregates, in weighted persons. The +projected share adds to the ASEC arm as much of the ACS supply as the residual +takes, as a share of the target, if no earlier draw takes any of the draw's +candidates. The worst case lets earlier draws take them (below): | Draw | Target | ASEC arm | ACS residual | ACS supply | Projected share | Worst case | |---|---:|---:|---:|---:|---:|---:| @@ -146,9 +149,14 @@ weights them and scaled by the ACS arm's weight scale, 0.478. | TPS, other designated | 21,215 | 10,707 | 10,508 | 141,413 | 1.000 | 1.000 | - Four draws run short of candidates: U4U, Nicaraguan parole, Honduran TPS and - OAW. Under the old rule, U4U, the second draw in manifest order, would have - stopped every stacked build on these inputs. Under the new rule the four end - at 0.718 to 0.971 of their targets. + OAW. None of the four shares a candidate with an earlier draw, so each ends + at its projected share in the worst case too. Under the old rule, U4U, the + second draw in manifest order, would have stopped every stacked build on + these inputs. Under the new rule the four end at 0.718 to 0.971 of their + targets. +- TPS Venezuela may run short as well. Its supply exceeds its residual by + 38,923, but earlier draws could take up to 214,138 of its candidates, which + would leave it at 0.710 of its target. - The worst case lets every earlier draw take as much of a draw's candidates as it can, up to its own residual. It holds for parole and TPS draws, which take any non-citizen SSN code. Refugee and asylee draws take only rows the transfer From a5a1dda0b1e66690eec9a1d49f2248d0d0150ac7 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 10 Nov 2026 17:50:07 -0500 Subject: [PATCH 46/49] Project each stacked draw's expected share; count TPS Venezuela as short The third delta review of #779 (wf_06fd9b1f-5b7) confirmed two defects in the stacked-supply prose. Honduran TPS does share candidates with an earlier draw: every one also falls in the deportation-withheld draw's pool, whose zero target takes none, so its worst case still equals its projected share. And TPS Venezuela does not just "may" run short: Venezuelan parole, the fourth draw, takes 58,115 from a pool that shares 0.848 of its mass with TPS Venezuela's, about 49,300 of it, more than TPS Venezuela's 38,923 margin. stacked_supply.py now projects an expected share for each parole and TPS draw: each earlier parole or TPS draw takes its residual from the candidates still free, each with the same chance, as the reconciliation's person-keyed hash draws do. It leaves out refugee and asylee takes, which PUMS cannot show. TPS Venezuela's expected share is 0.983; every other draw's equals its projected share. The receipt gains that one field per draw and the script's new hash; every other value is unchanged. The README table gains the column, and its bullets count five short draws. Refs #767 Co-Authored-By: Claude Opus 5.5 --- .../us-immigration-statuses-767/README.md | 66 +++++++++++-------- .../receipt.stacked_supply.json | 15 ++++- .../scripts/stacked_supply.py | 35 +++++++++- 3 files changed, 87 insertions(+), 29 deletions(-) diff --git a/docs/evidence/us-immigration-statuses-767/README.md b/docs/evidence/us-immigration-statuses-767/README.md index 9f5225b2c7..1edea9608d 100644 --- a/docs/evidence/us-immigration-statuses-767/README.md +++ b/docs/evidence/us-immigration-statuses-767/README.md @@ -47,7 +47,7 @@ of its `immigration.py` and of the script, and every draw call's key. | `receipt.main_full_keys.json` | main at `764ae3aa7` | this PR's, through `--draw-keys full` | main's method on this PR's draws; its gate passes | | `receipt.pre_sync_42bb166b.json` | this PR at `42bb166b1`, before it merged main `1acd884fe` | none: the stage refuses before its first draw | the stage refuses the 2026 ASEC's `PEINUSYR` code 29 | | `receipt.head_medicaid_reporters.json` | this PR at `4f94e489a`, from the head run's labels | its own | 0.391 million of the 0.714 million people in the restored Medicaid statuses report Medicaid at interview | -| `receipt.stacked_supply.json` | this PR at `4f94e489a`, from `receipt.head.json` and the ACS 2024 1-year PUMS | none: it draws no rows | four draws run short of ACS candidates and a fifth may; none ends below half its target in the worst case | +| `receipt.stacked_supply.json` | this PR at `4f94e489a`, from `receipt.head.json` and the ACS 2024 1-year PUMS | none: it draws no rows | five draws run short of ACS candidates; none ends below half its target in the worst case | The script reran main's tree and the pre-sync tree from `git archive packages`, so those three receipts take their commit from `--source-commit`. @@ -131,32 +131,44 @@ weights them and scaled by the ACS arm's weight scale, 0.478. `receipt.stacked_supply.json` holds the aggregates, in weighted persons. The projected share adds to the ASEC arm as much of the ACS supply as the residual takes, as a share of the target, if no earlier draw takes any of the draw's -candidates. The worst case lets earlier draws take them (below): - -| Draw | Target | ASEC arm | ACS residual | ACS supply | Projected share | Worst case | -|---|---:|---:|---:|---:|---:|---:| -| Parole, Afghanistan (OAW) | 73,566 | 37,128 | 36,438 | 34,292 | 0.971 | 0.971 | -| Parole, Ukraine (U4U) | 158,000 | 56,467 | 101,533 | 57,046 | 0.718 | 0.718 | -| Parole, Nicaragua | 93,070 | 46,971 | 46,099 | 27,188 | 0.797 | 0.797 | -| Parole, Venezuela | 117,330 | 59,215 | 58,115 | 152,918 | 1.000 | 1.000 | -| Refugee | 160,000 | 80,750 | 79,250 | 363,809 | 1.000 | 0.505 | -| Asylee | 155,000 | 78,227 | 76,773 | 2,292,045 | 1.000 | 0.505 | -| TPS, Venezuela | 605,015 | 305,344 | 299,671 | 338,594 | 1.000 | 0.710 | -| TPS, El Salvador | 170,125 | 85,860 | 84,265 | 114,363 | 1.000 | 1.000 | -| TPS, Honduras | 51,225 | 25,853 | 25,372 | 22,197 | 0.938 | 0.938 | -| TPS, Nicaragua | 2,910 | 1,469 | 1,441 | 6,960 | 1.000 | 1.000 | -| TPS, Nepal | 7,160 | 3,614 | 3,546 | 16,986 | 1.000 | 1.000 | -| TPS, other designated | 21,215 | 10,707 | 10,508 | 141,413 | 1.000 | 1.000 | - -- Four draws run short of candidates: U4U, Nicaraguan parole, Honduran TPS and - OAW. None of the four shares a candidate with an earlier draw, so each ends - at its projected share in the worst case too. Under the old rule, U4U, the - second draw in manifest order, would have stopped every stacked build on - these inputs. Under the new rule the four end at 0.718 to 0.971 of their - targets. -- TPS Venezuela may run short as well. Its supply exceeds its residual by - 38,923, but earlier draws could take up to 214,138 of its candidates, which - would leave it at 0.710 of its target. +candidates. The expected share lets each earlier parole or TPS draw take its +residual from the candidates still free, each with the same chance, as the +reconciliation's person-keyed hash draws do; it leaves out what refugee and +asylee draws take. The worst case lets earlier draws take as much as they can +(below): + +| Draw | Target | ASEC arm | ACS residual | ACS supply | Projected share | Expected share | Worst case | +|---|---:|---:|---:|---:|---:|---:|---:| +| Parole, Afghanistan (OAW) | 73,566 | 37,128 | 36,438 | 34,292 | 0.971 | 0.971 | 0.971 | +| Parole, Ukraine (U4U) | 158,000 | 56,467 | 101,533 | 57,046 | 0.718 | 0.718 | 0.718 | +| Parole, Nicaragua | 93,070 | 46,971 | 46,099 | 27,188 | 0.797 | 0.797 | 0.797 | +| Parole, Venezuela | 117,330 | 59,215 | 58,115 | 152,918 | 1.000 | 1.000 | 1.000 | +| Refugee | 160,000 | 80,750 | 79,250 | 363,809 | 1.000 | n/a | 0.505 | +| Asylee | 155,000 | 78,227 | 76,773 | 2,292,045 | 1.000 | n/a | 0.505 | +| TPS, Venezuela | 605,015 | 305,344 | 299,671 | 338,594 | 1.000 | 0.983 | 0.710 | +| TPS, El Salvador | 170,125 | 85,860 | 84,265 | 114,363 | 1.000 | 1.000 | 1.000 | +| TPS, Honduras | 51,225 | 25,853 | 25,372 | 22,197 | 0.938 | 0.938 | 0.938 | +| TPS, Nicaragua | 2,910 | 1,469 | 1,441 | 6,960 | 1.000 | 1.000 | 1.000 | +| TPS, Nepal | 7,160 | 3,614 | 3,546 | 16,986 | 1.000 | 1.000 | 1.000 | +| TPS, other designated | 21,215 | 10,707 | 10,508 | 141,413 | 1.000 | 1.000 | 1.000 | + +- Five draws run short of candidates. Four of them, U4U, Nicaraguan parole, + Honduran TPS and OAW, run short even if no earlier draw takes theirs. No + earlier draw that takes any rows shares a candidate with them: OAW comes + first, U4U's and Nicaraguan parole's candidates overlap no earlier draw's, + and Honduran TPS's all fall inside the deportation-withheld draw's, whose + zero target takes none. So each of the four ends at its projected share in + the worst case too, 0.718 to 0.971 of its target. Under the old rule, U4U, + the second draw in manifest order, would have stopped every stacked build on + these inputs. +- TPS Venezuela runs short because of an earlier draw. Its supply exceeds its + residual by 38,923, but Venezuelan parole, the fourth draw, takes its 58,115 + from non-citizens born in Venezuela who arrived in 2022 or later, and 0.848 + of that supply, those who arrived in 2022 or 2023, are TPS Venezuela's + candidates too. The parole draw takes an expected 49,300 of them, more than + the margin, so TPS Venezuela ends at an expected 0.983 of its target before + refugee and asylee draws take any of its candidates, and at 0.710 in the + worst case. - The worst case lets every earlier draw take as much of a draw's candidates as it can, up to its own residual. It holds for parole and TPS draws, which take any non-citizen SSN code. Refugee and asylee draws take only rows the transfer diff --git a/docs/evidence/us-immigration-statuses-767/receipt.stacked_supply.json b/docs/evidence/us-immigration-statuses-767/receipt.stacked_supply.json index 19b07a97a9..9e40068e27 100644 --- a/docs/evidence/us-immigration-statuses-767/receipt.stacked_supply.json +++ b/docs/evidence/us-immigration-statuses-767/receipt.stacked_supply.json @@ -48,6 +48,7 @@ "asec_arm_share_of_target": 0.5046887663689159, "asec_stage_population": 155491.90244515426, "earlier_draws_worst_case_removal": 198844.79859674792, + "expected_share_of_target": null, "projected_share_of_target": 1.0, "shortfall_at_bound": 0.0, "target": 155000.0, @@ -61,6 +62,7 @@ "asec_arm_share_of_target": null, "asec_stage_population": 0.0, "earlier_draws_worst_case_removal": 76773.24121281803, + "expected_share_of_target": null, "projected_share_of_target": null, "shortfall_at_bound": 0.0, "target": 0.0, @@ -74,6 +76,7 @@ "asec_arm_share_of_target": 0.5046887663689159, "asec_stage_population": 74154.29223282929, "earlier_draws_worst_case_removal": 0, + "expected_share_of_target": 0.9708269261956168, "projected_share_of_target": 0.9708269261956168, "shortfall_at_bound": 2146.1463474932534, "target": 73566.0, @@ -87,6 +90,7 @@ "asec_arm_share_of_target": 0.5046887663689159, "asec_stage_population": 93112.73319928072, "earlier_draws_worst_case_removal": 0.0, + "expected_share_of_target": 0.7968152766088243, "projected_share_of_target": 0.7968152766088243, "shortfall_at_bound": 18910.402206016715, "target": 93070.0, @@ -100,6 +104,7 @@ "asec_arm_share_of_target": 0.3573852013089204, "asec_stage_population": 112933.72361361884, "earlier_draws_worst_case_removal": 0.0, + "expected_share_of_target": 0.718435071918735, "projected_share_of_target": 0.718435071918735, "shortfall_at_bound": 44487.258636839855, "target": 158000.0, @@ -113,6 +118,7 @@ "asec_arm_share_of_target": 0.5046887663689159, "asec_stage_population": 117688.41694796596, "earlier_draws_worst_case_removal": 0.0, + "expected_share_of_target": 1.0, "projected_share_of_target": 1.0, "shortfall_at_bound": 0.0, "target": 117330.0, @@ -126,6 +132,7 @@ "asec_arm_share_of_target": 0.5046887663689159, "asec_stage_population": 160451.65727945004, "earlier_draws_worst_case_removal": 123713.9927204219, + "expected_share_of_target": null, "projected_share_of_target": 1.0, "shortfall_at_bound": 0.0, "target": 160000.0, @@ -139,6 +146,7 @@ "asec_arm_share_of_target": 0.5046887663689159, "asec_stage_population": 170410.32409187898, "earlier_draws_worst_case_removal": 0.0, + "expected_share_of_target": 1.0, "projected_share_of_target": 1.0, "shortfall_at_bound": 0.0, "target": 170125.0, @@ -152,6 +160,7 @@ "asec_arm_share_of_target": 0.5046887663689159, "asec_stage_population": 52053.03129856825, "earlier_draws_worst_case_removal": 0.0, + "expected_share_of_target": 0.9380050784706966, "projected_share_of_target": 0.9380050784706966, "shortfall_at_bound": 3175.6898553385654, "target": 51225.0, @@ -165,6 +174,7 @@ "asec_arm_share_of_target": 0.5046887663689159, "asec_stage_population": 7349.0149911888375, "earlier_draws_worst_case_removal": 0.0, + "expected_share_of_target": 1.0, "projected_share_of_target": 1.0, "shortfall_at_bound": 0.0, "target": 7160.0, @@ -178,6 +188,7 @@ "asec_arm_share_of_target": 0.5046887663689159, "asec_stage_population": 3192.9463618784666, "earlier_draws_worst_case_removal": 0.0, + "expected_share_of_target": 1.0, "projected_share_of_target": 1.0, "shortfall_at_bound": 0.0, "target": 2910.0, @@ -191,6 +202,7 @@ "asec_arm_share_of_target": 0.5046887663689159, "asec_stage_population": 21484.827706756125, "earlier_draws_worst_case_removal": 49690.537583766, + "expected_share_of_target": 1.0, "projected_share_of_target": 1.0, "shortfall_at_bound": 0.0, "target": 21215.0, @@ -204,6 +216,7 @@ "asec_arm_share_of_target": 0.5046887663689159, "asec_stage_population": 605629.9639509566, "earlier_draws_worst_case_removal": 214137.90563572658, + "expected_share_of_target": 0.9828393132859046, "projected_share_of_target": 1.0, "shortfall_at_bound": 0.0, "target": 605015.0, @@ -213,5 +226,5 @@ "lowest_worst_case_share_of_target": 0.5046887663689159, "receipt_file": "receipt.head.json", "receipt_sha256": "d9bdd7e626acaf9638c8c696f65ce6b9ba377263eff752dc128498c0abe30a2f", - "script_sha256": "7b3490eef29ce1207fd03c08f7d191bf11a6c2b7821aa181679edb9e553aec3a" + "script_sha256": "369e9ad1da6e60ac83b6e0e40ea4a7bf965d04f5a8572c0edb4cf478056acc14" } diff --git a/docs/evidence/us-immigration-statuses-767/scripts/stacked_supply.py b/docs/evidence/us-immigration-statuses-767/scripts/stacked_supply.py index d793025a6f..0e9538d67d 100644 --- a/docs/evidence/us-immigration-statuses-767/scripts/stacked_supply.py +++ b/docs/evidence/us-immigration-statuses-767/scripts/stacked_supply.py @@ -46,6 +46,19 @@ ``OTHER_NON_CITIZEN``, which PUMS cannot show, so its worst case is its ASEC arm share. +One estimate sits between them, for parole and TPS draws only: + +- ``expected_share_of_target``: the ASEC arm plus the ACS mass the draw + expects to take when each earlier parole or TPS draw takes its residual + from the candidates no earlier draw took. The reconciliation selects by + person-keyed hash draws, which order the free candidates as a uniform + random order would, so each free candidate of a draw that takes mass + ``take`` from free mass ``available`` stays free with probability + ``1 - take / available``. Refugee, asylee and withholding draws take only + ``OTHER_NON_CITIZEN`` rows, which PUMS cannot show, so the estimate leaves + their takes out and overstates a later draw's share by whatever they take + of its candidates. + The output holds aggregates only. """ @@ -218,6 +231,8 @@ def _read_pums(path: Path, token: str, usecols: list[str]) -> pd.DataFrame: draws = {} # Each earlier draw's candidates and residual, in manifest order. earlier: list[tuple[np.ndarray, float]] = [] +# Each ACS person's probability that no earlier parole or TPS draw took them. +free = np.ones(len(acs_weight), dtype=np.float64) for draw in immigration.us_immigration_controls().humanitarian: target = float(draw.target) stage = float(achieved[draw.label]["population"]) @@ -240,6 +255,13 @@ def _read_pums(path: Path, token: str, usecols: list[str]) -> pd.DataFrame: worst_supply = max(0.0, supply - removed) exact_ssn_rule = draw.category in {"paroled_one_year", "tps"} worst_reach = immutable + (min(worst_supply, residual) if exact_ssn_rule else 0.0) + expected_reach = None + if exact_ssn_rule: + available = kappa * float((acs_weight * free)[candidates].sum()) + take = min(residual, available) + if available > 0: + free[candidates] *= 1.0 - take / available + expected_reach = immutable + take draws[draw.label] = { "target": target, "asec_stage_population": stage, @@ -254,12 +276,23 @@ def _read_pums(path: Path, token: str, usecols: list[str]) -> pd.DataFrame: ), "asec_arm_share_of_target": immutable / target if target > 0 else None, "worst_case_share_of_target": worst_reach / target if target > 0 else None, + "expected_share_of_target": ( + expected_reach / target + if expected_reach is not None and target > 0 + else None + ), } print( f"{draw.label:32s} target {target:9,.0f} ASEC arm {immutable:9,.0f} " f"residual {residual:9,.0f} ACS bound {supply:9,.0f} " f"short {max(0.0, residual - supply):9,.0f} " - f"worst-case share {worst_reach / target if target > 0 else float('nan'):.3f}" + f"worst-case share {worst_reach / target if target > 0 else float('nan'):.3f} " + "expected share " + + ( + f"{expected_reach / target:.3f}" + if expected_reach is not None and target > 0 + else "n/a" + ) ) positive = [entry for entry in draws.values() if entry["target"] > 0] lowest_worst_case = min(entry["worst_case_share_of_target"] for entry in positive) From d12edbd051982cbce3546fbde77ce3eabaa31a59 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 10 Nov 2026 18:07:31 -0500 Subject: [PATCH 47/49] Place the expected share between the worst case and the projection The delta review of a5a1dda0b (wf_78421e89-31e) found two docstring claims in stacked_supply.py that its code contradicts. The expected share sits between the worst case and the projected share, not between the two lower bounds: for TPS Venezuela it is 0.983 against a worst case of 0.710. And leaving refugee, asylee and withholding takes out does not overstate a later draw's share by all they take of its candidates: those takes can only lower the share, and only once the draw's free supply falls below its residual, as TPS Venezuela's does. The receipt changes only in script_sha256. Refs #767 Co-Authored-By: Claude Opus 5.5 --- .../receipt.stacked_supply.json | 2 +- .../scripts/stacked_supply.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/evidence/us-immigration-statuses-767/receipt.stacked_supply.json b/docs/evidence/us-immigration-statuses-767/receipt.stacked_supply.json index 9e40068e27..61fea0c103 100644 --- a/docs/evidence/us-immigration-statuses-767/receipt.stacked_supply.json +++ b/docs/evidence/us-immigration-statuses-767/receipt.stacked_supply.json @@ -226,5 +226,5 @@ "lowest_worst_case_share_of_target": 0.5046887663689159, "receipt_file": "receipt.head.json", "receipt_sha256": "d9bdd7e626acaf9638c8c696f65ce6b9ba377263eff752dc128498c0abe30a2f", - "script_sha256": "369e9ad1da6e60ac83b6e0e40ea4a7bf965d04f5a8572c0edb4cf478056acc14" + "script_sha256": "589da21466cd18ad6df9df313009c0208daad007d1bdd225b67698539d1ab056" } diff --git a/docs/evidence/us-immigration-statuses-767/scripts/stacked_supply.py b/docs/evidence/us-immigration-statuses-767/scripts/stacked_supply.py index 0e9538d67d..725a7e7735 100644 --- a/docs/evidence/us-immigration-statuses-767/scripts/stacked_supply.py +++ b/docs/evidence/us-immigration-statuses-767/scripts/stacked_supply.py @@ -46,18 +46,20 @@ ``OTHER_NON_CITIZEN``, which PUMS cannot show, so its worst case is its ASEC arm share. -One estimate sits between them, for parole and TPS draws only: +One estimate sits between the worst case and the projected share, for parole +and TPS draws only: - ``expected_share_of_target``: the ASEC arm plus the ACS mass the draw expects to take when each earlier parole or TPS draw takes its residual from the candidates no earlier draw took. The reconciliation selects by person-keyed hash draws, which order the free candidates as a uniform random order would, so each free candidate of a draw that takes mass - ``take`` from free mass ``available`` stays free with probability + ``take`` from free mass ``available`` stays free with probability about ``1 - take / available``. Refugee, asylee and withholding draws take only ``OTHER_NON_CITIZEN`` rows, which PUMS cannot show, so the estimate leaves - their takes out and overstates a later draw's share by whatever they take - of its candidates. + their takes out. Those takes can only lower a later draw's share, and they + lower it only once its free supply falls below its residual, as TPS + Venezuela's does. The output holds aggregates only. """ From 1f0009cbeddb87c04389bec4dea01173e946ad1b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 10 Nov 2026 18:15:44 -0500 Subject: [PATCH 48/49] Bound the refugee and asylee caveat by the crossing tie group The confirmation of d12edbd05 (wf_abe4b4fc-c03) found that a draw that does not exhaust its candidates also takes the tie group that crosses its residual, and a refugee, asylee or withholding take can move that overshoot either way, by less than one row's weight. The docstring now takes the reviewer's wording: those takes can only lower a later draw's share, up to the weight of that tie group, and lower it by more only once the draw's free supply falls below its residual. The receipt changes only in script_sha256. Refs #767 Co-Authored-By: Claude Opus 5.5 --- .../receipt.stacked_supply.json | 2 +- .../us-immigration-statuses-767/scripts/stacked_supply.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/evidence/us-immigration-statuses-767/receipt.stacked_supply.json b/docs/evidence/us-immigration-statuses-767/receipt.stacked_supply.json index 61fea0c103..dcfe67754b 100644 --- a/docs/evidence/us-immigration-statuses-767/receipt.stacked_supply.json +++ b/docs/evidence/us-immigration-statuses-767/receipt.stacked_supply.json @@ -226,5 +226,5 @@ "lowest_worst_case_share_of_target": 0.5046887663689159, "receipt_file": "receipt.head.json", "receipt_sha256": "d9bdd7e626acaf9638c8c696f65ce6b9ba377263eff752dc128498c0abe30a2f", - "script_sha256": "589da21466cd18ad6df9df313009c0208daad007d1bdd225b67698539d1ab056" + "script_sha256": "0c5718ce093fe583a410c79af40b4e28785014c1aeeb3bf1e0294c3367eda503" } diff --git a/docs/evidence/us-immigration-statuses-767/scripts/stacked_supply.py b/docs/evidence/us-immigration-statuses-767/scripts/stacked_supply.py index 725a7e7735..456c18949a 100644 --- a/docs/evidence/us-immigration-statuses-767/scripts/stacked_supply.py +++ b/docs/evidence/us-immigration-statuses-767/scripts/stacked_supply.py @@ -57,9 +57,10 @@ ``take`` from free mass ``available`` stays free with probability about ``1 - take / available``. Refugee, asylee and withholding draws take only ``OTHER_NON_CITIZEN`` rows, which PUMS cannot show, so the estimate leaves - their takes out. Those takes can only lower a later draw's share, and they - lower it only once its free supply falls below its residual, as TPS - Venezuela's does. + their takes out. Those takes can only lower a later draw's share, up to + the weight of the one tie group that crosses its residual, and they lower + it by more than that only once its free supply falls below its residual, + as TPS Venezuela's does. The output holds aggregates only. """ From 173766c974e20c29672ae261c15c07d191bef83f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 10 Nov 2026 22:12:07 -0500 Subject: [PATCH 49/49] Re-run CI on the pinned runner image (no file changes) The engine-free (3.14) leg of this head's run drew an ubuntu-26.04 runner twice (jobs 130198769545 and 130281464672). uv took that image's Python 3.14.4 there, and the head-to-head scorer's worker-recycling test needs 3.14.7. Main pins every Tests job to ubuntu-24.04 since #1576, and a re-run keeps the old workflow, so this empty commit starts a run that reads main's. Co-Authored-By: Claude Fable 5.2