From 0b15232c2addeae90e112d6a024b5087f7603505 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 05:46:51 -0400 Subject: [PATCH 1/4] Carry SIPP/SCF assets on the declared household head (#1215) us/source_stages.json declares the wealth stage's carry as head_carry with condition is_household_head (P_SEQ == 1, the CPS reference person), and the housing stages use that flag. The runtime selector instead took the lowest A_LINENO. They disagree in 243 of the certified frame's 57,240 households; on the 2024 income year, where A_EXPRRP is observed, is_household_head is the reference person in every household and the line rule misses 73. household_head_carrier_mask (sipp_financial_assets) now uses the flag where it marks exactly one person in a household and falls back to the lowest line, then row order, otherwise; scf_wealth's _household_head_mask calls it, so both halves of the blend carry the same person. Re-pins the attested digests the change moves: seed protocol and seed map, the synthetic and real US spec goldens, and the coverage receipt. Co-Authored-By: Claude Opus 5.5 --- .../us-wealth-carrier-household-head.fixed.md | 1 + docs/evidence/spec-engine/us-f0-coverage.json | 16 ++-- .../build/spec_engine/inventory_coverage.py | 8 +- .../microcosm/build/us_runtime/scf_wealth.py | 23 ++---- .../build/us_runtime/sipp_financial_assets.py | 33 ++++++-- .../shared/test_spec_engine_loader.py | 2 +- .../engine_free/us/test_us_scf_wealth.py | 80 +++++++++++++++++++ .../us/test_us_multispine_pool_tool.py | 2 +- 8 files changed, 130 insertions(+), 35 deletions(-) create mode 100644 changelog.d/us-wealth-carrier-household-head.fixed.md diff --git a/changelog.d/us-wealth-carrier-household-head.fixed.md b/changelog.d/us-wealth-carrier-household-head.fixed.md new file mode 100644 index 0000000000..0f29a77c06 --- /dev/null +++ b/changelog.d/us-wealth-carrier-household-head.fixed.md @@ -0,0 +1 @@ +The SIPP/SCF liquid-asset stage carries each household's draw on the declared `is_household_head` (the CPS reference person, `P_SEQ == 1`), as `us/source_stages.json` declares and the housing stages use, instead of the lowest `A_LINENO` (part of #1215). A household whose flag does not mark exactly one person, or a frame without the flag, keeps the lowest-line rule. The two rules disagree in 243 of the certified frame's 57,240 households; on the 2024 income year, where `A_EXPRRP` is observed, the flag is the reference person in every household and the line rule misses 73. Both halves of the blend share the selector. Future builds only. diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index 978c19dcd8..08a824a87f 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -1656,13 +1656,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "7d03cc07e973165f38781d5932d5fe6d16ca0b1e4dc854e7841460737fd043fb", - "protocol_sha256": "37da8f68444677850a97bdf958b7f62b426a60da570b80a002eeba8af4861f68" + "map_sha256": "e04d6a85c24e536f9350abab3dc0bfed2215c1e4679dd30e449d4034511af9d4", + "protocol_sha256": "7855f2a7d3ef74377791070359bd45da766f371b49d76c1065a183d64fb2cf51" }, "failures": [], "observed": { - "map_sha256": "7d03cc07e973165f38781d5932d5fe6d16ca0b1e4dc854e7841460737fd043fb", - "protocol_sha256": "37da8f68444677850a97bdf958b7f62b426a60da570b80a002eeba8af4861f68" + "map_sha256": "e04d6a85c24e536f9350abab3dc0bfed2215c1e4679dd30e449d4034511af9d4", + "protocol_sha256": "7855f2a7d3ef74377791070359bd45da766f371b49d76c1065a183d64fb2cf51" }, "status": "covered" }, @@ -1677,7 +1677,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "37da8f68444677850a97bdf958b7f62b426a60da570b80a002eeba8af4861f68", + "implementation_sha256": "7855f2a7d3ef74377791070359bd45da766f371b49d76c1065a183d64fb2cf51", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1698,7 +1698,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "37da8f68444677850a97bdf958b7f62b426a60da570b80a002eeba8af4861f68", + "implementation_sha256": "7855f2a7d3ef74377791070359bd45da766f371b49d76c1065a183d64fb2cf51", "protocol": "legacy-v1", "streams": [ "build_model", @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "fbf25dbae68242198972f5bef4f1fc20f65ee0769615b8d94639e6a80121f31f" + "spec_sha256": "651e8c2c5078a26ffa1aa48b842050d74c92cd24510f454af416cffc2ff145c2" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "fbf25dbae68242198972f5bef4f1fc20f65ee0769615b8d94639e6a80121f31f" + "spec_sha256": "651e8c2c5078a26ffa1aa48b842050d74c92cd24510f454af416cffc2ff145c2" }, "status": "pass" } diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 2a38471603..bb9e0f8d2a 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -360,9 +360,11 @@ "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "fdf23da429f8c501e198a2f5b719763d523a41565c2153fdf17b0344f21bcd7c", # #1178: puf_support's source bytes attest CPS Social Security preservation. - # RNG rules and the ordered QRF targets are unchanged. - "seed_map": "7d03cc07e973165f38781d5932d5fe6d16ca0b1e4dc854e7841460737fd043fb", - "seed_protocol": "37da8f68444677850a97bdf958b7f62b426a60da570b80a002eeba8af4861f68", + # #1215: scf_wealth/sipp_financial_assets source bytes attest the declared + # is_household_head carrier. RNG rules and the ordered QRF targets are + # unchanged. + "seed_map": "e04d6a85c24e536f9350abab3dc0bfed2215c1e4679dd30e449d4034511af9d4", + "seed_protocol": "7855f2a7d3ef74377791070359bd45da766f371b49d76c1065a183d64fb2cf51", "source_manifest": "f4f8bfeb79268a4043a843b47cec1d1c0e58e9f5dd6bc23417ada95446dd7b8e", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/scf_wealth.py b/packages/microcosm-build/src/microcosm/build/us_runtime/scf_wealth.py index e77540daae..41947feca4 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/scf_wealth.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/scf_wealth.py @@ -115,6 +115,7 @@ SIPP_FINANCIAL_ASSET_SOURCE_COLUMNS, SIPP_FINANCIAL_ASSET_TARGET_ALLOCATION_COLUMNS, SIPP_FINANCIAL_ASSET_TARGET_SOURCE_COLUMNS, + household_head_carrier_mask, impute_us_sipp_financial_assets, ) from microcosm.frame import Frame @@ -673,25 +674,13 @@ def _sum_present(person: pd.DataFrame, columns: tuple[str, ...]) -> np.ndarray: def _household_head_mask(person: pd.DataFrame) -> np.ndarray: """One reference person (household head) per household. - The head is the lowest-line-number person within each household (the CPS - householder is line 1); ties/absent line numbers fall back to the first - row. This mirrors the retired pipeline's reference-person rule (the first - person of each household). + The declared ``is_household_head`` (``P_SEQ == 1``) where it marks exactly + one person in the household, else the lowest-line-number person, then the + first row. The SIPP side shares this selector so both halves of the blend + carry the same person (microcosm#1215). """ - household = pd.to_numeric(person[_HOUSEHOLD_ID_COLUMN], errors="coerce").to_numpy() - lineno = pd.to_numeric(person["A_LINENO"], errors="coerce").fillna(9_999).to_numpy() - position = np.arange(len(person)) - # Sort by household, then line number, then original position; the first - # row of each household group in that order is its head. - order = np.lexsort((position, lineno, household)) - ordered_household = household[order] - is_first_in_group = np.empty(len(person), dtype=bool) - is_first_in_group[0] = True - is_first_in_group[1:] = ordered_household[1:] != ordered_household[:-1] - mask = np.zeros(len(person), dtype=bool) - mask[order[is_first_in_group]] = True - return mask + return household_head_carrier_mask(person) def _household_head_predictor_table(person: pd.DataFrame) -> pd.DataFrame: diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/sipp_financial_assets.py b/packages/microcosm-build/src/microcosm/build/us_runtime/sipp_financial_assets.py index fd3583f015..74836c8bff 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/sipp_financial_assets.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/sipp_financial_assets.py @@ -558,7 +558,21 @@ def _recipient_is_married(person: pd.DataFrame) -> np.ndarray: return np.zeros(len(person), dtype=np.float64) -def _household_head_mask(person: pd.DataFrame) -> np.ndarray: +def household_head_carrier_mask(person: pd.DataFrame) -> np.ndarray: + """One financial-asset carrier per household: the declared household head. + + ``us/source_stages.json`` declares the wealth stage's carry as + ``head_carry`` with ``condition: is_household_head``, the CPS reference + person (``P_SEQ == 1``; ``relationship_inputs``), as the housing stages + use. A household whose flag does not mark exactly one person, or a frame + without the flag, falls back to its lowest ``A_LINENO``, then row order. + On the 2024 income year, where ``A_EXPRRP`` is observed, the flag is the + reference person in every household and the line rule misses 73 + (microcosm#1215). Both the SCF and SIPP sides use this one selector. + """ + + if len(person) == 0: + return np.zeros(0, dtype=bool) household = pd.to_numeric(person[_HOUSEHOLD_ID_COLUMN], errors="coerce").to_numpy() if "A_LINENO" in person.columns: line = ( @@ -572,9 +586,18 @@ def _household_head_mask(person: pd.DataFrame) -> np.ndarray: first = np.empty(len(person), dtype=bool) first[0] = True first[1:] = ordered_household[1:] != ordered_household[:-1] - mask = np.zeros(len(person), dtype=bool) - mask[order[first]] = True - return mask + lowest_line = np.zeros(len(person), dtype=bool) + lowest_line[order[first]] = True + if "is_household_head" not in person.columns: + return lowest_line + flag = person["is_household_head"].fillna(False).astype(bool).to_numpy() + flags_in_household = ( + pd.Series(flag.astype(np.int64)) + .groupby(household, dropna=False, sort=False) + .transform("sum") + .to_numpy() + ) + return np.where(flags_in_household == 1, flag, lowest_line) def _recipient_sipp_asset_predictor_table(person: pd.DataFrame) -> pd.DataFrame: @@ -660,7 +683,7 @@ def impute_us_sipp_financial_assets( f"SIPP financial-asset donor table missing column(s): {missing}." ) - head_mask = _household_head_mask(person) + head_mask = household_head_carrier_mask(person) head_features = _recipient_sipp_asset_predictor_table(person).loc[head_mask] augmented = head_features.copy() draws = pd.DataFrame(index=head_features.index) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py index d5ba0c21f6..4726b06369 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py @@ -12,7 +12,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # Pin the domain separator, normalization rules, schema-set receipt, and # exact normative projection as one reviewable golden vector. assert first.spec_sha256 == ( - "32305247436ed439181e8b5a667343436a4b5622bff0402d2f13cc08a51e2bfc" + "01fd31683397159f61822f2aff2705f2802d68b52a9d6509dc108f52d99b8ad2" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_scf_wealth.py b/packages/microcosm-build/tests/engine_free/us/test_us_scf_wealth.py index 114793c6d8..7ebebf7d3b 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_scf_wealth.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_scf_wealth.py @@ -1,6 +1,9 @@ """Tests split from packages/microcosm-build/tests/test_us_scf_wealth.py.""" # ruff: noqa: F403, F405 +from hypothesis import given, settings +from hypothesis import strategies as st + from test_support.microcosm_build.us_scf_wealth import * @@ -119,6 +122,83 @@ def test_household_head_mask_is_one_per_household_at_lowest_line() -> None: assert mask.sum() == 2 +def test_household_head_mask_follows_the_declared_household_head() -> None: + """microcosm#1215: the carry is declared ``condition: is_household_head``. + A household whose flag marks exactly one person carries on that person, + even when another member has a lower line number; a household with no + flagged person or two falls back to its lowest line.""" + person = pd.DataFrame( + { + "person_household_id": [5, 5, 5, 9, 9, 12, 12, 12], + "A_LINENO": [1, 2, 3, 1, 2, 2, 1, 3], + "is_household_head": [False, True, False, False, False, True, True, False], + } + ) + mask = _household_head_mask(person) + # hh 5: the flagged line-2 person; hh 9: no flag, lowest line (index 3); + # hh 12: two flags, lowest line (index 6). + np.testing.assert_array_equal( + mask, [False, True, False, True, False, False, True, False] + ) + + +def test_both_blend_halves_share_one_carrier_selector() -> None: + from microcosm.build.us_runtime import sipp_financial_assets + + person = pd.DataFrame( + { + "person_household_id": [1, 1, 2], + "A_LINENO": [1, 2, 1], + "is_household_head": pd.array([False, True, pd.NA], dtype="boolean"), + } + ) + np.testing.assert_array_equal( + _household_head_mask(person), + sipp_financial_assets.household_head_carrier_mask(person), + ) + np.testing.assert_array_equal(_household_head_mask(person), [False, True, True]) + assert _household_head_mask(person.iloc[:0]).shape == (0,) + + +@settings(max_examples=100, deadline=None) +@given( + st.lists( + st.tuples( + st.integers(0, 6), # household + st.integers(1, 9), # line number + st.booleans(), # is_household_head + ), + min_size=1, + max_size=40, + ) +) +def test_household_head_mask_marks_exactly_one_carrier_per_household(rows) -> None: + # CPS line numbers are unique within a household. + person = ( + pd.DataFrame( + rows, columns=["person_household_id", "A_LINENO", "is_household_head"] + ) + .drop_duplicates(["person_household_id", "A_LINENO"]) + .reset_index(drop=True) + ) + mask = _household_head_mask(person) + per_household = pd.Series(mask).groupby(person["person_household_id"]).sum() + assert (per_household == 1).all() + flagged = person.groupby("person_household_id")["is_household_head"].transform( + "sum" + ) + single = (flagged == 1).to_numpy() + # Where exactly one person is flagged, the carrier is that person. + np.testing.assert_array_equal( + mask[single], person["is_household_head"].to_numpy()[single] + ) + # Permuting rows moves the carrier with its row. + order = np.random.default_rng(len(rows)).permutation(len(person)) + np.testing.assert_array_equal( + _household_head_mask(person.iloc[order].reset_index(drop=True)), mask[order] + ) + + def test_impute_head_carries_and_zeroes_non_heads() -> None: person = _person_rows(60) donor = _donor_table() diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py index 5b1861afdd..45bf229162 100644 --- a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py @@ -338,7 +338,7 @@ def capture_equality(expected: object, actual: object) -> None: "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "fbf25dbae68242198972f5bef4f1fc20f65ee0769615b8d94639e6a80121f31f", + "spec_sha256": "651e8c2c5078a26ffa1aa48b842050d74c92cd24510f454af416cffc2ff145c2", }, } From db53b5f6b2c057eb50104b26ff53e61444cbe0a5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 06:59:16 -0400 Subject: [PATCH 2/4] Address review: flag coercion, missing-id grouping, downstream test - Only true booleans and nonzero numbers flag a head; strings such as "False" or "" never do. - Flags are counted within the same groups the lowest-line fallback sorts, so a missing household id is its own household in both rules. - New blend-plus-gate test with the declared head on line 2: all three leaves land on the head, other members hold zero, and the gate's blend-audit checks agree. - Re-pins the seed protocol and map, synthetic and US spec goldens, and the coverage receipt for the new source bytes. Co-Authored-By: Claude Opus 5.5 --- docs/evidence/spec-engine/us-f0-coverage.json | 16 ++-- .../build/spec_engine/inventory_coverage.py | 4 +- .../build/us_runtime/sipp_financial_assets.py | 19 ++-- .../shared/test_spec_engine_loader.py | 2 +- .../engine_free/us/test_us_scf_wealth.py | 90 +++++++++++++++++++ .../us/test_us_multispine_pool_tool.py | 2 +- 6 files changed, 114 insertions(+), 19 deletions(-) diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index 08a824a87f..0f63461534 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -1656,13 +1656,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "e04d6a85c24e536f9350abab3dc0bfed2215c1e4679dd30e449d4034511af9d4", - "protocol_sha256": "7855f2a7d3ef74377791070359bd45da766f371b49d76c1065a183d64fb2cf51" + "map_sha256": "be852c4fa3f1cb368528d99beb3b29d42020256d354fa89bb94d49a577d39bd4", + "protocol_sha256": "d6861b4b1caba2f7ecc66e825b32523b746f5a026ef676b9c1f77e2eb3b2e507" }, "failures": [], "observed": { - "map_sha256": "e04d6a85c24e536f9350abab3dc0bfed2215c1e4679dd30e449d4034511af9d4", - "protocol_sha256": "7855f2a7d3ef74377791070359bd45da766f371b49d76c1065a183d64fb2cf51" + "map_sha256": "be852c4fa3f1cb368528d99beb3b29d42020256d354fa89bb94d49a577d39bd4", + "protocol_sha256": "d6861b4b1caba2f7ecc66e825b32523b746f5a026ef676b9c1f77e2eb3b2e507" }, "status": "covered" }, @@ -1677,7 +1677,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "7855f2a7d3ef74377791070359bd45da766f371b49d76c1065a183d64fb2cf51", + "implementation_sha256": "d6861b4b1caba2f7ecc66e825b32523b746f5a026ef676b9c1f77e2eb3b2e507", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1698,7 +1698,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "7855f2a7d3ef74377791070359bd45da766f371b49d76c1065a183d64fb2cf51", + "implementation_sha256": "d6861b4b1caba2f7ecc66e825b32523b746f5a026ef676b9c1f77e2eb3b2e507", "protocol": "legacy-v1", "streams": [ "build_model", @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "651e8c2c5078a26ffa1aa48b842050d74c92cd24510f454af416cffc2ff145c2" + "spec_sha256": "50297c7b2513d3f80fda70b4b5a75cccdf27eae480c5addbcfc81ab28f70023e" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "651e8c2c5078a26ffa1aa48b842050d74c92cd24510f454af416cffc2ff145c2" + "spec_sha256": "50297c7b2513d3f80fda70b4b5a75cccdf27eae480c5addbcfc81ab28f70023e" }, "status": "pass" } diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index bb9e0f8d2a..f5f0e8d5c0 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -363,8 +363,8 @@ # #1215: scf_wealth/sipp_financial_assets source bytes attest the declared # is_household_head carrier. RNG rules and the ordered QRF targets are # unchanged. - "seed_map": "e04d6a85c24e536f9350abab3dc0bfed2215c1e4679dd30e449d4034511af9d4", - "seed_protocol": "7855f2a7d3ef74377791070359bd45da766f371b49d76c1065a183d64fb2cf51", + "seed_map": "be852c4fa3f1cb368528d99beb3b29d42020256d354fa89bb94d49a577d39bd4", + "seed_protocol": "d6861b4b1caba2f7ecc66e825b32523b746f5a026ef676b9c1f77e2eb3b2e507", "source_manifest": "f4f8bfeb79268a4043a843b47cec1d1c0e58e9f5dd6bc23417ada95446dd7b8e", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/sipp_financial_assets.py b/packages/microcosm-build/src/microcosm/build/us_runtime/sipp_financial_assets.py index 74836c8bff..9769fe38c5 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/sipp_financial_assets.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/sipp_financial_assets.py @@ -590,13 +590,18 @@ def household_head_carrier_mask(person: pd.DataFrame) -> np.ndarray: lowest_line[order[first]] = True if "is_household_head" not in person.columns: return lowest_line - flag = person["is_household_head"].fillna(False).astype(bool).to_numpy() - flags_in_household = ( - pd.Series(flag.astype(np.int64)) - .groupby(household, dropna=False, sort=False) - .transform("sum") - .to_numpy() - ) + raw_flag = person["is_household_head"] + if pd.api.types.is_bool_dtype(raw_flag.dtype): + flag = raw_flag.fillna(False).astype(bool).to_numpy() + else: + # Only true booleans and nonzero numbers flag a head; strings such as + # "False" or "" never do. + flag = pd.to_numeric(raw_flag, errors="coerce").fillna(0).ne(0).to_numpy() + # Count flags within the same groups the fallback sorts, so a missing + # household id is its own group in both rules. + group = np.empty(len(person), dtype=np.int64) + group[order] = np.cumsum(first) - 1 + flags_in_household = np.bincount(group, weights=flag.astype(np.float64))[group] return np.where(flags_in_household == 1, flag, lowest_line) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py index 4726b06369..18d2da180b 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py @@ -12,7 +12,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # Pin the domain separator, normalization rules, schema-set receipt, and # exact normative projection as one reviewable golden vector. assert first.spec_sha256 == ( - "01fd31683397159f61822f2aff2705f2802d68b52a9d6509dc108f52d99b8ad2" + "52fa1207f6a68d287180038c05e68bebb1260c2729ea3e9036f9668ff06cbafd" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_scf_wealth.py b/packages/microcosm-build/tests/engine_free/us/test_us_scf_wealth.py index 7ebebf7d3b..f02e493216 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_scf_wealth.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_scf_wealth.py @@ -313,6 +313,96 @@ def test_blend_uses_one_complete_source_vector_and_reference_person_carry( ) +def test_blend_carries_on_a_declared_head_above_line_one( + sipp_scf_blend_case: dict[str, object], +) -> None: + """microcosm#1215: where the declared head is the line-2 person, all three + leaves land on that person, every other member holds zero, and the gate's + audit check uses the same carrier.""" + person = sipp_scf_blend_case["person"].copy() + assert isinstance(person, pd.DataFrame) + line_two_head = (person["person_household_id"] % 2 == 0).to_numpy() + person["is_household_head"] = np.where( + line_two_head, person["A_LINENO"] == 2, person["A_LINENO"] == 1 + ) + blended = impute_us_sipp_scf_financial_assets( + person, + _donor_table(), + _sipp_donor_table(), + seed=23, + time_period=TIME_PERIOD, + n_estimators=8, + ) + head = person["is_household_head"].to_numpy() + np.testing.assert_array_equal(_household_head_mask(person), head) + leaves = list(US_SCF_FINANCIAL_ASSET_OUTPUT_COLUMNS) + assert blended.loc[~head, leaves].eq(0.0).all(axis=None) + assert blended.loc[head & line_two_head, "bank_account_assets"].sum() > 0 + assert ( + blended.loc[(person["A_LINENO"] == 1).to_numpy() & line_two_head, leaves] + .eq(0.0) + .all(axis=None) + ) + + base = _us_frame(person) + tables = {entity: base.table(entity).copy() for entity in base.entities} + for column in leaves: + tables["person"][column] = blended[column].to_numpy(dtype=np.float64) + tables["person"]["is_household_head"] = head + tables["person"].attrs[FINANCIAL_ASSET_BLEND_AUDIT_KEY] = dict( + blended.attrs[FINANCIAL_ASSET_BLEND_AUDIT_KEY] + ) + net_worth = np.linspace(10_000.0, 250_000.0, len(tables["household"])) + net_worth[:24] *= -1.0 + tables["household"]["net_worth"] = net_worth + frame = Frame( + tables, + base.schema, + {entity: base.weights_for(entity) for entity in base.weighted_entities}, + ) + gate = us_scf_wealth_signal_gate(frame, require_sipp_blend=True) + # The gate recomputes each household's source and the bank medians from the + # carrier mask; none of its blend-audit checks may disagree. (This small + # synthetic donor can trip unrelated distribution-plausibility checks.) + audit_failures = [ + failure + for failure in gate.failures + if any( + token in failure.lower() + for token in ("blend", "median", "low-tail", "source share") + ) + ] + assert audit_failures == [] + assert gate.details["financial_asset_blend"]["sipp_household_count"] > 0 + + +def test_household_head_mask_ignores_string_flags_and_isolates_missing_ids() -> None: + from microcosm.build.us_runtime import sipp_financial_assets + + strings = pd.DataFrame( + { + "person_household_id": [1, 1], + "A_LINENO": [2, 1], + "is_household_head": ["False", ""], + } + ) + # No real flag: the lowest line carries. + np.testing.assert_array_equal( + sipp_financial_assets.household_head_carrier_mask(strings), [False, True] + ) + missing_ids = pd.DataFrame( + { + "person_household_id": [np.nan, np.nan], + "A_LINENO": [1, 2], + "is_household_head": [False, True], + } + ) + # Each missing id is its own household, as in the lowest-line rule. + np.testing.assert_array_equal( + sipp_financial_assets.household_head_carrier_mask(missing_ids), [True, True] + ) + + def test_impute_missing_donor_column_raises() -> None: person = _person_rows(10) donor = _donor_table().drop(columns=["bond_assets"]) diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py index 45bf229162..e1e6d2f37b 100644 --- a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py @@ -338,7 +338,7 @@ def capture_equality(expected: object, actual: object) -> None: "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "651e8c2c5078a26ffa1aa48b842050d74c92cd24510f454af416cffc2ff145c2", + "spec_sha256": "50297c7b2513d3f80fda70b4b5a75cccdf27eae480c5addbcfc81ab28f70023e", }, } From eff5d8343ea80ced89205c997c0c2c9c576d94e9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 11 Nov 2026 00:35:34 -0500 Subject: [PATCH 3/4] Re-pin the US spec-engine contracts on main After the merge, main's pins no longer match this branch's scf_wealth.py and sipp_financial_assets.py bytes. tools/repin_us_spec_engine_contracts.py recomputes them: - EXPECTED_HASHES seed_map and seed_protocol, and the matching rows in us-f0-coverage.json; - the synthetic loader golden and the real US spec_sha256 (915d29b0...). tools/spec_engine_coverage.py reports 42,394/42,394 configuration fields and 41/41 inventory checks. The repinner's first pass also rewrote the coverage test's covered_item_count from 41 to 40, because it reads that count from a report built against the stale seed hashes. A second pass, run once the hashes matched, restored 41 and changed nothing else. Co-Authored-By: Claude Opus 5.5 --- docs/evidence/spec-engine/us-f0-coverage.json | 16 ++++++++-------- .../build/spec_engine/inventory_coverage.py | 4 ++-- .../shared/test_spec_engine_loader.py | 2 +- .../us/test_us_multispine_pool_tool.py | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index 5a20b58ca7..c65e878830 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -1642,13 +1642,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "fa48af060da522bb1edcb80580098f258763757139292602759cbccbfff76b37", - "protocol_sha256": "5840ac6cefd5ce3d2172aca9e3bfc08d2344e97a8e9b42a723a4edcaa9f8ff53" + "map_sha256": "5f8b7c620abee639d31f3dd50ed61e9c9a0daad89b8b4886d485b71d5cb882bd", + "protocol_sha256": "3e432297573840bbae497370efe3e7cafed5c2c571db029208d10722035a7470" }, "failures": [], "observed": { - "map_sha256": "fa48af060da522bb1edcb80580098f258763757139292602759cbccbfff76b37", - "protocol_sha256": "5840ac6cefd5ce3d2172aca9e3bfc08d2344e97a8e9b42a723a4edcaa9f8ff53" + "map_sha256": "5f8b7c620abee639d31f3dd50ed61e9c9a0daad89b8b4886d485b71d5cb882bd", + "protocol_sha256": "3e432297573840bbae497370efe3e7cafed5c2c571db029208d10722035a7470" }, "status": "covered" }, @@ -1663,7 +1663,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "5840ac6cefd5ce3d2172aca9e3bfc08d2344e97a8e9b42a723a4edcaa9f8ff53", + "implementation_sha256": "3e432297573840bbae497370efe3e7cafed5c2c571db029208d10722035a7470", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1684,7 +1684,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "5840ac6cefd5ce3d2172aca9e3bfc08d2344e97a8e9b42a723a4edcaa9f8ff53", + "implementation_sha256": "3e432297573840bbae497370efe3e7cafed5c2c571db029208d10722035a7470", "protocol": "legacy-v1", "streams": [ "build_model", @@ -2585,7 +2585,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "4ccbd01720871e59ad8e3c00e1c1296231facef91e83c561076ee499e0a138b2" + "spec_sha256": "915d29b0a5419d8e9ff4c9945fb5a92fb18576440dbbbcef2a7680546cfc4441" } }, "report_schema_version": 3, @@ -2595,7 +2595,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "4ccbd01720871e59ad8e3c00e1c1296231facef91e83c561076ee499e0a138b2" + "spec_sha256": "915d29b0a5419d8e9ff4c9945fb5a92fb18576440dbbbcef2a7680546cfc4441" }, "status": "pass" } diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 3206916b84..dabf240d92 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -380,8 +380,8 @@ # microcosm#1259's recipient-share bound in puf_support moves both again. # microcosm#1261 moves both again: the PUF donor carries partnership and # S-corp line-32 shares as separate leaves. - "seed_map": "fa48af060da522bb1edcb80580098f258763757139292602759cbccbfff76b37", - "seed_protocol": "5840ac6cefd5ce3d2172aca9e3bfc08d2344e97a8e9b42a723a4edcaa9f8ff53", + "seed_map": "5f8b7c620abee639d31f3dd50ed61e9c9a0daad89b8b4886d485b71d5cb882bd", + "seed_protocol": "3e432297573840bbae497370efe3e7cafed5c2c571db029208d10722035a7470", "source_manifest": "dc3879886d2a69f20fb982a675d7fb322c0e4017afef6471be54b0ee3cbd2af6", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", "tail": "565f0ceeadab69d3e96a886a00dd0305f7425fa3acde24d12ba48db2b37d5fc0", diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py index 26910035b3..32143a3fee 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py @@ -25,7 +25,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # ssi_disability_criteria's attested bytes. # The opt-in cross-fitted gate calibration moves the fit kernels' bytes. assert first.spec_sha256 == ( - "5b39d83cafb6f312ce73e5550ab36da6676ce265779d615f49c7987f5cc5d36e" + "ddb925d9603cd269c358f46ff433d5c4adf67991aa17782331a23c0b55767a05" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py index 5f77431853..7b39df575e 100644 --- a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py @@ -353,7 +353,7 @@ def capture_equality(expected: object, actual: object) -> None: # The opt-in cross-fitted gate calibration in microcosm.fit.qrf moves it. # microcosm#1073's scf_wealth education-debt declaration moves it # through the bundle's source stages. - "spec_sha256": "4ccbd01720871e59ad8e3c00e1c1296231facef91e83c561076ee499e0a138b2", + "spec_sha256": "915d29b0a5419d8e9ff4c9945fb5a92fb18576440dbbbcef2a7680546cfc4441", }, } From 5932199a5ffa4c4cda160261416b460ad2337f3f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 11 Nov 2026 01:08:56 -0500 Subject: [PATCH 4/4] Restore the #1215 note above the US seed pins The re-pin rewrote the seed_map and seed_protocol values and dropped the comment that names this PR among the changes that move them. One comment line now records it beside the other PRs that moved both. Co-Authored-By: Claude Opus 5.5 --- .../src/microcosm/build/spec_engine/inventory_coverage.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index dabf240d92..97c22108fe 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -379,7 +379,9 @@ # gains the passive partnership/S-corp leaf, so the PUF QRF targets change. # microcosm#1259's recipient-share bound in puf_support moves both again. # microcosm#1261 moves both again: the PUF donor carries partnership and - # S-corp line-32 shares as separate leaves. + # S-corp line-32 shares as separate leaves. microcosm#1215 moves both again: + # scf_wealth and sipp_financial_assets carry the assets on the declared + # is_household_head carrier. "seed_map": "5f8b7c620abee639d31f3dd50ed61e9c9a0daad89b8b4886d485b71d5cb882bd", "seed_protocol": "3e432297573840bbae497370efe3e7cafed5c2c571db029208d10722035a7470", "source_manifest": "dc3879886d2a69f20fb982a675d7fb322c0e4017afef6471be54b0ee3cbd2af6",