diff --git a/changelog.d/scf-bond-docstring.fixed.md b/changelog.d/scf-bond-docstring.fixed.md index 81090c1cbe..a5a6de8fb2 100644 --- a/changelog.d/scf-bond-docstring.fixed.md +++ b/changelog.d/scf-bond-docstring.fixed.md @@ -1 +1 @@ -Correct the `scf_wealth` docstring: SCF `bond` is directly held tax-exempt, mortgage-backed, U.S. government and agency, and corporate and foreign bonds. It excludes U.S. savings bonds, which the SCF reports separately as `savbnd` and this stage does not read, and bond funds, which are in `nmmf`. The docstring had said `bond` included savings bonds. Because the spec engine hashes `scf_wealth.py`'s bytes into the legacy-v1 seed-protocol attestation, the seed-protocol and seed-map digests, the US `spec_sha256`, the loader golden vector and `docs/evidence/spec-engine/us-f0-coverage.json` are re-pinned. No behaviour or data changes. +Correct the `scf_wealth` docstring: SCF `bond` is directly held tax-exempt, mortgage-backed, U.S. government and agency, and corporate and foreign bonds. It excludes U.S. savings bonds, which the SCF reports separately as `savbnd`, and bond funds, which are in `nmmf`. The docstring had said `bond` included savings bonds. Because the spec engine hashes `scf_wealth.py`'s bytes into the legacy-v1 seed-protocol attestation, the seed-protocol and seed-map digests, the US `spec_sha256`, the loader golden vector and `docs/evidence/spec-engine/us-f0-coverage.json` are re-pinned. No behaviour or data changes. diff --git a/changelog.d/scf-savings-bonds.fixed.md b/changelog.d/scf-savings-bonds.fixed.md new file mode 100644 index 0000000000..1558c11e20 --- /dev/null +++ b/changelog.d/scf-savings-bonds.fixed.md @@ -0,0 +1 @@ +Count U.S. savings bonds in the SCF half of `bond_assets`. The `scf_wealth` stage now maps `bond_assets` to SCF `bond` + `savbnd` instead of `bond` alone. PolicyEngine-US cites POMS SI 01140.240 (U.S. Savings Bonds) for `bond_assets`, and SIPP 2023 `TVAL_BOND`, the other half of the 50/50 blend, already includes savings bonds through government securities. On the pinned SCF 2022 extract, households with a nonzero SCF bond target rise from 1.11% to 7.36% by weight, and the weighted total rises from $1,398B to $1,699B (face value). In PolicyEngine-US 2.2.1, `bond_assets` feeds SSI countable resources, the SNAP asset test, `spm_unit_cash_assets` (read by many state TANF and local general-assistance resource tests) and Missouri TANF, so all of them see the change. This departs from archived commit `42ed5d45`, which kept `savbnd` as a separate net-worth component that microcosm does not build. The seed-kernel attestation is re-pinned, as for any byte edit to `scf_wealth.py`. Released data changes only through a rebuild and release. diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index f99b29cdf6..ca45a96d6a 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -1656,13 +1656,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "0fa116d9f5dae4ba0378f190d6c822b92f3dcf1e3fe80c9880bcaf87431c283c", - "protocol_sha256": "b4afa376b62fbc073fd10d74e40921f5ab60c1efe366e2c8504872091ea69d08" + "map_sha256": "767ac1c449018ceeb743e8990a7b7822c93051d995a63b9cb5743fdf327c5c74", + "protocol_sha256": "62b2155f6dd9383b607277218fa1acf5bfeaacb81243413bab2cc31c2e62607b" }, "failures": [], "observed": { - "map_sha256": "0fa116d9f5dae4ba0378f190d6c822b92f3dcf1e3fe80c9880bcaf87431c283c", - "protocol_sha256": "b4afa376b62fbc073fd10d74e40921f5ab60c1efe366e2c8504872091ea69d08" + "map_sha256": "767ac1c449018ceeb743e8990a7b7822c93051d995a63b9cb5743fdf327c5c74", + "protocol_sha256": "62b2155f6dd9383b607277218fa1acf5bfeaacb81243413bab2cc31c2e62607b" }, "status": "covered" }, @@ -1677,7 +1677,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "b4afa376b62fbc073fd10d74e40921f5ab60c1efe366e2c8504872091ea69d08", + "implementation_sha256": "62b2155f6dd9383b607277218fa1acf5bfeaacb81243413bab2cc31c2e62607b", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1698,7 +1698,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "b4afa376b62fbc073fd10d74e40921f5ab60c1efe366e2c8504872091ea69d08", + "implementation_sha256": "62b2155f6dd9383b607277218fa1acf5bfeaacb81243413bab2cc31c2e62607b", "protocol": "legacy-v1", "streams": [ "build_model", @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "d1df6b316e4937e809f5f020ea2d3a7bd38bccc1845f3ae6e104f7eed6a73323" + "spec_sha256": "d70f33e454eaae91e1bec20f04466051a5cd574d4612912f89118ccf0a8d9ac8" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "d1df6b316e4937e809f5f020ea2d3a7bd38bccc1845f3ae6e104f7eed6a73323" + "spec_sha256": "d70f33e454eaae91e1bec20f04466051a5cd574d4612912f89118ccf0a8d9ac8" }, "status": "pass" } diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 3fddff951a..bcce196d60 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -359,8 +359,8 @@ "late_schedule": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "fdf23da429f8c501e198a2f5b719763d523a41565c2153fdf17b0344f21bcd7c", - "seed_map": "0fa116d9f5dae4ba0378f190d6c822b92f3dcf1e3fe80c9880bcaf87431c283c", - "seed_protocol": "b4afa376b62fbc073fd10d74e40921f5ab60c1efe366e2c8504872091ea69d08", + "seed_map": "767ac1c449018ceeb743e8990a7b7822c93051d995a63b9cb5743fdf327c5c74", + "seed_protocol": "62b2155f6dd9383b607277218fa1acf5bfeaacb81243413bab2cc31c2e62607b", "source_manifest": "f4f8bfeb79268a4043a843b47cec1d1c0e58e9f5dd6bc23417ada95446dd7b8e", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/scf_wealth.py b/packages/microcosm-build/src/microcosm/build/us_runtime/scf_wealth.py index e77540daae..4a02d52d2b 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/scf_wealth.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/scf_wealth.py @@ -19,18 +19,44 @@ restores the realistic low liquid-asset mass absent from the SCF-only stage. The SCF side uses the public summary extract (``rscfp2022.dta``) and the -regime-gated weighted QRF (``microcosm.fit.QRF``). Its targets exactly match -archived commit ``42ed5d45`` (``utils/asset_imputation.py``): +regime-gated weighted QRF (``microcosm.fit.QRF``). Its targets match archived +commit ``42ed5d45`` (``utils/asset_imputation.py``) except ``bond_assets``, +which also reads U.S. savings bonds: - ``bank_account_assets`` ← SCF ``liq`` (checking, savings, money-market, - call accounts). + call and prepaid-card accounts; the Fed's macro sets it to at least $1 for + a household holding any such account). - ``stock_assets`` ← SCF ``stocks`` + ``nmmf`` (directly-held stock plus - stock/non-money-market mutual funds — the one code-level summation the - canonical pipeline applies). -- ``bond_assets`` ← SCF ``bond`` (directly held tax-exempt, mortgage-backed, - U.S. government and agency, and corporate and foreign bonds). Bond funds - are in ``nmmf``, so they land in ``stock_assets``; U.S. savings bonds are - the SCF's separate ``savbnd``, which this stage does not read. + stock/non-money-market mutual funds). +- ``bond_assets`` ← SCF ``bond`` + ``savbnd``. ``bond`` is directly held + tax-exempt, mortgage-backed, U.S. government and agency, and corporate and + foreign bonds; the Fed's extract macro excludes bond funds and savings bonds + from it. ``savbnd`` is U.S. savings bonds (``X3902``). Bond funds are in + ``nmmf``, so they land in ``stock_assets``. + +The archive read ``bond`` alone and kept ``savbnd`` as ``scf_savings_bonds``, +a construction-only component of its net-worth reconciliation. Microcosm +builds none of those components (see ``net_worth`` below), so under the +archive mapping an SCF-drawn household's savings bonds reached no SSI leaf. +PolicyEngine-US documents ``bond_assets`` as "Value of bonds and government +securities. Imputed from SIPP TVAL_BOND." and cites POMS SI 01140.240 (U.S. +Savings Bonds) and SI 01140.250 (Municipal, Corporate and Government Bonds). +SIPP 2023 ``TVAL_BOND`` sums the government-securities and municipal/corporate +bond values, and the 2023 SIPP data dictionary defines government securities +as "such as savings bonds, T-Bills, T-Bonds, T-Notes" (``EOWN_GOVS``). Reading +``bond`` + ``savbnd`` gives the SCF half the same concept as the SIPP half and +the variable's cited sources. If the archive's net-worth components are ever +built, ``scf_savings_bonds`` must leave that partition, because +``bond_assets`` now carries ``savbnd``. + +Two measurement limits remain, for this leaf as before. The SCF records face +values: ``X3902`` for savings bonds, and face-value items for every component +of ``bond``. SIPP asks balance or market value including interest, while POMS +counts other bonds at current market value (SI 01140.250) and savings bonds at +redemption value (SI 01140.240). And neither survey identifies savings bonds +still in their mandatory retention period (12 months for Series EE and I bonds +issued on or after 2/1/2003), which SI 01140.240 says are not resources. Both +limits affect amounts, not which leaf savings bonds belong to. The SCF QRF's eight predictors are age, sex, race, marriage, own children, employment income, interest/dividend income, and Social Security/pension @@ -201,12 +227,14 @@ US_SCF_NET_WORTH_OUTPUT_COLUMNS ) -#: Each output column and the SCF summary-extract components it sums -#: (retired pipeline @ 42ed5d45, ``utils/asset_imputation.py``). +#: Each output column and the SCF summary-extract components it sums. These +#: are the retired pipeline's mappings (42ed5d45, ``utils/asset_imputation.py``) +#: except ``bond_assets``, which adds U.S. savings bonds (``savbnd``) as the +#: module docstring explains. No component feeds more than one leaf. SCF_FINANCIAL_ASSET_TARGET_COMPONENTS: dict[str, tuple[str, ...]] = { "bank_account_assets": ("liq",), "stock_assets": ("stocks", "nmmf"), - "bond_assets": ("bond",), + "bond_assets": ("bond", "savbnd"), } SCF_NET_WORTH_TARGET_COMPONENTS: dict[str, tuple[str, ...]] = { "net_worth": ("networth",), @@ -262,9 +290,10 @@ _HOUSEHOLD_ID_COLUMN = "person_household_id" #: Weighted person-level nonzero-share plausibility bands. Centred on the -#: pinned incumbent eCPS parity reference (bank 0.54, stock 0.16, bond 0.03) -#: with generous width — the gate exists to catch an all-zero or constant -#: surface, not to pin a point estimate. +#: pinned incumbent eCPS parity reference (bank 0.54, stock 0.16, bond 0.03; +#: those reference values are unweighted record shares) with generous width — +#: the gate exists to catch an all-zero or constant surface, not to pin a +#: point estimate. _BANK_NONZERO_SHARE_BAND = (0.25, 0.85) _STOCK_NONZERO_SHARE_BAND = (0.03, 0.40) _BOND_NONZERO_SHARE_BAND = (0.001, 0.12) @@ -560,6 +589,20 @@ def fetch_scf_2022_summary_extract( return target +def _scf_financial_asset_targets(raw: pd.DataFrame) -> pd.DataFrame: + """Sum each SSI leaf's SCF components, sentinel-cleaned, floored at zero.""" + + targets = pd.DataFrame(index=raw.index) + for output, components in SCF_FINANCIAL_ASSET_TARGET_COMPONENTS.items(): + total = np.zeros(len(raw), dtype=np.float64) + for component in components: + total = total + _replace_sentinels(raw[component]).to_numpy( + dtype=np.float64 + ) + targets[output] = np.maximum(total, 0.0) + return targets + + def load_scf_2022_financial_asset_donor(path: str | Path) -> pd.DataFrame: """Read the SCF 2022 summary extract into a financial-asset donor table. @@ -579,10 +622,11 @@ def load_scf_2022_financial_asset_donor(path: str | Path) -> pd.DataFrame: raw = pd.read_stata(path, convert_categoricals=False) required = { - "liq", - "stocks", - "nmmf", - "bond", + *( + component + for components in SCF_FINANCIAL_ASSET_TARGET_COMPONENTS.values() + for component in components + ), "networth", *_SCF_SUMMARY_PREDICTOR_SOURCE_COLUMNS, } @@ -592,15 +636,7 @@ def load_scf_2022_financial_asset_donor(path: str | Path) -> pd.DataFrame: f"SCF 2022 summary extract missing required column(s): {missing}." ) - donor = pd.DataFrame(index=raw.index) - # Targets: component sums, sentinel-cleaned, floored at zero. - for output, components in SCF_FINANCIAL_ASSET_TARGET_COMPONENTS.items(): - total = np.zeros(len(raw), dtype=np.float64) - for component in components: - total = total + _replace_sentinels(raw[component]).to_numpy( - dtype=np.float64 - ) - donor[output] = np.maximum(total, 0.0) + donor = _scf_financial_asset_targets(raw) # SCF networth is already the complete signed balance-sheet aggregate. # Preserve negative values: they are indebted-household source signal, not # missing-value sentinels. @@ -1014,9 +1050,10 @@ def _financial_assets_carry_signal(person: pd.DataFrame) -> bool: Nonfinite values mark a corrupted surface and force re-imputation. The engine-default check is JOINT across the three leaves: a surface where every leaf is constant (the all-zero engine default) must be re-imputed, - but a single legitimately constant leaf must not force a redraw — bond - holdings are ~97% zero in the donor, so a small or chunked frame can - draw a constant bond column from a perfectly healthy imputation. A + but a single legitimately constant leaf must not force a redraw — about + 93% of SCF donor households by survey weight hold neither bonds nor + savings bonds, so a small or chunked frame can draw a constant bond + column from a perfectly healthy imputation. A per-leaf nonconstancy test made pass-through platform-dependent (the #510 CI failure) and would silently redraw small production chunks; flattened cross-leaf uniqueness would accept three DISTINCT constant diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py index 718f4ff565..b2977e2728 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py @@ -12,7 +12,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # Pin the domain separator, normalization rules, schema-set receipt, and # exact normative projection as one reviewable golden vector. assert first.spec_sha256 == ( - "6273f82f0e4899e97d1bede4db0662829b3df6f420dba32c0e67018d010c2c91" + "8d578ce26106423171bfe57be0cf36026c6c6febd1b4e572d784c3cf96409550" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_scf_wealth.py b/packages/microcosm-build/tests/engine_free/us/test_us_scf_wealth.py index 114793c6d8..f50ce432c3 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_scf_wealth.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_scf_wealth.py @@ -1,6 +1,9 @@ """Tests split from packages/microcosm-build/tests/test_us_scf_wealth.py.""" # ruff: noqa: F403, F405 +from hypothesis import given, settings +from hypothesis import strategies as st + from test_support.microcosm_build.us_scf_wealth import * @@ -26,7 +29,43 @@ def test_output_columns_are_the_ssi_countable_resource_leaves() -> None: def test_stock_target_sums_stocks_and_nmmf() -> None: assert SCF_FINANCIAL_ASSET_TARGET_COMPONENTS["stock_assets"] == ("stocks", "nmmf") assert SCF_FINANCIAL_ASSET_TARGET_COMPONENTS["bank_account_assets"] == ("liq",) - assert SCF_FINANCIAL_ASSET_TARGET_COMPONENTS["bond_assets"] == ("bond",) + assert SCF_FINANCIAL_ASSET_TARGET_COMPONENTS["bond_assets"] == ("bond", "savbnd") + + +#: The Fed's total financial assets, ``FIN``, is the sum of these ten disjoint +#: summary-extract constituents (bulletin.macro.txt line 2177: +#: ``FIN=LIQ+CDS+NMMF+STOCKS+BOND+RETQLIQ+SAVBND+CASHLI+OTHMA+OTHFIN``). +_SCF_FIN_CONSTITUENTS = frozenset( + { + "liq", + "cds", + "nmmf", + "stocks", + "bond", + "retqliq", + "savbnd", + "cashli", + "othma", + "othfin", + } +) + + +def test_ssi_leaves_draw_on_distinct_scf_fin_constituents() -> None: + components = [ + component + for leaf_components in SCF_FINANCIAL_ASSET_TARGET_COMPONENTS.values() + for component in leaf_components + ] + # No SCF component feeds two SSI leaves, so the leaves cannot double count. + assert len(components) == len(set(components)) + # Every component is one of FIN's disjoint constituents (the leaves use five + # of the ten), so wherever the other constituents are nonnegative the three + # leaves together cannot exceed the household's total financial assets. + assert set(components) <= _SCF_FIN_CONSTITUENTS + # U.S. savings bonds (POMS SI 01140.240) reach an SSI leaf, as SIPP's + # TVAL_BOND does on the other half of the blend. + assert "savbnd" in SCF_FINANCIAL_ASSET_TARGET_COMPONENTS["bond_assets"] def test_net_worth_target_is_the_direct_signed_scf_anchor() -> None: @@ -70,6 +109,14 @@ def test_load_donor_derives_targets_predictors_and_weight(tmp_path) -> None: np.testing.assert_allclose( np.sort(donor["stock_assets"].to_numpy()), np.sort(expected_stock), rtol=1e-6 ) + # bond_assets is the bond + savbnd sum, row for row (every fixture weight is + # positive, so the loader keeps row order). + assert (raw["savbnd"] > 0).any() + np.testing.assert_allclose( + donor["bond_assets"].to_numpy(), + raw["bond"].to_numpy() + raw["savbnd"].to_numpy(), + rtol=1e-6, + ) # All weights positive; targets non-negative. assert (donor[_DONOR_WEIGHT_COLUMN] > 0).all() for column in US_SCF_FINANCIAL_ASSET_OUTPUT_COLUMNS: @@ -88,6 +135,137 @@ def test_load_donor_missing_column_raises(tmp_path) -> None: load_scf_2022_financial_asset_donor(path) +@pytest.mark.parametrize( + "column", + sorted( + { + component + for components in SCF_FINANCIAL_ASSET_TARGET_COMPONENTS.values() + for component in components + } + | {"networth"} + ), +) +def test_load_donor_requires_every_target_component(tmp_path, column) -> None: + raw = _raw_scf_summary().drop(columns=[column]) + path = tmp_path / "rscfp2022.dta" + raw.to_stata(path, write_index=False) + with pytest.raises(ValueError, match=f"missing required column.*{column}"): + load_scf_2022_financial_asset_donor(path) + + +_COMPONENT_COLUMNS = tuple( + sorted( + { + component + for components in SCF_FINANCIAL_ASSET_TARGET_COMPONENTS.values() + for component in components + } + ) +) +#: A raw SCF amount: a missing-data sentinel or a nonnegative dollar value. +_scf_amount = st.one_of( + st.sampled_from([float(code) for code in _SCF_SENTINELS]), + st.floats(min_value=0.0, max_value=1e10, allow_nan=False), +) + + +@st.composite +def _raw_component_tables(draw) -> pd.DataFrame: + rows = draw(st.integers(min_value=1, max_value=12)) + return pd.DataFrame( + { + column: draw(st.lists(_scf_amount, min_size=rows, max_size=rows)) + for column in _COMPONENT_COLUMNS + } + ) + + +def _cleaned(values: pd.Series) -> np.ndarray: + array = values.to_numpy(dtype=np.float64) + return np.where(np.isin(array, _SCF_SENTINELS), 0.0, array) + + +@settings(max_examples=200, deadline=None) +@given(raw=_raw_component_tables()) +def test_scf_targets_sum_cleaned_components(raw) -> None: + # Differential check against a direct restatement of the rule: each leaf is + # the sum of its sentinel-cleaned components, floored at zero. + targets = _scf_financial_asset_targets(raw) + assert list(targets.columns) == list(US_SCF_FINANCIAL_ASSET_OUTPUT_COLUMNS) + for leaf, components in SCF_FINANCIAL_ASSET_TARGET_COMPONENTS.items(): + expected = np.maximum( + sum(_cleaned(raw[component]) for component in components), 0.0 + ) + np.testing.assert_allclose(targets[leaf].to_numpy(), expected, rtol=1e-12) + assert np.isfinite(targets[leaf].to_numpy()).all() + assert (targets[leaf].to_numpy() >= 0.0).all() + + +#: A signed non-sentinel amount, so the zero floor can bind. +_signed_scf_amount = st.floats(min_value=-1e10, max_value=1e10, allow_nan=False).filter( + lambda value: value not in _SCF_SENTINELS +) + + +@settings(max_examples=200, deadline=None) +@given( + rows=st.integers(min_value=1, max_value=12).flatmap( + lambda rows: st.fixed_dictionaries( + { + column: st.lists(_signed_scf_amount, min_size=rows, max_size=rows) + for column in _COMPONENT_COLUMNS + } + ) + ) +) +def test_scf_targets_floor_negative_sums_at_zero(rows) -> None: + raw = pd.DataFrame(rows) + targets = _scf_financial_asset_targets(raw) + for leaf, components in SCF_FINANCIAL_ASSET_TARGET_COMPONENTS.items(): + expected = np.maximum(sum(raw[c].to_numpy() for c in components), 0.0) + np.testing.assert_allclose(targets[leaf].to_numpy(), expected, rtol=1e-12) + assert (targets[leaf].to_numpy() >= 0.0).all() + + +@settings(max_examples=200, deadline=None) +@given(raw=_raw_component_tables()) +def test_savings_bonds_add_to_bond_assets_and_nothing_else(raw) -> None: + # Components here are nonnegative or sentinels, as in the SCF extract; with a + # negative non-sentinel bond the zero floor could absorb savings bonds. + targets = _scf_financial_asset_targets(raw) + without_savings_bonds = _scf_financial_asset_targets(raw.assign(savbnd=0.0)) + # Savings bonds raise bond_assets by exactly their cleaned value, so the + # leaf never falls below the archive's bond-only target ... + np.testing.assert_allclose( + targets["bond_assets"].to_numpy(), + without_savings_bonds["bond_assets"].to_numpy() + _cleaned(raw["savbnd"]), + rtol=1e-12, + ) + assert ( + targets["bond_assets"].to_numpy() + >= without_savings_bonds["bond_assets"].to_numpy() + ).all() + # ... and leave the bank and stock leaves untouched. + for leaf in ("bank_account_assets", "stock_assets"): + np.testing.assert_array_equal( + targets[leaf].to_numpy(), without_savings_bonds[leaf].to_numpy() + ) + + +@settings(max_examples=200, deadline=None) +@given( + raw=_raw_component_tables(), + column=st.sampled_from(_COMPONENT_COLUMNS), + sentinel=st.sampled_from(_SCF_SENTINELS), +) +def test_scf_sentinels_count_as_zero(raw, column, sentinel) -> None: + np.testing.assert_array_equal( + _scf_financial_asset_targets(raw.assign(**{column: float(sentinel)})), + _scf_financial_asset_targets(raw.assign(**{column: 0.0})), + ) + + def test_replace_sentinels_zeroes_scf_missing_codes() -> None: series = pd.Series([-1.0, -7.0, -8.0, -9.0, 5_000.0, 0.0]) cleaned = _replace_sentinels(series).to_numpy() @@ -277,8 +455,9 @@ def test_with_inputs_writes_asset_and_net_worth_columns() -> None: def test_carry_signal_tolerates_a_single_constant_leaf() -> None: - # Bond holdings are ~97% zero in the donor: a healthy draw on a small - # frame can produce an all-zero bond column. That must NOT read as an + # About 93% of SCF donor households (by weight) hold neither bonds nor + # savings bonds: a healthy draw on a small frame can produce an all-zero + # bond column. That must NOT read as an # engine-default surface (per-leaf nonconstancy made pass-through # platform-dependent — the #510 CI failure). Only an all-leaves-constant # surface forces re-imputation. diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py index ca014679be..8f810ac808 100644 --- a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py @@ -338,7 +338,7 @@ def capture_equality(expected: object, actual: object) -> None: "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "d1df6b316e4937e809f5f020ea2d3a7bd38bccc1845f3ae6e104f7eed6a73323", + "spec_sha256": "d70f33e454eaae91e1bec20f04466051a5cd574d4612912f89118ccf0a8d9ac8", }, } diff --git a/test_support/microcosm_build/us_scf_wealth.py b/test_support/microcosm_build/us_scf_wealth.py index b6961ffb21..f30c0479a5 100644 --- a/test_support/microcosm_build/us_scf_wealth.py +++ b/test_support/microcosm_build/us_scf_wealth.py @@ -46,9 +46,11 @@ with_us_scf_wealth_inputs, ) from microcosm.build.us_runtime.scf_wealth import ( + _SCF_SENTINELS, _household_head_mask, _recipient_cps_race, _replace_sentinels, + _scf_financial_asset_targets, ) from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights @@ -85,6 +87,8 @@ def _raw_scf_summary() -> pd.DataFrame: "wageinc": rng.gamma(2.0, 20_000.0, n), "intdivinc": rng.gamma(1.0, 1_000.0, n), "ssretinc": rng.gamma(1.0, 8_000.0, n), + # Drawn last so every column above keeps its earlier values. + "savbnd": np.where(rng.random(n) < 0.07, rng.gamma(1.0, 3_000.0, n), 0.0), } )