From 451e396e519b07b3895346dc0156fc5a2a7d296b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 13 Oct 2026 07:31:53 -0400 Subject: [PATCH 1/8] Bound each PUF QRF draw by the population slice its recipient stands for The #982 income-rank predictor sends the top-ranked survey tax units, each standing for several hundred units, onto the PUF's top-ranked returns, which stand for one return or less (the synthetic Forbes-tail records weigh about 0.13). One draw put $141B of line-32 mass on a weight-644 unit in the #1251 rebuild. Weight-aware leaf draws (#481) cannot fix this: the slice such a donor belongs to is smaller than one recipient. QRF finalization now caps each raw amount draw at the positive mass per unit of slice weight of the donor's top slice of the recipient's own weight share, and floors losses from the bottom slice, on every amount output. The rule is bound into the primary PUF execution config (schema 6), and each bounded output writes a tail-bound diagnostics record. Co-Authored-By: Claude Opus 5.5 --- .../us-puf-recipient-share-bound.fixed.md | 1 + .../spec_engine/schema/imputation.schema.json | 48 ++ .../microcosm/build/us_runtime/puf_support.py | 247 +++++- .../build/us_runtime/stacked_spine.py | 17 +- .../us/test_us_multispine_pool_h5_io.py | 2 + .../us/test_us_puf_recipient_share_bound.py | 740 ++++++++++++++++++ .../engine_free/us/test_us_puf_tail_bound.py | 29 +- .../engine_free/us/test_us_stacked_spine.py | 47 +- 8 files changed, 1111 insertions(+), 20 deletions(-) create mode 100644 changelog.d/us-puf-recipient-share-bound.fixed.md create mode 100644 packages/microcosm-build/tests/engine_free/us/test_us_puf_recipient_share_bound.py diff --git a/changelog.d/us-puf-recipient-share-bound.fixed.md b/changelog.d/us-puf-recipient-share-bound.fixed.md new file mode 100644 index 0000000000..404d224e6e --- /dev/null +++ b/changelog.d/us-puf-recipient-share-bound.fixed.md @@ -0,0 +1 @@ +A single PUF-channel QRF draw can no longer carry more mass than the population slice its recipient stands for. The #982 income-rank predictor sends the top-ranked survey tax units, each standing for several hundred units, onto the PUF's top-ranked returns, which stand for one return or less: the synthetic Forbes-tail records weigh about 0.13 each. One such draw put $141B of line-32 mass on a single weight-644 unit in the #1251 rebuild. QRF finalization now caps each raw amount draw at the positive mass, per unit of slice weight, of the donor's top slice of the recipient's own weight share, and floors losses from the bottom slice, on every amount output (booleans and origination years excepted). The rule is bound into the primary PUF execution config, which moves to schema 6; every bounded output writes a tail-bound diagnostics record. diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/imputation.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/imputation.schema.json index be210f2657..50aa683d32 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/imputation.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/imputation.schema.json @@ -3274,6 +3274,53 @@ "non_sch_d_capital_gains" ] }, + "recipient_share_bound": { + "type": "object", + "additionalProperties": false, + "properties": { + "rule": { + "type": "string" + }, + "schema_version": { + "type": "integer" + }, + "scope": { + "type": "string" + }, + "excluded_output_classes": { + "type": "array", + "items": { + "type": "string" + } + }, + "recipient_share": { + "type": "string" + }, + "cap": { + "type": "string" + }, + "floor": { + "type": "string" + }, + "snapped_outputs": { + "type": "string" + }, + "seam": { + "type": "string" + } + }, + "required": [ + "rule", + "schema_version", + "scope", + "excluded_output_classes", + "recipient_share", + "cap", + "floor", + "snapped_outputs", + "seam" + ] + }, "worker_execution": { "x-spec-surface": "execution_profile", "type": "object", @@ -3347,6 +3394,7 @@ "required": [ "invocation_mode", "tail_bound_quantiles", + "recipient_share_bound", "worker_execution" ] }, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py index 165cf5a446..acbb94376e 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py @@ -75,6 +75,7 @@ "PUF_TAX_DETAIL_EARNINGS_COMPONENTS", "PUF_TAX_DETAIL_EARNINGS_INDICATOR_SOURCE", "PUF_TAX_DETAIL_INCOME_RANK_SOURCE", + "PUF_TAX_DETAIL_RECIPIENT_SHARE_BOUND_RULE", "PufTaxDetailChainInputs", "PUF_ABSENT_CELLS_LEGACY_ZERO_FILL", "PUF_ABSENT_CELLS_PRESERVE_NULLS", @@ -97,6 +98,7 @@ "impute_us_puf_tax_detail_support", "puf_tax_detail_clone_mask", "puf_recipient_predictor_universe_receipt", + "puf_tax_detail_recipient_share_bound_identity", "puf_tax_detail_tail_bound_quantiles_identity", "puf_tax_unit_donor_from_arrays", "prepare_us_puf_tax_detail_chain_inputs", @@ -389,6 +391,55 @@ def puf_tax_detail_tail_bound_quantiles_identity() -> dict[str, float]: return dict(sorted(_PUF_TAX_DETAIL_TAIL_BOUND_QUANTILES.items())) +# Recipient-share bound at the same finalize seam. A PUF-clone recipient whose +# weight is the share ``s`` of the recipient population stands for ``s`` of the +# population, but a QRF draw hands it one donor's value. At the top of the +# income rank a recipient stands for ~600 tax units while the donors it reaches +# stand for a fraction of one: the PUF's synthetic Forbes-tail records carry +# ~0.13 weight each. One such draw carried $141B of line-32 mass in the #1251 +# base, about 5,000 times what its donor represents. Weight-aware leaf draws +# (#481) cannot fix this: the slice of the population such a donor belongs to +# is smaller than one recipient, so whichever donor is drawn, one record +# carries it. The bound caps each raw amount draw at the positive mass, per +# unit of slice weight, of the donor's top slice of the recipient's own share +# (and floors losses from the bottom slice), so no record carries more mass +# than the most extreme slice of the population it could stand for. Tail +# support below pool resolution is the pool's job (DESIGN.md); this keeps the +# fit from delivering it. Discrete and boolean outputs are not amounts and +# are left alone. +PUF_TAX_DETAIL_RECIPIENT_SHARE_BOUND_RULE = "recipient_share_slice_mass" +_PUF_TAX_DETAIL_RECIPIENT_SHARE_BOUND_SCHEMA_VERSION = 1 + + +def puf_tax_detail_recipient_share_bound_identity() -> dict[str, object]: + """Return the exact identity of the recipient-share finalization bound.""" + + return { + "rule": PUF_TAX_DETAIL_RECIPIENT_SHARE_BOUND_RULE, + "schema_version": _PUF_TAX_DETAIL_RECIPIENT_SHARE_BOUND_SCHEMA_VERSION, + "scope": "every_amount_output_of_the_invocation", + "excluded_output_classes": [ + "boolean_person_outputs", + "discrete_tax_unit_outputs", + ], + "recipient_share": "household_design_weight_over_puf_recipient_total", + "cap": "top_slice_positive_mass_per_slice_weight", + "floor": "bottom_slice_negative_mass_per_slice_weight", + "snapped_outputs": "cap_lowered_to_largest_snap_grid_value", + "seam": "raw_tax_unit_draws_after_tail_bound_quantiles", + } + + +def _recipient_share_bound_amount_outputs(outputs: Sequence[str]) -> tuple[str, ...]: + """Every amount output of an invocation, in chain order.""" + + excluded = ( + _PUF_TAX_DETAIL_BOOLEAN_PERSON_OUTPUTS + | _PUF_TAX_DETAIL_DISCRETE_TAX_UNIT_OUTPUTS + ) + return tuple(output for output in outputs if output not in excluded) + + # ASEC directly measures recipient alimony. The PUF QRF therefore sparsifies # only the cloned PUF half for this leaf; pruning the ASEC half would discard # reported source observations. Expense has no ASEC analogue, so its zero ASEC @@ -1903,6 +1954,7 @@ def finalize_us_puf_tax_detail_predictions( tax_unit_outputs: Sequence[str] = PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS, tail_bound_diagnostics: list[dict[str, object]] | None = None, tail_bound_quantiles: Mapping[str, float] | None = None, + recipient_share_bound_outputs: Sequence[str] | None = None, absent_cells: str = PUF_ABSENT_CELLS_LEGACY_ZERO_FILL, ) -> Frame: """Finalize a complete raw PUF QRF chain onto its support channel. @@ -1922,6 +1974,17 @@ def finalize_us_puf_tax_detail_predictions( weighted mass over the affected raw tax-unit draws before and after clipping; build callers must provide the sink and publish every active cap. + The recipient-share bound then applies to every amount output of the + invocation (see :func:`puf_tax_detail_recipient_share_bound_identity`); an + explicit ``recipient_share_bound_outputs`` narrows it for tests and must + name amount outputs of this invocation. It writes one diagnostics record + per bounded output, including outputs it leaves unchanged. It holds on the + raw tax-unit draws and, for snapped outputs, after snapping. Two later + steps rescale a whole column to the donor instrument and so lift every + kept value by one common factor: the donor-rate sparsification of sparse + outputs restores the column's mass after zeroing carriers, and the + signed-mass calibration scales each leg of its outputs. + ``absent_cells`` selects the finalization policy for cells outside the PUF clone arm (microcosm#578 audit item 1). The legacy default reproduces the historical global zero-fill byte for byte. Under @@ -1983,7 +2046,20 @@ def finalize_us_puf_tax_detail_predictions( "donor support." ) from None resolved_tail_bounds[output] = (quantile, bound) - if resolved_tail_bounds and tail_bound_diagnostics is None: + amount_outputs = _recipient_share_bound_amount_outputs(outputs) + if recipient_share_bound_outputs is None: + share_bound_outputs = amount_outputs + else: + share_bound_outputs = tuple(recipient_share_bound_outputs) + not_amounts = [ + output for output in share_bound_outputs if output not in amount_outputs + ] + if not_amounts or len(set(share_bound_outputs)) != len(share_bound_outputs): + raise ValueError( + "PUF recipient-share bound outputs must be distinct amount " + f"outputs of this invocation; got {not_amounts or 'duplicates'}." + ) + if (resolved_tail_bounds or share_bound_outputs) and tail_bound_diagnostics is None: raise ValueError( "PUF tax-detail tail-bound finalization requires a diagnostics sink; " "tail bounds must not be silent." @@ -2001,13 +2077,29 @@ def finalize_us_puf_tax_detail_predictions( "PUF raw predictions changed recipient row order or index before " "finalization." ) - if resolved_tail_bounds: + if resolved_tail_bounds or share_bound_outputs: recipient_tax_unit_ids = frame.table("tax_unit").loc[puf_mask, "tax_unit_id"] recipient_weights = _tax_unit_household_weights( frame, recipient_tax_unit_ids, ) - + resolved_share_bounds = { + output: _recipient_share_slice_bounds( + donor[output], donor["weight"], recipient_weights + ) + for output in share_bound_outputs + } + snapped_outputs = ( + _PUF_TAX_DETAIL_SPARSE_PERSON_OUTPUTS | _PUF_TAX_DETAIL_SPARSE_TAX_UNIT_OUTPUTS + ) + for output in share_bound_outputs: + if output in snapped_outputs: + floor, cap = resolved_share_bounds[output] + resolved_share_bounds[output] = ( + floor, + _largest_grid_values_within(cap, _observed_snap_grid(donor[output])), + ) + if resolved_tail_bounds: for output, (quantile, bound) in resolved_tail_bounds.items(): values = predictions[output].to_numpy(dtype=np.float64, copy=False) clipped = values > bound @@ -2031,6 +2123,32 @@ def finalize_us_puf_tax_detail_predictions( "clipped_mass_after": clipped_mass_after, } ) + for output, (floor, cap) in resolved_share_bounds.items(): + values = predictions[output].to_numpy(dtype=np.float64, copy=True) + bounded = np.minimum(np.maximum(values, floor), cap) + capped = values > cap + floored = values < floor + moved = capped | floored + weighted_before = np.abs(values * recipient_weights) + weighted_after = np.abs(bounded * recipient_weights) + predictions[output] = bounded + tail_bound_diagnostics.append( + { + "output": output, + "rule": PUF_TAX_DETAIL_RECIPIENT_SHARE_BOUND_RULE, + "clipped_row_count": int(moved.sum()), + "capped_row_count": int(capped.sum()), + "floored_row_count": int(floored.sum()), + "clipped_mass_before": float( + (values[moved] * recipient_weights[moved]).sum() + ), + "clipped_mass_after": float( + (bounded[moved] * recipient_weights[moved]).sum() + ), + "largest_weighted_draw_before": float(weighted_before.max(initial=0.0)), + "largest_weighted_draw_after": float(weighted_after.max(initial=0.0)), + } + ) for column in outputs: if column in _PUF_TAX_DETAIL_NONNEGATIVE_OUTPUTS: @@ -2674,6 +2792,117 @@ def _weighted_positive_donor_quantile( ) +def _top_slice_means( + values: np.ndarray, + weights: np.ndarray, + slice_weights: np.ndarray, +) -> np.ndarray: + """Weighted mean of each requested top slice of a donor column. + + The donors are ordered from the largest value down. A slice of donor weight + ``t`` takes whole donors until their weight reaches ``t`` and the needed + fraction of the donor at the edge, so the slice holds exactly ``t``. A + zero-weight slice has no mean; it reads the largest donor value, which no + draw can exceed, so the bound it implies is vacuous. + """ + + order = np.argsort(-values, kind="stable") + ordered_values = values[order] + ordered_weights = weights[order] + cumulative_weight = np.cumsum(ordered_weights) + cumulative_mass = np.cumsum(ordered_values * ordered_weights) + take = np.clip(slice_weights, 0.0, cumulative_weight[-1]) + edge = np.minimum( + np.searchsorted(cumulative_weight, take, side="left"), + len(ordered_values) - 1, + ) + previous = edge - 1 + weight_before = np.where( + previous >= 0, cumulative_weight[np.maximum(previous, 0)], 0.0 + ) + mass_before = np.where(previous >= 0, cumulative_mass[np.maximum(previous, 0)], 0.0) + mass = mass_before + (take - weight_before) * ordered_values[edge] + with np.errstate(divide="ignore", invalid="ignore"): + means = np.where(take > 0.0, mass / take, ordered_values[0]) + # The mean of a slice lies between its inner edge and the largest value. + # Running sums can miss that by the last bits (a slice made only of donors + # at one value can come out a hair below it), which would clip draws that + # equal the slice's own values, so the exact bounds are restored here. + return np.clip(means, ordered_values[edge], ordered_values[0]) + + +def _recipient_share_slice_bounds( + donor_values: Sequence[Any], + donor_weights: Sequence[Any], + recipient_weights: Sequence[Any], +) -> tuple[np.ndarray, np.ndarray]: + """Return each recipient's (floor, cap) under the recipient-share bound. + + A recipient whose weight is the share ``s`` of all PUF recipients stands + for ``s`` of the population. The most positive mass it can carry on an + amount column is the positive mass of the donor slice holding the same + share ``s`` of donor weight at the top of the column, so a positive value + is capped at that mass per unit of slice weight; losses are floored the + same way from the bottom slice. Within the positive donors the cap is the + slice's weighted mean, and it falls as the share grows. A side is infinite + only when the donor has no value of that sign. Shares make the bound + independent of either population's weight scale. + """ + + values = pd.to_numeric(pd.Series(donor_values), errors="coerce").to_numpy( + dtype=np.float64 + ) + weights = pd.to_numeric(pd.Series(donor_weights), errors="coerce").to_numpy( + dtype=np.float64 + ) + recipients = np.asarray(recipient_weights, dtype=np.float64) + if values.ndim != 1 or values.shape != weights.shape: + raise ValueError( + "Recipient-share bound donor values and weights must be aligned " + f"1-D arrays, got {values.shape} and {weights.shape}." + ) + if recipients.ndim != 1: + raise ValueError("Recipient-share bound recipient weights must be 1-D.") + if len(values) == 0: + raise ValueError("Recipient-share bound needs at least one donor.") + if not (np.isfinite(values).all() and np.isfinite(weights).all()): + raise ValueError("Recipient-share bound donor values must be finite.") + if not np.isfinite(recipients).all(): + raise ValueError("Recipient-share bound recipient weights must be finite.") + if (weights < 0.0).any() or (recipients < 0.0).any(): + raise ValueError("Recipient-share bound weights must be nonnegative.") + donor_total = float(weights.sum()) + recipient_total = float(recipients.sum()) + if donor_total <= 0.0 or recipient_total <= 0.0: + raise ValueError("Recipient-share bound weights must have positive mass.") + slice_weights = recipients / recipient_total * donor_total + # Ordering by the positive part orders the top slice exactly as the values + # do; donors at or below zero tie at zero and add no positive mass. + top = _top_slice_means(np.maximum(values, 0.0), weights, slice_weights) + bottom = -_top_slice_means(np.maximum(-values, 0.0), weights, slice_weights) + cap = np.where(top > 0.0, top, np.inf) + floor = np.where(bottom < 0.0, bottom, -np.inf) + return floor, cap + + +def _largest_grid_values_within(caps: np.ndarray, grid: np.ndarray) -> np.ndarray: + """Lower each finite cap to the largest ``grid`` value not above it. + + A capped value that is itself a grid value snaps to itself, and every value + at or below it snaps to a grid value at or below it, so snapping after the + bound cannot carry a value past it. ``grid`` is sorted and starts at its + smallest value; a cap below it keeps that smallest value. + """ + + lowered = np.array(caps, dtype=np.float64, copy=True) + finite = np.isfinite(lowered) + if not finite.any(): + return lowered + position = np.searchsorted(grid, lowered[finite], side="right") - 1 + lowered[finite] = grid[np.maximum(position, 0)] + return lowered + + def _tax_unit_household_weights( frame: Frame, tax_unit_ids: Sequence[Any], @@ -3564,6 +3793,13 @@ def _ensure_boolean_output_column(table: pd.DataFrame, column: str) -> None: ) +def _observed_snap_grid(observed: Sequence[Any]) -> np.ndarray: + """The sorted donor values :func:`_snap_to_observed_values` snaps onto.""" + + observed_values = pd.to_numeric(pd.Series(observed), errors="coerce").fillna(0.0) + return np.unique(np.rint(observed_values.to_numpy(dtype=np.float64)).clip(min=0.0)) + + def _snap_to_observed_values( values: Sequence[Any], observed: Sequence[Any], @@ -3579,10 +3815,7 @@ def _snap_to_observed_values( """ value_array = pd.to_numeric(pd.Series(values), errors="coerce").fillna(0.0) - observed_values = pd.to_numeric(pd.Series(observed), errors="coerce").fillna(0.0) - observed_array = np.unique( - np.rint(observed_values.to_numpy(dtype=np.float64)).clip(min=0.0) - ) + observed_array = _observed_snap_grid(observed) if len(observed_array) == 0: return value_array.to_numpy(dtype=np.float64) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py b/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py index d71320311f..340700bff6 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py @@ -180,6 +180,7 @@ bind_puf_clone_attachment_tail_descendant, clone_us_frame_for_puf_support, impute_us_puf_tax_detail_support, + puf_tax_detail_recipient_share_bound_identity, puf_tax_detail_tail_bound_quantiles_identity, validate_puf_clone_attachment, ) @@ -5406,7 +5407,7 @@ def _late_resource_binding_schema_version( kind = _late_virtual_resource_kind(column) versions = { "acs_pums_earnings_universe_execution_config": 2, - "primary_puf_execution_config": 5, + "primary_puf_execution_config": 6, "post_clone_source_execution_config": 3, "source_finalizer_execution_config": 2, "late_transfer_model_config": 3, @@ -5607,8 +5608,13 @@ def require_positive_integer(value: object, *, label: str) -> int: "tax_unit_outputs", "invocation_mode", "tail_bound_quantiles", + "recipient_share_bound", "worker_execution", } + if legacy_worker_authentication is not None: + # The attested schema-4 pool predates the recipient-share bound and + # truthfully records none; it is admitted for scoring only. + qrf_keys.discard("recipient_share_bound") if not isinstance(qrf, Mapping) or set(qrf) != qrf_keys: raise ValueError(f"{boundary}: late primary-QRF config is malformed.") require_nonnegative_integer(qrf.get("seed"), label="QRF seed") @@ -5644,6 +5650,12 @@ def require_positive_integer(value: object, *, label: str) -> int: puf_tax_detail_tail_bound_quantiles_identity() ): raise ValueError(f"{boundary}: late primary-QRF tail bounds changed.") + if legacy_worker_authentication is None and qrf.get( + "recipient_share_bound" + ) != (puf_tax_detail_recipient_share_bound_identity()): + raise ValueError( + f"{boundary}: late primary-QRF recipient-share bound changed." + ) recorded_worker = qrf.get("worker_execution") if legacy_worker_authentication is not None: if ( @@ -6110,7 +6122,7 @@ def _late_primary_execution_config_binding( ) return { "resource_kind": "primary_puf_execution_config", - "schema_version": 5, + "schema_version": 6, "clone_attachment": { "fraction": float(clone_attachment_fraction), "seed": clone_attachment_seed, @@ -6138,6 +6150,7 @@ def _late_primary_execution_config_binding( ), }, "tail_bound_quantiles": (puf_tax_detail_tail_bound_quantiles_identity()), + "recipient_share_bound": (puf_tax_detail_recipient_share_bound_identity()), "worker_execution": _late_primary_qrf_worker_execution_binding(), }, "doctrines": { diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_multispine_pool_h5_io.py b/packages/microcosm-build/tests/engine_free/us/test_us_multispine_pool_h5_io.py index 68c5646e06..866f628ee2 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_multispine_pool_h5_io.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_multispine_pool_h5_io.py @@ -1382,6 +1382,8 @@ def _rewrite_as_legacy_relocated_worker_pool( } config["schema_version"] = 4 config["qrf"]["worker_execution"] = recorded_worker + # A schema-4 pool predates the recipient-share bound and records none. + config["qrf"].pop("recipient_share_bound") config_receipt["binding_sha256"] = _json_sha256(config) config_receipt_sha256 = _json_sha256(config_receipt) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_puf_recipient_share_bound.py b/packages/microcosm-build/tests/engine_free/us/test_us_puf_recipient_share_bound.py new file mode 100644 index 0000000000..9b0600730a --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/us/test_us_puf_recipient_share_bound.py @@ -0,0 +1,740 @@ +"""The PUF recipient-share bound: one draw cannot outweigh the slice it stands for. + +A PUF-clone recipient whose weight is the share ``s`` of the recipient +population stands for ``s`` of the population. The bound limits its raw draw +to the weighted mean of the donor slice holding the same share at that end of +the distribution, so ``w_r * value <= (W_r / W_d) * mass(slice)``. +""" + +from __future__ import annotations + +import numpy as np +import pytest +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +import microcosm.build.us_runtime.puf_support as puf_support_module + +_bounds = puf_support_module._recipient_share_slice_bounds +_grid_within = puf_support_module._largest_grid_values_within +_snap = puf_support_module._snap_to_observed_values + +_PROPERTY_SETTINGS = settings( + max_examples=300, + deadline=None, + suppress_health_check=[HealthCheck.too_slow], +) + +_amounts = st.floats( + min_value=-1e9, max_value=1e9, allow_nan=False, allow_infinity=False +).map(lambda value: float(np.round(value, 2))) +_positive_weights = st.floats( + min_value=1e-3, max_value=1e4, allow_nan=False, allow_infinity=False +) + + +@st.composite +def _donor_and_recipients(draw: st.DrawFn) -> tuple[np.ndarray, ...]: + donors = draw(st.integers(min_value=1, max_value=40)) + values = np.asarray(draw(st.lists(_amounts, min_size=donors, max_size=donors))) + weights = np.asarray( + draw(st.lists(_positive_weights, min_size=donors, max_size=donors)) + ) + recipients = draw(st.integers(min_value=1, max_value=25)) + recipient_weights = np.asarray( + draw(st.lists(_positive_weights, min_size=recipients, max_size=recipients)) + ) + return values, weights, recipient_weights + + +def _reference_slice_mean( + values: np.ndarray, weights: np.ndarray, take: float, *, top: bool +) -> float: + """One slice's same-sign mass per unit weight, donor by donor.""" + + order = sorted(range(len(values)), key=lambda i: -values[i] if top else values[i]) + remaining = take + mass = 0.0 + for index in order: + used = min(weights[index], remaining) + part = max(values[index], 0.0) if top else min(values[index], 0.0) + mass += used * part + remaining -= used + if remaining <= 0.0: + break + return mass / take + + +def _reference_bounds( + values: np.ndarray, weights: np.ndarray, recipient_weights: np.ndarray +) -> tuple[np.ndarray, np.ndarray]: + donor_total = weights.sum() + recipient_total = recipient_weights.sum() + floors, caps = [], [] + for weight in recipient_weights: + take = weight / recipient_total * donor_total + top = _reference_slice_mean(values, weights, take, top=True) + bottom = _reference_slice_mean(values, weights, take, top=False) + caps.append(top if top > 0.0 else np.inf) + floors.append(bottom if bottom < 0.0 else -np.inf) + return np.asarray(floors), np.asarray(caps) + + +def _close(actual: np.ndarray, expected: np.ndarray) -> None: + """Equal infinities, and finite values within float accumulation error.""" + + np.testing.assert_array_equal(np.isinf(actual), np.isinf(expected)) + finite = np.isfinite(expected) + np.testing.assert_allclose(actual[finite], expected[finite], rtol=1e-9, atol=1e-6) + + +def _top_slice_edge(values: np.ndarray, weights: np.ndarray, take: float) -> float: + """The value at the inner edge of the top slice holding ``take`` weight.""" + + order = np.argsort(-values, kind="stable") + cumulative = np.cumsum(weights[order]) + edge = min(int(np.searchsorted(cumulative, take, side="left")), len(values) - 1) + return float(values[order][edge]) + + +def test_bound_is_the_slice_mean_on_a_worked_example() -> None: + # Donor population of weight 1,000: 990 at 10, 9 at 1,000, 1 at 100,000. + values = np.asarray([10.0, 1_000.0, 100_000.0, -50.0]) + weights = np.asarray([980.0, 9.0, 1.0, 10.0]) + # Recipients of share 0.001, 0.01 and 0.989 of their own population. + recipient_weights = np.asarray([1.0, 10.0, 989.0]) + + floor, cap = _bounds(values, weights, recipient_weights) + + # Share 0.001 of 1,000 donor weight is the 100,000 record alone. + assert cap[0] == 100_000.0 + # Share 0.01 is weight 10: the 100,000 record and all 9 at 1,000. + assert cap[1] == pytest.approx((100_000.0 + 9 * 1_000.0) / 10) + # The bottom slice of weight 10 is the ten records at -50. + assert floor[1] == -50.0 + # Share 0.989 is weight 989: on top, everything but 11 of the 980 at 10. + top_mass = 100_000.0 + 9_000.0 + 979.0 * 10.0 + assert cap[2] == pytest.approx(top_mass / 989.0) + # The bottom slice of weight 989 holds every loss, so the heaviest + # recipient may carry at most the population's whole loss mass. + assert floor[2] == pytest.approx(-500.0 / 989.0) + assert floor[0] == -50.0 + + +def test_one_recipient_is_bounded_by_the_donor_mean() -> None: + values = np.asarray([1.0, 2.0, 1_000.0]) + weights = np.asarray([10.0, 10.0, 1.0]) + + floor, cap = _bounds(values, weights, np.asarray([7.0])) + + assert cap[0] == pytest.approx(1_030.0 / 21.0) + assert floor[0] == -np.inf + + +def test_a_slice_of_tied_donors_caps_exactly_at_their_value() -> None: + # Statutory caps tie many donors at one value: a slice made only of them + # must cap at that value exactly, or draws equal to it would be clipped. + weights = np.asarray([0.1, 3.7, 1234.5, 0.3, 77.0] * 40) + values = np.full(len(weights), 2_500.0) + values[::7] = 1_000.0 + recipient_weights = np.linspace(0.5, 900.0, 64) + + _, cap = _bounds(values, weights, recipient_weights / recipient_weights.sum()) + + top_tie_weight = weights[values == 2_500.0].sum() + within = recipient_weights / recipient_weights.sum() * weights.sum() <= ( + top_tie_weight + ) + assert within.any() + assert (cap[within] == 2_500.0).all() + assert (cap <= 2_500.0).all() + + +def test_a_column_with_no_positive_donor_has_no_cap() -> None: + floor, cap = _bounds( + np.asarray([0.0, -5.0, -1.0]), + np.asarray([1.0, 1.0, 1.0]), + np.asarray([1.0, 2.0]), + ) + assert np.isinf(cap).all() + assert (floor < 0.0).all() + + +@pytest.mark.parametrize( + ("donor_values", "donor_weights", "recipient_weights", "message"), + [ + ([1.0], [1.0, 2.0], [1.0], "aligned"), + ([], [], [1.0], "at least one donor"), + ([np.nan], [1.0], [1.0], "finite"), + ([1.0], [1.0], [np.inf], "finite"), + ([1.0], [-1.0], [1.0], "nonnegative"), + ([1.0], [1.0], [-1.0], "nonnegative"), + ([1.0], [0.0], [1.0], "positive mass"), + ([1.0], [1.0], [0.0], "positive mass"), + ], +) +def test_bound_refuses_malformed_inputs( + donor_values: list[float], + donor_weights: list[float], + recipient_weights: list[float], + message: str, +) -> None: + with pytest.raises(ValueError, match=message): + _bounds( + np.asarray(donor_values, dtype=np.float64), + np.asarray(donor_weights, dtype=np.float64), + np.asarray(recipient_weights, dtype=np.float64), + ) + + +@_PROPERTY_SETTINGS +@given(_donor_and_recipients()) +def test_vectorized_bound_matches_the_donor_by_donor_reference( + case: tuple[np.ndarray, ...], +) -> None: + values, weights, recipient_weights = case + + floor, cap = _bounds(values, weights, recipient_weights) + expected_floor, expected_cap = _reference_bounds(values, weights, recipient_weights) + + _close(cap, expected_cap) + _close(floor, expected_floor) + + +@_PROPERTY_SETTINGS +@given(_donor_and_recipients()) +def test_one_record_never_carries_more_than_its_slice_mass( + case: tuple[np.ndarray, ...], +) -> None: + values, weights, recipient_weights = case + donor_total = weights.sum() + recipient_total = recipient_weights.sum() + + floor, cap = _bounds(values, weights, recipient_weights) + + for index, weight in enumerate(recipient_weights): + take = weight / recipient_total * donor_total + scale = recipient_total / donor_total + if np.isfinite(cap[index]): + top_mass = take * _reference_slice_mean(values, weights, take, top=True) + assert weight * cap[index] == pytest.approx(scale * top_mass, rel=1e-9) + if np.isfinite(floor[index]): + bottom_mass = take * _reference_slice_mean(values, weights, take, top=False) + assert weight * floor[index] == pytest.approx(scale * bottom_mass, rel=1e-9) + + +@_PROPERTY_SETTINGS +@given(_donor_and_recipients()) +def test_bound_stays_inside_the_donor_support_and_slice( + case: tuple[np.ndarray, ...], +) -> None: + values, weights, recipient_weights = case + donor_total = weights.sum() + recipient_total = recipient_weights.sum() + + floor, cap = _bounds(values, weights, recipient_weights) + + for index, weight in enumerate(recipient_weights): + take = weight / recipient_total * donor_total + if np.isfinite(cap[index]): + assert 0.0 < cap[index] <= values.max() + # The mean of the slice is at least its inner edge, exactly. + edge = _top_slice_edge(values, weights, take) + assert cap[index] >= edge + if np.isfinite(floor[index]): + assert values.min() <= floor[index] < 0.0 + edge = -_top_slice_edge(-values, weights, take) + assert floor[index] <= edge + + +@_PROPERTY_SETTINGS +@given(_donor_and_recipients()) +def test_heavier_recipients_get_tighter_bounds(case: tuple[np.ndarray, ...]) -> None: + values, weights, recipient_weights = case + + floor, cap = _bounds(values, weights, recipient_weights) + + order = np.argsort(recipient_weights, kind="stable") + ordered_cap = cap[order] + ordered_floor = floor[order] + tolerance = 1e-9 * max(1.0, float(np.abs(values).max())) + assert (np.diff(ordered_cap[np.isfinite(ordered_cap)]) <= tolerance).all() + assert (np.diff(ordered_floor[np.isfinite(ordered_floor)]) >= -tolerance).all() + # A side applies to every recipient or to none: it is infinite only when + # the donor has no value of that sign. + assert np.isfinite(cap).all() == bool((values > 0.0).any()) + assert np.isinf(cap).all() == (not (values > 0.0).any()) + assert np.isfinite(floor).all() == bool((values < 0.0).any()) + assert np.isinf(floor).all() == (not (values < 0.0).any()) + + +@_PROPERTY_SETTINGS +@given( + _donor_and_recipients(), + st.floats(min_value=1e-3, max_value=1e3), + st.floats(min_value=1e-3, max_value=1e3), +) +def test_bound_depends_only_on_weight_shares( + case: tuple[np.ndarray, ...], donor_scale: float, recipient_scale: float +) -> None: + values, weights, recipient_weights = case + + floor, cap = _bounds(values, weights, recipient_weights) + scaled_floor, scaled_cap = _bounds( + values, weights * donor_scale, recipient_weights * recipient_scale + ) + + _close(scaled_cap, cap) + _close(scaled_floor, floor) + + +@_PROPERTY_SETTINGS +@given(_donor_and_recipients(), st.randoms(use_true_random=False)) +def test_bound_ignores_donor_order_and_follows_recipient_order( + case: tuple[np.ndarray, ...], random: object +) -> None: + values, weights, recipient_weights = case + donor_order = list(range(len(values))) + recipient_order = list(range(len(recipient_weights))) + random.shuffle(donor_order) # type: ignore[attr-defined] + random.shuffle(recipient_order) # type: ignore[attr-defined] + + floor, cap = _bounds(values, weights, recipient_weights) + shuffled_floor, shuffled_cap = _bounds( + values[donor_order], weights[donor_order], recipient_weights[recipient_order] + ) + + _close(shuffled_cap, cap[recipient_order]) + _close(shuffled_floor, floor[recipient_order]) + + +@_PROPERTY_SETTINGS +@given(_donor_and_recipients(), st.floats(min_value=0.01, max_value=0.99)) +def test_splitting_a_donor_record_leaves_the_bound_unchanged( + case: tuple[np.ndarray, ...], fraction: float +) -> None: + values, weights, recipient_weights = case + split_values = np.concatenate([values, values[:1]]) + split_weights = np.concatenate( + [[weights[0] * fraction], weights[1:], [weights[0] * (1.0 - fraction)]] + ) + + floor, cap = _bounds(values, weights, recipient_weights) + split_floor, split_cap = _bounds(split_values, split_weights, recipient_weights) + + _close(split_cap, cap) + _close(split_floor, floor) + + +@_PROPERTY_SETTINGS +@given( + st.lists( + st.floats(min_value=0.0, max_value=1e8, allow_nan=False).map(round), + min_size=1, + max_size=30, + ), + st.lists( + st.floats(min_value=-1e8, max_value=1e9, allow_nan=False), + min_size=1, + max_size=30, + ), + st.floats(min_value=0.0, max_value=1e9, allow_nan=False), +) +def test_snapping_after_the_lowered_cap_never_crosses_it( + observed: list[float], draws: list[float], cap: float +) -> None: + grid = np.unique(np.rint(np.asarray(observed, dtype=np.float64)).clip(min=0.0)) + caps = np.full(len(draws), cap) + lowered = _grid_within(caps, grid) + bounded = np.minimum(np.asarray(draws, dtype=np.float64), lowered) + + snapped = _snap(bounded, observed) + + assert (lowered <= np.maximum(cap, grid[0])).all() + assert np.isin(lowered, grid).all() + assert (snapped <= lowered).all() + + +def test_lowering_keeps_infinite_caps_infinite() -> None: + grid = np.asarray([0.0, 5.0, 10.0]) + lowered = _grid_within(np.asarray([np.inf, 7.0, 10.0, 3.0]), grid) + np.testing.assert_array_equal(lowered, [np.inf, 5.0, 10.0, 0.0]) + + +# --------------------------------------------------------------------------- # +# The bound at the finalize seam # +# --------------------------------------------------------------------------- # + +import pandas as pd # noqa: E402 + +from microcosm.build.us_runtime import ( # noqa: E402 + PUF_TAX_DETAIL_SUPPORT_CHANNEL, + clone_us_frame_for_puf_support, + support_channel_column, + support_source_id_column, +) +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights # noqa: E402 + +_PASSTHROUGH = "partnership_income" +_SNAPPED = "taxable_interest_income" +_CALIBRATED = "farm_operations_income" +_BOOLEAN = "business_is_sstb" + + +def _recipient_frame(household_weights: list[float]) -> Frame: + count = len(household_weights) + ids = np.arange(1, count + 1, dtype=np.int64) + tables = { + "person": pd.DataFrame( + { + "person_id": ids, + "person_household_id": ids, + "person_tax_unit_id": ids, + "person_spm_unit_id": ids, + "person_family_id": ids, + "person_marital_unit_id": ids, + } + ), + "household": pd.DataFrame( + {"household_id": ids, "state_fips": np.full(count, 6, dtype=np.int64)} + ), + "tax_unit": pd.DataFrame( + {"tax_unit_id": ids, "filing_status_input": ["SINGLE"] * count} + ), + "spm_unit": pd.DataFrame({"spm_unit_id": ids}), + "family": pd.DataFrame({"family_id": ids}), + "marital_unit": pd.DataFrame({"marital_unit_id": ids}), + } + return clone_us_frame_for_puf_support( + Frame( + tables, + US_SCHEMA, + { + "household": Weights( + np.asarray(household_weights, dtype=np.float64), + WeightKind.DESIGN, + ) + }, + ) + ) + + +def _recipient_index(frame: Frame) -> pd.Index: + tax_unit = frame.table("tax_unit") + return tax_unit.index[ + tax_unit[support_channel_column("tax_unit")] == PUF_TAX_DETAIL_SUPPORT_CHANNEL + ] + + +def _puf_values(frame: Frame, column: str) -> np.ndarray: + person = frame.table("person") + puf = person[ + person[support_channel_column("person")] == PUF_TAX_DETAIL_SUPPORT_CHANNEL + ].sort_values(support_source_id_column("person")) + return puf[column].to_numpy(dtype=np.float64) + + +def _recipient_weights(frame: Frame) -> np.ndarray: + ids = frame.table("tax_unit").loc[_recipient_index(frame), "tax_unit_id"] + return puf_support_module._tax_unit_household_weights(frame, ids) + + +def _finalize( + frame: Frame, + donor: pd.DataFrame, + raw: dict[str, np.ndarray], + *, + diagnostics: list[dict[str, object]] | None, + **kwargs: object, +) -> Frame: + predictions = pd.DataFrame( + { + column: np.asarray(values, dtype=np.float64) + for column, values in raw.items() + }, + index=_recipient_index(frame), + ) + return puf_support_module.finalize_us_puf_tax_detail_predictions( + frame, + donor, + predictions, + person_outputs=tuple(raw), + tax_unit_outputs=(), + tail_bound_diagnostics=diagnostics, + **kwargs, + ) + + +def _heavy_tail_donor(column: str, *, losses: bool = True) -> pd.DataFrame: + # 1,000 donor units: most small, nine large, one extreme record of weight + # 0.13 (the Forbes-tail synthetic records' weight), and a loss tail. + values = [100.0, 5_000.0, 2.0e8, -40_000.0 if losses else 0.0, 0.0] + weights = [900.0, 9.0, 0.13, 10.0, 80.87] + return pd.DataFrame({column: values, "weight": weights}) + + +def test_one_extreme_draw_is_bounded_by_its_slice_and_recorded() -> None: + # Recipient shares 0.01, 0.49, 0.01 and 0.49. The light recipients' slices + # hold weight 10: the top one reaches the extreme record, the bottom one is + # exactly the loss records, so neither of their draws moves. + frame = _recipient_frame([20.0, 980.0, 20.0, 980.0]) + donor = _heavy_tail_donor(_PASSTHROUGH) + raw = np.asarray([150.0, 2.0e8, -40_000.0, 4_000.0]) + diagnostics: list[dict[str, object]] = [] + + finalized = _finalize(frame, donor, {_PASSTHROUGH: raw}, diagnostics=diagnostics) + + weights = _recipient_weights(frame) + floor, cap = _bounds(donor[_PASSTHROUGH], donor["weight"], weights) + expected = np.minimum(np.maximum(raw, floor), cap) + actual = _puf_values(finalized, _PASSTHROUGH) + np.testing.assert_array_equal(actual, expected) + # Only the extreme draw moved; its weighted mass fell to its slice's mass. + assert actual[1] < raw[1] + np.testing.assert_array_equal(actual[[0, 2, 3]], raw[[0, 2, 3]]) + assert diagnostics == [ + { + "output": _PASSTHROUGH, + "rule": puf_support_module.PUF_TAX_DETAIL_RECIPIENT_SHARE_BOUND_RULE, + "clipped_row_count": 1, + "capped_row_count": 1, + "floored_row_count": 0, + "clipped_mass_before": float(raw[1] * weights[1]), + "clipped_mass_after": float(expected[1] * weights[1]), + "largest_weighted_draw_before": float(np.abs(raw * weights).max()), + "largest_weighted_draw_after": float(np.abs(expected * weights).max()), + } + ] + + +_FINALIZE_SETTINGS = settings( + max_examples=60, + deadline=None, + suppress_health_check=[HealthCheck.too_slow, HealthCheck.data_too_large], +) + + +@st.composite +def _finalize_case(draw: st.DrawFn) -> tuple[list[float], pd.DataFrame, np.ndarray]: + recipients = draw(st.integers(min_value=1, max_value=8)) + household_weights = draw( + st.lists( + st.floats(min_value=0.5, max_value=5e3), + min_size=recipients, + max_size=recipients, + ) + ) + donors = draw(st.integers(min_value=1, max_value=12)) + donor = pd.DataFrame( + { + _PASSTHROUGH: draw(st.lists(_amounts, min_size=donors, max_size=donors)), + "weight": draw( + st.lists(_positive_weights, min_size=donors, max_size=donors) + ), + } + ) + raw = np.asarray(draw(st.lists(_amounts, min_size=recipients, max_size=recipients))) + return household_weights, donor, raw + + +@_FINALIZE_SETTINGS +@given(_finalize_case()) +def test_finalized_draws_obey_the_bound_and_move_only_beyond_it( + case: tuple[list[float], pd.DataFrame, np.ndarray], +) -> None: + household_weights, donor, raw = case + frame = _recipient_frame(household_weights) + + finalized = _finalize(frame, donor, {_PASSTHROUGH: raw}, diagnostics=[]) + + floor, cap = _bounds( + donor[_PASSTHROUGH], donor["weight"], _recipient_weights(frame) + ) + actual = _puf_values(finalized, _PASSTHROUGH) + # Bound. + assert (actual <= cap).all() and (actual >= floor).all() + # Untouched inside the bound, bit for bit. + inside = (raw <= cap) & (raw >= floor) + np.testing.assert_array_equal( + actual[inside].view(np.uint64), raw[inside].view(np.uint64) + ) + # Sign and zero preserving, shrink only. + np.testing.assert_array_equal(np.sign(actual), np.sign(raw)) + assert (np.abs(actual) <= np.abs(raw)).all() + # Moved draws land exactly on their own bound. + np.testing.assert_array_equal(actual[raw > cap], cap[raw > cap]) + np.testing.assert_array_equal(actual[raw < floor], floor[raw < floor]) + + +@_FINALIZE_SETTINGS +@given(_finalize_case()) +def test_diagnostics_account_exactly_for_the_moved_mass( + case: tuple[list[float], pd.DataFrame, np.ndarray], +) -> None: + household_weights, donor, raw = case + frame = _recipient_frame(household_weights) + diagnostics: list[dict[str, object]] = [] + + finalized = _finalize(frame, donor, {_PASSTHROUGH: raw}, diagnostics=diagnostics) + + weights = _recipient_weights(frame) + actual = _puf_values(finalized, _PASSTHROUGH) + (record,) = diagnostics + moved = actual != raw + assert record["clipped_row_count"] == int(moved.sum()) + assert record["capped_row_count"] + record["floored_row_count"] == int(moved.sum()) + assert record["clipped_mass_before"] == pytest.approx( + float((raw[moved] * weights[moved]).sum()), rel=1e-12, abs=1e-6 + ) + assert record["clipped_mass_after"] == pytest.approx( + float((actual[moved] * weights[moved]).sum()), rel=1e-12, abs=1e-6 + ) + total_before = float((raw * weights).sum()) + total_after = float((actual * weights).sum()) + assert total_before - total_after == pytest.approx( + record["clipped_mass_before"] - record["clipped_mass_after"], + rel=1e-9, + abs=1e-3, + ) + + +def test_snapped_outputs_stay_on_observed_values_within_the_bound() -> None: + # A donor positive rate above the recipients' keeps the donor-rate + # sparsification idle, so the finalized values are the snapped bounded draws. + frame = _recipient_frame([20.0, 400.0, 600.0, 980.0]) + donor = pd.DataFrame( + {_SNAPPED: [100.0, 5_000.0, 2.0e8, 0.0], "weight": [990.0, 9.0, 0.13, 0.87]} + ) + raw = np.asarray([0.0, 1.9e8, 7_000.4, 4_000.0]) + + finalized = _finalize(frame, donor, {_SNAPPED: raw}, diagnostics=[]) + + _, cap = _bounds(donor[_SNAPPED], donor["weight"], _recipient_weights(frame)) + actual = _puf_values(finalized, _SNAPPED) + grid = puf_support_module._observed_snap_grid(donor[_SNAPPED]) + assert (actual <= cap).all() + assert np.isin(actual, grid).all() + assert actual[1] < raw[1] + # The cap of the heaviest-share draw is not itself a donor value; the draw + # lands on the largest donor value within it. + assert actual[1] == grid[grid <= cap[1]].max() + + +def test_sparsification_rescales_bounded_draws_by_one_common_factor() -> None: + # Every recipient draws positive while the donor is 90.9% positive, so the + # donor-rate sparsification zeroes the smallest draw and rescales the rest + # to keep the column's mass: the bound holds before that, and after it the + # kept draws all carry the same factor. + frame = _recipient_frame([20.0, 400.0, 600.0, 980.0]) + donor = _heavy_tail_donor(_SNAPPED, losses=False) + raw = np.asarray([150.0, 1.9e8, 7_000.0, 4_000.0]) + + finalized = _finalize(frame, donor, {_SNAPPED: raw}, diagnostics=[]) + + weights = _recipient_weights(frame) + floor, cap = _bounds(donor[_SNAPPED], donor["weight"], weights) + grid = puf_support_module._observed_snap_grid(donor[_SNAPPED]) + lowered = puf_support_module._largest_grid_values_within(cap, grid) + before = puf_support_module._snap_to_observed_values( + np.minimum(np.maximum(raw, floor), lowered), donor[_SNAPPED] + ) + actual = _puf_values(finalized, _SNAPPED) + assert (before <= cap).all() + kept = actual > 0.0 + assert kept.any() and not kept.all() + factors = actual[kept] / before[kept] + np.testing.assert_allclose(factors, factors[0], rtol=1e-12) + assert float((actual * weights).sum()) == pytest.approx( + float((before * weights).sum()), rel=1e-12 + ) + + +def test_signed_mass_calibration_still_pins_both_legs_after_the_bound() -> None: + frame = _recipient_frame([20.0, 400.0, 600.0, 980.0, 300.0]) + donor = _heavy_tail_donor(_CALIBRATED) + raw = np.asarray([150.0, 2.0e8, -40_000.0, 4_000.0, -100.0]) + + finalized = _finalize(frame, donor, {_CALIBRATED: raw}, diagnostics=[]) + + weights = _recipient_weights(frame) + actual = _puf_values(finalized, _CALIBRATED) + values = donor[_CALIBRATED].to_numpy() + donor_weights = donor["weight"].to_numpy() + for leg, donor_leg in ((actual > 0.0, values > 0.0), (actual < 0.0, values < 0.0)): + assert float((actual[leg] * weights[leg]).sum()) / weights.sum() == ( + pytest.approx( + float((values[donor_leg] * donor_weights[donor_leg]).sum()) + / donor_weights.sum(), + rel=1e-9, + ) + ) + # The calibration rescales whole legs, so the bound's effect survives it: + # the extreme draw no longer holds nearly all of the positive leg. + positive = actual > 0.0 + leg_mass = actual[positive] * weights[positive] + assert leg_mass.max() < 0.99 * leg_mass.sum() + + +def test_boolean_outputs_are_not_bounded() -> None: + frame = _recipient_frame([20.0, 400.0]) + donor = pd.DataFrame({_BOOLEAN: [0.0, 1.0], "weight": [999.0, 1.0]}) + diagnostics: list[dict[str, object]] = [] + + finalized = _finalize(frame, donor, {_BOOLEAN: [1.0, 1.0]}, diagnostics=diagnostics) + + assert diagnostics == [] + assert _puf_values(finalized, _BOOLEAN).tolist() == [1.0, 1.0] + + +def test_amount_outputs_require_a_diagnostics_sink() -> None: + frame = _recipient_frame([20.0, 400.0]) + with pytest.raises(ValueError, match="diagnostics sink"): + _finalize( + frame, + _heavy_tail_donor(_PASSTHROUGH), + {_PASSTHROUGH: [1.0, 2.0]}, + diagnostics=None, + ) + + +def test_explicit_narrowing_is_validated_and_can_empty_the_bound() -> None: + frame = _recipient_frame([20.0, 400.0]) + donor = _heavy_tail_donor(_PASSTHROUGH) + raw = np.asarray([150.0, 2.0e8]) + + unbounded = _finalize( + frame, + donor, + {_PASSTHROUGH: raw}, + diagnostics=None, + recipient_share_bound_outputs=(), + ) + np.testing.assert_array_equal(_puf_values(unbounded, _PASSTHROUGH), raw) + + for bad in ((_BOOLEAN,), ("not_an_output",), (_PASSTHROUGH, _PASSTHROUGH)): + with pytest.raises(ValueError, match="distinct amount outputs"): + _finalize( + frame, + donor, + {_PASSTHROUGH: raw}, + diagnostics=[], + recipient_share_bound_outputs=bad, + ) + + +def test_bound_identity_is_exact_and_names_its_rule() -> None: + identity = puf_support_module.puf_tax_detail_recipient_share_bound_identity() + assert identity["rule"] == ( + puf_support_module.PUF_TAX_DETAIL_RECIPIENT_SHARE_BOUND_RULE + ) + assert ( + identity == puf_support_module.puf_tax_detail_recipient_share_bound_identity() + ) + canonical = ( + *puf_support_module.PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS, + *puf_support_module.PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS, + ) + bounded = puf_support_module._recipient_share_bound_amount_outputs(canonical) + assert "partnership_income" in bounded + assert "first_home_mortgage_origination_year" not in bounded + assert not set(bounded) & set(puf_support_module.US_QBI_BOOLEAN_OUTPUT_COLUMNS) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_puf_tail_bound.py b/packages/microcosm-build/tests/engine_free/us/test_us_puf_tail_bound.py index fbf7fe5618..d1919c2009 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_puf_tail_bound.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_puf_tail_bound.py @@ -96,6 +96,7 @@ def _finalize( *, person_outputs: tuple[str, ...] = (_TARGET,), diagnostics: list[dict[str, object]] | None = None, + **kwargs: object, ) -> Frame: return puf_support_module.finalize_us_puf_tax_detail_predictions( frame, @@ -104,6 +105,7 @@ def _finalize( person_outputs=person_outputs, tax_unit_outputs=(), tail_bound_diagnostics=diagnostics, + **kwargs, ) @@ -150,16 +152,20 @@ def test_finalizer_clips_tail_without_changing_participation_or_lower_bits() -> np.testing.assert_array_equal(actual, [0.0, 50.0, 100.0, 100.0]) assert np.count_nonzero(actual > 0.0) == np.count_nonzero(raw > 0.0) np.testing.assert_array_equal(actual[:2].view(np.uint64), raw[:2].view(np.uint64)) - assert diagnostics == [ - { - "output": _TARGET, - "quantile": 0.999, - "bound_value": 100.0, - "clipped_row_count": 2, - "clipped_mass_before": 1_903.0, - "clipped_mass_after": 700.0, - } - ] + assert diagnostics[0] == { + "output": _TARGET, + "quantile": 0.999, + "bound_value": 100.0, + "clipped_row_count": 2, + "clipped_mass_before": 1_903.0, + "clipped_mass_after": 700.0, + } + # The recipient-share bound runs after the quantile cap; every draw is + # already within its slice, so it records the output and moves nothing. + assert len(diagnostics) == 2 + assert diagnostics[1]["output"] == _TARGET + assert diagnostics[1]["rule"] == "recipient_share_slice_mass" + assert diagnostics[1]["clipped_row_count"] == 0 def test_nonconfigured_output_is_bit_identical_through_finalizer() -> None: @@ -174,12 +180,15 @@ def test_nonconfigured_output_is_bit_identical_through_finalizer() -> None: ) diagnostics: list[dict[str, object]] = [] + # Isolate the quantile map: the recipient-share bound covers every amount + # output and has its own tests (test_us_puf_recipient_share_bound.py). finalized = _finalize( frame, _tail_donor(**{_NONCONFIGURED_OUTPUT: [1.0, 2.0, 3.0]}), predictions, person_outputs=(_TARGET, _NONCONFIGURED_OUTPUT), diagnostics=diagnostics, + recipient_share_bound_outputs=(), ) actual = _puf_person_values(finalized, _NONCONFIGURED_OUTPUT) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_stacked_spine.py b/packages/microcosm-build/tests/engine_free/us/test_us_stacked_spine.py index 773b3b06e4..c1b3fb6553 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_stacked_spine.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_stacked_spine.py @@ -4180,7 +4180,7 @@ def test_late_primary_resources_bind_donor_content_and_execution_config( "tax_unit_outputs": "canonical_default", } execution = baseline["tax_unit.@primary_puf_execution_config"]["binding"] - assert execution["schema_version"] == 5 + assert execution["schema_version"] == 6 assert execution["clone_attachment"]["support_channels"] == [ stacked_spine_module.BASE_ASEC_SUPPORT_CHANNEL, stacked_spine_module.PUF_TAX_DETAIL_SUPPORT_CHANNEL, @@ -4191,6 +4191,12 @@ def test_late_primary_resources_bind_donor_content_and_execution_config( assert execution["qrf"]["tail_bound_quantiles"] == { "non_sch_d_capital_gains": 0.999 } + assert execution["qrf"]["recipient_share_bound"] == ( + stacked_spine_module.puf_tax_detail_recipient_share_bound_identity() + ) + assert execution["qrf"]["recipient_share_bound"]["rule"] == ( + "recipient_share_slice_mass" + ) assert execution["doctrines"]["whole_pool_output_universes"] == { "person.s_corp_income": { "rule_id": "puf_tax_detail_s_corp_income_universe_zero_v1", @@ -9255,3 +9261,42 @@ def test_end_to_end_stack_gap_fill_puf_pass_gates_and_battery(tmp_path) -> None: ] assert origin_detail.notna().all() assert (origin_detail > 0.0).any() + + +def _primary_execution_binding() -> dict[str, object]: + return stacked_spine_module._late_primary_execution_config_binding( + clone_attachment_fraction=1.0, + clone_attachment_seed=0, + seed=0, + n_estimators=32, + predictors=None, + person_outputs=None, + tax_unit_outputs=None, + fit_records_enabled=True, + tail_bound_diagnostics_enabled=True, + ) + + +def _validate_primary_execution_binding(binding: dict[str, object]) -> None: + stacked_spine_module._validate_late_resource_binding( + binding, + producer=stacked_spine_module.US_LATE_PRIMARY_PUF_STAGE, + entity="tax_unit", + column="@primary_puf_execution_config", + boundary="test boundary", + ) + + +def test_primary_execution_binding_carries_the_recipient_share_bound() -> None: + binding = _primary_execution_binding() + _validate_primary_execution_binding(binding) + + changed = deepcopy(binding) + changed["qrf"]["recipient_share_bound"]["cap"] = "top_slice_boundary" + with pytest.raises(ValueError, match="recipient-share bound changed"): + _validate_primary_execution_binding(changed) + + missing = deepcopy(binding) + missing["qrf"].pop("recipient_share_bound") + with pytest.raises(ValueError, match="late primary-QRF config is malformed"): + _validate_primary_execution_binding(missing) From d9f180852d1c9125e44e0d1bc2ef6680776da376 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 13 Oct 2026 09:15:17 -0400 Subject: [PATCH 2/8] Bound snapped outputs on their grid; give finalize callers a sink; repin - Snapped (sparse) outputs are bounded on their snap-grid donor values. A slice inside one grid value now caps at that value exactly; computing on unrounded values clipped every draw at an aged statutory maximum to the grid value below (1,163 qualified-tuition rows per seed in the sweep). - Every test that finalizes amount outputs passes a tail-bound diagnostics sink, now required because the recipient-share bound is always active. - Regenerated imputation.yaml (primary execution config schema 6), field usage counts and claim digest, inventory digests (seed protocol and map, producer registry, resource semantics, full checkpoint identity), the pointer inventory digest, coverage evidence, the pool-tool spec sha and the loader golden vector. Co-Authored-By: Claude Opus 5.5 --- docs/evidence/spec-engine/us-f0-coverage.json | 46 +++++++++---------- .../build/spec_engine/field_usage.py | 8 ++-- .../build/spec_engine/inventory_coverage.py | 13 +++--- .../microcosm/build/us/spec/imputation.yaml | 14 +++++- .../microcosm/build/us_runtime/puf_support.py | 34 ++++++++------ .../us/test_us_stacked_spine.py | 1 + .../shared/test_spec_engine_loader.py | 2 +- .../engine_free/us/test_puf_qrf_chain.py | 25 +++++++--- .../us/test_us_education_inputs.py | 1 + .../tests/engine_free/us/test_us_form_4952.py | 1 + .../us/test_us_puf_income_rank_predictors.py | 3 ++ .../us/test_us_puf_recipient_share_bound.py | 32 +++++++++++-- .../us/test_us_puf_social_security.py | 2 + .../engine_free/us/test_us_puf_support.py | 10 ++++ .../us/test_us_salt_refund_income.py | 1 + .../engine_free/us/test_us_stacked_spine.py | 6 +++ .../us/test_us_educator_expenses.py | 1 + .../us/test_us_multispine_pool_tool.py | 2 +- .../us_farm_business_income.py | 1 + .../microcosm_build/us_puf_support.py | 3 ++ tools/spec_engine_coverage.py | 2 +- 21 files changed, 148 insertions(+), 60 deletions(-) diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index bbced0b031..8649930da3 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -6,7 +6,7 @@ "country": "us", "documentation_sha256": "0931cbd3aae2abcd8c072b78eba40c1c006b1bf7fa23c6a6681f9a895e47070e", "field_usage": { - "authored_normative_field_count": 32404, + "authored_normative_field_count": 32414, "claim_count": 49, "claims": [ { @@ -274,8 +274,8 @@ ], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 24488, - "pointer_sha256": "6382a3ed07016414c31462b8bedb24590d92b792405a89c927d85fa1bd2a8347", + "pointer_count": 24498, + "pointer_sha256": "92968ea779967b45e7ea5b60722a253239947806de05bf46437cf386b2ae6d23", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/spec~1imputation.yaml/producer_graph", @@ -772,20 +772,20 @@ "verifier": "vintages" } ], - "configuration_field_count": 42184, - "consumed_field_count": 42184, + "configuration_field_count": 42194, + "consumed_field_count": 42194, "generation0_effect_counts": { - "legacy_behavior": 38498, + "legacy_behavior": 38508, "no_generation0_effect": 3686 }, "mode_counts": { - "compiler_semantic": 27717, + "compiler_semantic": 27727, "front_end_validation": 354, "identity_only": 103, "legacy_behavior": 14010 }, "multiple_primary_use_field_count": 0, - "pointer_inventory_sha256": "8f655670b9794adae905ede64bdceba6395eeb8d642b6c5896d5c516f1f6a6a7", + "pointer_inventory_sha256": "4523cb2c2ccca11f484b6fdd1efb860652b1fb1622a4e39675ba652b9136f4f6", "resolved_binding_field_count": 9780, "unused_field_count": 0 }, @@ -1338,12 +1338,12 @@ ], "expected": { "nodes": 38, - "sha256": "40cd51ffdfe2e9d9d08d48c08e8ded9de1e4b134783bab05c4abc6ad5c72ca1e" + "sha256": "e74fb1e6017cf77397708cbf2ad6e323dffd6be89425bc6544b5679e883cb674" }, "failures": [], "observed": { "nodes": 38, - "sha256": "40cd51ffdfe2e9d9d08d48c08e8ded9de1e4b134783bab05c4abc6ad5c72ca1e" + "sha256": "e74fb1e6017cf77397708cbf2ad6e323dffd6be89425bc6544b5679e883cb674" }, "status": "covered" }, @@ -1361,12 +1361,12 @@ ], "expected": { "producer_count": 38, - "sha256": "cc34ba11288d01145fc2dc1a669df3f7c4ef2019d73afb02dd8eeee852b16d20" + "sha256": "b2580f8725297ea535191416d37b77ccf891003b76db8e30d3c949e986b5cef6" }, "failures": [], "observed": { "producer_count": 38, - "sha256": "cc34ba11288d01145fc2dc1a669df3f7c4ef2019d73afb02dd8eeee852b16d20" + "sha256": "b2580f8725297ea535191416d37b77ccf891003b76db8e30d3c949e986b5cef6" }, "status": "covered" }, @@ -1656,13 +1656,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "41a64b8820bf161e316c49e3f2b666a247b58f6f6d50dd4395b89c8f5b1399f6", - "protocol_sha256": "0e4c8a4ace5e10747065669a009691d0eee9179ca42b344f8912c5ca239ebde6" + "map_sha256": "2eaa714e90df4c8b261741de9b24028a7f1074780d5d177ef74619e2944db63b", + "protocol_sha256": "f114fcb7f01afa54f3bf5ca65dc96342cdd69f5756774f1196ae32e9a71aad3d" }, "failures": [], "observed": { - "map_sha256": "41a64b8820bf161e316c49e3f2b666a247b58f6f6d50dd4395b89c8f5b1399f6", - "protocol_sha256": "0e4c8a4ace5e10747065669a009691d0eee9179ca42b344f8912c5ca239ebde6" + "map_sha256": "2eaa714e90df4c8b261741de9b24028a7f1074780d5d177ef74619e2944db63b", + "protocol_sha256": "f114fcb7f01afa54f3bf5ca65dc96342cdd69f5756774f1196ae32e9a71aad3d" }, "status": "covered" }, @@ -1677,7 +1677,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "0e4c8a4ace5e10747065669a009691d0eee9179ca42b344f8912c5ca239ebde6", + "implementation_sha256": "f114fcb7f01afa54f3bf5ca65dc96342cdd69f5756774f1196ae32e9a71aad3d", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1698,7 +1698,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "0e4c8a4ace5e10747065669a009691d0eee9179ca42b344f8912c5ca239ebde6", + "implementation_sha256": "f114fcb7f01afa54f3bf5ca65dc96342cdd69f5756774f1196ae32e9a71aad3d", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1887,7 +1887,7 @@ "alpha", "zeta" ], - "sha256": "3db095ce2d1ef89028d3862f0edc827d704f099c5b3323f9b115642a3c27e007" + "sha256": "d7b84ec565db19067d984e0596ca1805110038c38bec6b5d02ac89fde7f261cf" }, "failures": [], "observed": { @@ -1911,7 +1911,7 @@ "alpha", "zeta" ], - "sha256": "3db095ce2d1ef89028d3862f0edc827d704f099c5b3323f9b115642a3c27e007" + "sha256": "d7b84ec565db19067d984e0596ca1805110038c38bec6b5d02ac89fde7f261cf" }, "status": "covered" }, @@ -1975,7 +1975,7 @@ "take_up_contract", "us_qbi_reconciliation_contract" ], - "sha256": "61a5484eab9fae1d6651b3ae15f1793d4934ea2f6ed2c08d05716b745c221014" + "sha256": "fcb03fb129f808ee52a27c742a7d9a4f0f1fd9f1729f4be68dcfd7b8fdaca372" }, "status": "covered" }, @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "3080fe938df62216dc4853c1163639817de3c537e0e932309642fbd3f7d8f266" + "spec_sha256": "f56dd06c602f065546c659c353114d2c9a4f6a7cd7e13daadfec14f48b53152a" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "3080fe938df62216dc4853c1163639817de3c537e0e932309642fbd3f7d8f266" + "spec_sha256": "f56dd06c602f065546c659c353114d2c9a4f6a7cd7e13daadfec14f48b53152a" }, "status": "pass" } diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py index 56f39195cf..cf9deac40e 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py @@ -26,9 +26,9 @@ ) from .schemas import load_schema_registry -EXPECTED_AUTHORED_FIELD_COUNT = 32_404 +EXPECTED_AUTHORED_FIELD_COUNT = 32_414 EXPECTED_RESOLVED_BINDING_FIELD_COUNT = 9_780 -EXPECTED_CONFIGURATION_FIELD_COUNT = 42_184 +EXPECTED_CONFIGURATION_FIELD_COUNT = 42_194 class FieldUsageError(AssertionError): @@ -421,8 +421,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "e1dd7dc5123ab0f39d08ea4939d98dd09a6fdb8e7449a7ca3125fb1ddbd5b4e9", ), "imputation_producer_graph": ( - 24_488, - "6382a3ed07016414c31462b8bedb24590d92b792405a89c927d85fa1bd2a8347", + 24_498, + "92968ea779967b45e7ea5b60722a253239947806de05bf46437cf386b2ae6d23", ), "imputation_transfer_execution": ( 97, diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index b68874986d..93238e87bd 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -350,20 +350,21 @@ "acs_person_predictors": "878c788a6f037d7aca12b3586ea034eff04f3034ffa11935a736493042551f25", "authority": "9d4a9672a0f03039b1fe874b9fe21ed575be0d29f14afc396d03cdf5c809bdd2", "early_families": "4aa9f736fd76e83955477ad1667e58f48f264783f05bdc7f0102cd32d61323bd", - "full_checkpoint": "3db095ce2d1ef89028d3862f0edc827d704f099c5b3323f9b115642a3c27e007", + "full_checkpoint": "d7b84ec565db19067d984e0596ca1805110038c38bec6b5d02ac89fde7f261cf", "gap_fill_schedule": "1c31f9868f7884347cc19cf1ff65da43f950b9114941a715bab168246db414a7", - "graph_nodes": "40cd51ffdfe2e9d9d08d48c08e8ded9de1e4b134783bab05c4abc6ad5c72ca1e", + "graph_nodes": "e74fb1e6017cf77397708cbf2ad6e323dffd6be89425bc6544b5679e883cb674", "geography_assignment": "44e7f5f6cd7ceeef85d532c3d46306d03de4f4ee811376e6d29920742746b65c", "late_families": "d91f9ff0eb52f43e7b6eed3d5c58c37abe1620c3a11021da15dae9c10e16d382", - "late_resource_semantics": "cc34ba11288d01145fc2dc1a669df3f7c4ef2019d73afb02dd8eeee852b16d20", + "late_resource_semantics": "b2580f8725297ea535191416d37b77ccf891003b76db8e30d3c949e986b5cef6", "late_schedule": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "fdf23da429f8c501e198a2f5b719763d523a41565c2153fdf17b0344f21bcd7c", # #1178: puf_support's source bytes attest CPS Social Security preservation. # RNG rules and the ordered QRF targets are unchanged. microcosm#104 moves - # both again: the protocol attests microcosm.calibrate.solve. - "seed_map": "41a64b8820bf161e316c49e3f2b666a247b58f6f6d50dd4395b89c8f5b1399f6", - "seed_protocol": "0e4c8a4ace5e10747065669a009691d0eee9179ca42b344f8912c5ca239ebde6", + # both again: the protocol attests microcosm.calibrate.solve. The + # recipient-share bound in puf_support moves both again (#1259). + "seed_map": "2eaa714e90df4c8b261741de9b24028a7f1074780d5d177ef74619e2944db63b", + "seed_protocol": "f114fcb7f01afa54f3bf5ca65dc96342cdd69f5756774f1196ae32e9a71aad3d", "source_manifest": "f4f8bfeb79268a4043a843b47cec1d1c0e58e9f5dd6bc23417ada95446dd7b8e", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/imputation.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/imputation.yaml index 7676bdc8fe..445d7cf5b6 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/imputation.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/imputation.yaml @@ -4397,7 +4397,7 @@ producer_graph: resolution: static_exact binding: resource_kind: primary_puf_execution_config - schema_version: 5 + schema_version: 6 qrf: invocation_mode: predictors: canonical_default @@ -4405,6 +4405,18 @@ producer_graph: tax_unit_outputs: canonical_default tail_bound_quantiles: non_sch_d_capital_gains: 0.999 + recipient_share_bound: + rule: recipient_share_slice_mass + schema_version: 1 + scope: every_amount_output_of_the_invocation + excluded_output_classes: + - boolean_person_outputs + - discrete_tax_unit_outputs + recipient_share: household_design_weight_over_puf_recipient_total + cap: top_slice_positive_mass_per_slice_weight + floor: bottom_slice_negative_mass_per_slice_weight + snapped_outputs: bounded_on_snap_grid_values_cap_lowered_onto_grid + seam: raw_tax_unit_draws_after_tail_bound_quantiles worker_execution: surface: execution_profile resolve_as: worker_execution diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py index acbb94376e..2a2a85ed15 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py @@ -425,7 +425,7 @@ def puf_tax_detail_recipient_share_bound_identity() -> dict[str, object]: "recipient_share": "household_design_weight_over_puf_recipient_total", "cap": "top_slice_positive_mass_per_slice_weight", "floor": "bottom_slice_negative_mass_per_slice_weight", - "snapped_outputs": "cap_lowered_to_largest_snap_grid_value", + "snapped_outputs": "bounded_on_snap_grid_values_cap_lowered_onto_grid", "seam": "raw_tax_unit_draws_after_tail_bound_quantiles", } @@ -2083,22 +2083,24 @@ def finalize_us_puf_tax_detail_predictions( frame, recipient_tax_unit_ids, ) - resolved_share_bounds = { - output: _recipient_share_slice_bounds( - donor[output], donor["weight"], recipient_weights - ) - for output in share_bound_outputs - } snapped_outputs = ( _PUF_TAX_DETAIL_SPARSE_PERSON_OUTPUTS | _PUF_TAX_DETAIL_SPARSE_TAX_UNIT_OUTPUTS ) + resolved_share_bounds: dict[str, tuple[np.ndarray, np.ndarray]] = {} for output in share_bound_outputs: if output in snapped_outputs: - floor, cap = resolved_share_bounds[output] - resolved_share_bounds[output] = ( - floor, - _largest_grid_values_within(cap, _observed_snap_grid(donor[output])), + # A snapped output lives on its snap grid, so it is bounded on the + # donor's grid values: a slice inside one grid value caps at that + # value exactly, and the cap is then lowered onto the grid. + floor, cap = _recipient_share_slice_bounds( + _snap_grid_values(donor[output]), donor["weight"], recipient_weights ) + cap = _largest_grid_values_within(cap, _observed_snap_grid(donor[output])) + else: + floor, cap = _recipient_share_slice_bounds( + donor[output], donor["weight"], recipient_weights + ) + resolved_share_bounds[output] = (floor, cap) if resolved_tail_bounds: for output, (quantile, bound) in resolved_tail_bounds.items(): values = predictions[output].to_numpy(dtype=np.float64, copy=False) @@ -3793,11 +3795,17 @@ def _ensure_boolean_output_column(table: pd.DataFrame, column: str) -> None: ) +def _snap_grid_values(observed: Sequence[Any]) -> np.ndarray: + """Each donor value as :func:`_snap_to_observed_values` places it on its grid.""" + + observed_values = pd.to_numeric(pd.Series(observed), errors="coerce").fillna(0.0) + return np.rint(observed_values.to_numpy(dtype=np.float64)).clip(min=0.0) + + def _observed_snap_grid(observed: Sequence[Any]) -> np.ndarray: """The sorted donor values :func:`_snap_to_observed_values` snaps onto.""" - observed_values = pd.to_numeric(pd.Series(observed), errors="coerce").fillna(0.0) - return np.unique(np.rint(observed_values.to_numpy(dtype=np.float64)).clip(min=0.0)) + return np.unique(_snap_grid_values(observed)) def _snap_to_observed_values( diff --git a/packages/microcosm-build/tests/engine_contract/us/test_us_stacked_spine.py b/packages/microcosm-build/tests/engine_contract/us/test_us_stacked_spine.py index 632aaf756a..c525d799f6 100644 --- a/packages/microcosm-build/tests/engine_contract/us/test_us_stacked_spine.py +++ b/packages/microcosm-build/tests/engine_contract/us/test_us_stacked_spine.py @@ -36,6 +36,7 @@ def test_puf_finalize_masks_earnings_allocation_to_age_15_plus() -> None: person_outputs=tuple(predictions.columns), tax_unit_outputs=(), absent_cells=PUF_ABSENT_CELLS_PRESERVE_NULLS, + tail_bound_diagnostics=[], ) finalized_person = finalized.table("person") acs = finalized_person[support_channel_column("person")].eq("acs") diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py index 070bbc6d23..48ac736ec4 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py @@ -12,7 +12,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # Pin the domain separator, normalization rules, schema-set receipt, and # exact normative projection as one reviewable golden vector. assert first.spec_sha256 == ( - "0c0c8eea8f5d350f3a73867aebfcb82450cb606124dc85cc3772cc2eae971ddb" + "b635bb5527ac9e9fef469e4de9a7ff5478354f3ece7119a7b88be51124c50e23" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") diff --git a/packages/microcosm-build/tests/engine_free/us/test_puf_qrf_chain.py b/packages/microcosm-build/tests/engine_free/us/test_puf_qrf_chain.py index 924212e93a..a261779dd1 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_puf_qrf_chain.py +++ b/packages/microcosm-build/tests/engine_free/us/test_puf_qrf_chain.py @@ -328,7 +328,9 @@ def test_primary_qrf_legacy_object_bank_loads_without_rewrite_or_identity_failur assert live.table("tax_unit")[support_channel].dtype == CANONICAL_STRING_DTYPE for target_index in range(len((*_PERSON_OUTPUTS, *_TAX_UNIT_OUTPUTS))): run_primary_puf_qrf_target(checkpoint_dir, target_index) - _finalized, weight_kind = finalize_primary_puf_qrf_chain(live, checkpoint_dir) + _finalized, weight_kind = finalize_primary_puf_qrf_chain( + live, checkpoint_dir, tail_bound_diagnostics=[] + ) assert weight_kind == "design" assert { @@ -476,8 +478,11 @@ def test_target_subprocess_chain_matches_monolith_raw_bits_and_final_frame( monolith_raw.copy(), person_outputs=_PERSON_OUTPUTS, tax_unit_outputs=_TAX_UNIT_OUTPUTS, + tail_bound_diagnostics=[], + ) + actual, weight_kind = finalize_primary_puf_qrf_chain( + frame, checkpoint_dir, tail_bound_diagnostics=[] ) - actual, weight_kind = finalize_primary_puf_qrf_chain(frame, checkpoint_dir) assert weight_kind == "design" _assert_frames_bit_exact(actual, expected) @@ -487,7 +492,7 @@ def test_target_subprocess_chain_matches_monolith_raw_bits_and_final_frame( original_source_id = tax_unit.loc[first_puf_row, "tax_unit_source_id"] tax_unit.loc[first_puf_row, "tax_unit_source_id"] = original_source_id + 1 with pytest.raises(ValueError, match="changed PUF recipient identity"): - finalize_primary_puf_qrf_chain(frame, checkpoint_dir) + finalize_primary_puf_qrf_chain(frame, checkpoint_dir, tail_bound_diagnostics=[]) tax_unit.loc[first_puf_row, "tax_unit_source_id"] = original_source_id # Finalization must not rewrite raw target checkpoints. A second supervisor @@ -554,7 +559,9 @@ def test_resumed_earnings_chain_keeps_children_out_of_allocation( run_primary_puf_qrf_chain(checkpoint_dir) assert completed_target.read_bytes() == completed_bytes assert pending_target.is_file() - resumed, weight_kind = finalize_primary_puf_qrf_chain(frame, checkpoint_dir) + resumed, weight_kind = finalize_primary_puf_qrf_chain( + frame, checkpoint_dir, tail_bound_diagnostics=[] + ) assert weight_kind == "design" resumed_person = resumed.table("person") @@ -656,7 +663,9 @@ def test_primary_qrf_chain_stacked_finalization_preserves_unowned_nulls( for target_index in range(len((*_PERSON_OUTPUTS, *_TAX_UNIT_OUTPUTS))): run_primary_puf_qrf_target(checkpoint_dir, target_index) - finalized, weight_kind = finalize_primary_puf_qrf_chain(frame, checkpoint_dir) + finalized, weight_kind = finalize_primary_puf_qrf_chain( + frame, checkpoint_dir, tail_bound_diagnostics=[] + ) assert weight_kind == "design" person = finalized.table("person") @@ -703,7 +712,9 @@ def test_primary_qrf_chain_legacy_doctrine_defaults_remain_loadable( for target_index in range(len((*_PERSON_OUTPUTS, *_TAX_UNIT_OUTPUTS))): run_primary_puf_qrf_target(checkpoint_dir, target_index) - finalized, _weight_kind = finalize_primary_puf_qrf_chain(frame, checkpoint_dir) + finalized, _weight_kind = finalize_primary_puf_qrf_chain( + frame, checkpoint_dir, tail_bound_diagnostics=[] + ) person = finalized.table("person") native = person["person_support_clone_index"].eq(0) @@ -768,7 +779,7 @@ def test_primary_qrf_finalization_rejects_changed_recipient_features( ValueError, match="changed the PUF recipient predictor source universe or feature values", ): - finalize_primary_puf_qrf_chain(frame, checkpoint_dir) + finalize_primary_puf_qrf_chain(frame, checkpoint_dir, tail_bound_diagnostics=[]) @pytest.mark.parametrize("stale_version", (1, 2, 3, 4, 5)) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_education_inputs.py b/packages/microcosm-build/tests/engine_free/us/test_us_education_inputs.py index 7a6124b034..aff3eb4008 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_education_inputs.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_education_inputs.py @@ -401,6 +401,7 @@ def test_puf_support_to_education_stage_preserves_real_source_signal( tax_unit_outputs=(), seed=0, n_estimators=20, + tail_bound_diagnostics=[], ) person = imputed.table("person") channel = person[support_channel_column("person")] diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_form_4952.py b/packages/microcosm-build/tests/engine_free/us/test_us_form_4952.py index 18818f190c..21eb193dfc 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_form_4952.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_form_4952.py @@ -141,6 +141,7 @@ def test_puf_imputation_preserves_tax_unit_total_without_inventing_split() -> No tax_unit_outputs=(), n_estimators=4, seed=0, + tail_bound_diagnostics=[], ) person = imputed.table("person") channel = support_channel_column("person") diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_puf_income_rank_predictors.py b/packages/microcosm-build/tests/engine_free/us/test_us_puf_income_rank_predictors.py index 6b5d5a98b7..27ee44e2de 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_puf_income_rank_predictors.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_puf_income_rank_predictors.py @@ -272,6 +272,7 @@ def test_donor_without_person_roles_leaves_demographic_predictors_absent() -> No tax_unit_outputs=(), n_estimators=2, seed=0, + tail_bound_diagnostics=[], ) @@ -444,6 +445,7 @@ def test_strict_recipient_surface_names_a_tax_unit_without_a_head() -> None: seed=0, require_complete_recipient_predictors=True, absent_cells=puf_support.PUF_ABSENT_CELLS_PRESERVE_NULLS, + tail_bound_diagnostics=[], ) @@ -487,6 +489,7 @@ def _imputed_puf_wages( tax_unit_outputs=(), n_estimators=8, seed=0, + tail_bound_diagnostics=[], ) person = imputed.table("person") detail = person[support_channel_column("person")].eq(PUF_TAX_DETAIL_SUPPORT_CHANNEL) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_puf_recipient_share_bound.py b/packages/microcosm-build/tests/engine_free/us/test_us_puf_recipient_share_bound.py index 9b0600730a..010ccbeaf6 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_puf_recipient_share_bound.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_puf_recipient_share_bound.py @@ -609,9 +609,13 @@ def test_snapped_outputs_stay_on_observed_values_within_the_bound() -> None: finalized = _finalize(frame, donor, {_SNAPPED: raw}, diagnostics=[]) - _, cap = _bounds(donor[_SNAPPED], donor["weight"], _recipient_weights(frame)) - actual = _puf_values(finalized, _SNAPPED) grid = puf_support_module._observed_snap_grid(donor[_SNAPPED]) + _, cap = _bounds( + puf_support_module._snap_grid_values(donor[_SNAPPED]), + donor["weight"], + _recipient_weights(frame), + ) + actual = _puf_values(finalized, _SNAPPED) assert (actual <= cap).all() assert np.isin(actual, grid).all() assert actual[1] < raw[1] @@ -620,6 +624,26 @@ def test_snapped_outputs_stay_on_observed_values_within_the_bound() -> None: assert actual[1] == grid[grid <= cap[1]].max() +def test_draws_at_a_tied_maximum_on_the_snap_grid_are_not_clipped() -> None: + # Statutory caps aged by a factor leave the donor's maximum off the integer + # grid (4,504.8 here) yet tied across many returns. The bound is computed + # on the grid the column snaps to, so draws at that maximum snap to it + # unmoved instead of being clipped to the grid value below. + frame = _recipient_frame([20.0, 400.0, 600.0, 980.0]) + donor = pd.DataFrame( + {_SNAPPED: [4_504.8, 1_000.4, 0.0], "weight": [500.0, 400.0, 100.0]} + ) + raw = np.asarray([4_504.8, 4_504.8, 4_504.8, 0.0]) + diagnostics: list[dict[str, object]] = [] + + finalized = _finalize(frame, donor, {_SNAPPED: raw}, diagnostics=diagnostics) + + assert diagnostics[0]["clipped_row_count"] == 0 + np.testing.assert_array_equal( + _puf_values(finalized, _SNAPPED), [4_505.0, 4_505.0, 4_505.0, 0.0] + ) + + def test_sparsification_rescales_bounded_draws_by_one_common_factor() -> None: # Every recipient draws positive while the donor is 90.9% positive, so the # donor-rate sparsification zeroes the smallest draw and rescales the rest @@ -632,7 +656,9 @@ def test_sparsification_rescales_bounded_draws_by_one_common_factor() -> None: finalized = _finalize(frame, donor, {_SNAPPED: raw}, diagnostics=[]) weights = _recipient_weights(frame) - floor, cap = _bounds(donor[_SNAPPED], donor["weight"], weights) + floor, cap = _bounds( + puf_support_module._snap_grid_values(donor[_SNAPPED]), donor["weight"], weights + ) grid = puf_support_module._observed_snap_grid(donor[_SNAPPED]) lowered = puf_support_module._largest_grid_values_within(cap, grid) before = puf_support_module._snap_to_observed_values( diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_puf_social_security.py b/packages/microcosm-build/tests/engine_free/us/test_us_puf_social_security.py index fdb0b9aa1a..d4d2e7ddc5 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_puf_social_security.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_puf_social_security.py @@ -60,6 +60,7 @@ def _finalize( person_outputs=components, tax_unit_outputs=(), absent_cells=absent_cells, + tail_bound_diagnostics=[], ) @@ -218,6 +219,7 @@ def predict( n_estimators=1, seed=0, raw_predictions_callback=lambda draw: raw_draws.append(draw.copy(deep=True)), + tail_bound_diagnostics=[], ) assert len(raw_draws) == 1 diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_puf_support.py b/packages/microcosm-build/tests/engine_free/us/test_us_puf_support.py index 72ff0c8e72..205e2e43a6 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_puf_support.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_puf_support.py @@ -708,6 +708,7 @@ def test_puf_tax_detail_refuses_formula_owned_outputs() -> None: tax_unit_outputs=("interest_deduction",), n_estimators=4, seed=0, + tail_bound_diagnostics=[], ) @@ -1101,6 +1102,7 @@ def test_cps_carried_derivations_unblock_default_puf_predictors() -> None: tax_unit_outputs=(), n_estimators=4, seed=0, + tail_bound_diagnostics=[], ) puf_people = imputed.table("person")[ @@ -1171,6 +1173,7 @@ def predict( tax_unit_outputs=(), n_estimators=4, seed=0, + tail_bound_diagnostics=[], ) puf_people = imputed.table("person")[ @@ -1241,6 +1244,7 @@ def predict( tax_unit_outputs=(), n_estimators=4, seed=0, + tail_bound_diagnostics=[], ) person = imputed.table("person") @@ -1444,6 +1448,7 @@ def test_puf_tax_detail_imputation_writes_only_puf_channel() -> None: tax_unit_outputs=(), n_estimators=4, seed=0, + tail_bound_diagnostics=[], ) person = imputed.table("person") @@ -1505,6 +1510,7 @@ def test_puf_tax_detail_imputation_distributes_contributions_by_earnings() -> No tax_unit_outputs=(), n_estimators=4, seed=0, + tail_bound_diagnostics=[], ) person = imputed.table("person") @@ -1601,6 +1607,7 @@ def predict( tax_unit_outputs=(), n_estimators=4, seed=0, + tail_bound_diagnostics=[], ) person = imputed.table("person") @@ -1679,6 +1686,7 @@ def predict( ), n_estimators=4, seed=0, + tail_bound_diagnostics=[], ) puf_tax_units = imputed.table("tax_unit")[ @@ -1769,6 +1777,7 @@ def predict( tax_unit_outputs=(), n_estimators=4, seed=0, + tail_bound_diagnostics=[], ) final_pos, final_neg = _puf_channel_person_legs(imputed, column) @@ -1849,6 +1858,7 @@ def _features(rng: np.random.Generator, n: int) -> tuple[np.ndarray, np.ndarray] raw_predictions_callback=lambda predictions: raw_holder.__setitem__( "raw", predictions[column].to_numpy().copy() ), + tail_bound_diagnostics=[], ) # Recover the recipient tax-unit weights to score the raw draw's national net. diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_salt_refund_income.py b/packages/microcosm-build/tests/engine_free/us/test_us_salt_refund_income.py index b4d2332ef0..4a525fcddb 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_salt_refund_income.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_salt_refund_income.py @@ -128,6 +128,7 @@ def test_puf_imputation_preserves_tax_unit_total_without_inventing_split() -> No tax_unit_outputs=(), n_estimators=4, seed=0, + tail_bound_diagnostics=[], ) person = imputed.table("person") channel = support_channel_column("person") diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_stacked_spine.py b/packages/microcosm-build/tests/engine_free/us/test_us_stacked_spine.py index c1b3fb6553..5c5ecdde8c 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_stacked_spine.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_stacked_spine.py @@ -553,6 +553,7 @@ def test_finalize_preserve_nulls_keeps_unowned_cells_null() -> None: person_outputs=("taxable_interest_income",), tax_unit_outputs=("health_savings_account_ald",), absent_cells=PUF_ABSENT_CELLS_PRESERVE_NULLS, + tail_bound_diagnostics=[], ) person = finalized.table("person") @@ -609,6 +610,7 @@ def test_finalize_preserve_nulls_materializes_registry_boolean_outputs() -> None person_outputs=boolean_outputs, tax_unit_outputs=(), absent_cells=PUF_ABSENT_CELLS_PRESERVE_NULLS, + tail_bound_diagnostics=[], ) person = finalized.table("person") @@ -649,6 +651,7 @@ def test_finalize_preserve_nulls_rejects_numeric_boolean_materialization() -> No person_outputs=(column,), tax_unit_outputs=(), absent_cells=PUF_ABSENT_CELLS_PRESERVE_NULLS, + tail_bound_diagnostics=[], ) @@ -664,6 +667,7 @@ def test_finalize_legacy_zero_fill_reproduces_the_audited_defect() -> None: predictions.copy(), person_outputs=("taxable_interest_income",), tax_unit_outputs=("health_savings_account_ald",), + tail_bound_diagnostics=[], ) person = finalized.table("person") @@ -689,6 +693,7 @@ def test_finalize_legacy_zero_fill_matches_run_7_protocol_5_byte_pin() -> None: predictions.copy(), person_outputs=("taxable_interest_income",), tax_unit_outputs=("health_savings_account_ald",), + tail_bound_diagnostics=[], ) actual = ( @@ -717,6 +722,7 @@ def test_preserve_nulls_sparsification_never_rewrites_native_rows() -> None: person_outputs=("taxable_interest_income",), tax_unit_outputs=("health_savings_account_ald",), absent_cells=PUF_ABSENT_CELLS_PRESERVE_NULLS, + tail_bound_diagnostics=[], ) person = finalized.table("person") diff --git a/packages/microcosm-build/tests/engine_scenario/us/test_us_educator_expenses.py b/packages/microcosm-build/tests/engine_scenario/us/test_us_educator_expenses.py index 3c9576a99d..c79a27d1fc 100644 --- a/packages/microcosm-build/tests/engine_scenario/us/test_us_educator_expenses.py +++ b/packages/microcosm-build/tests/engine_scenario/us/test_us_educator_expenses.py @@ -79,6 +79,7 @@ def predict( tax_unit_outputs=(), n_estimators=4, seed=9, + tail_bound_diagnostics=[], ) person = imputed.table("person") diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py index 799d806d36..dfbb127429 100644 --- a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py @@ -338,7 +338,7 @@ def capture_equality(expected: object, actual: object) -> None: "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "3080fe938df62216dc4853c1163639817de3c537e0e932309642fbd3f7d8f266", + "spec_sha256": "f56dd06c602f065546c659c353114d2c9a4f6a7cd7e13daadfec14f48b53152a", }, } diff --git a/test_support/microcosm_build/us_farm_business_income.py b/test_support/microcosm_build/us_farm_business_income.py index d835aca228..a73315a8e8 100644 --- a/test_support/microcosm_build/us_farm_business_income.py +++ b/test_support/microcosm_build/us_farm_business_income.py @@ -133,6 +133,7 @@ def fit( person_outputs=US_FARM_BUSINESS_INCOME_OUTPUT_COLUMNS, tax_unit_outputs=(), seed=11, + tail_bound_diagnostics=[], ) diff --git a/test_support/microcosm_build/us_puf_support.py b/test_support/microcosm_build/us_puf_support.py index 502744f568..b8bbeb5943 100644 --- a/test_support/microcosm_build/us_puf_support.py +++ b/test_support/microcosm_build/us_puf_support.py @@ -276,6 +276,7 @@ def _impute(self, fit_records): n_estimators=4, seed=0, fit_records=fit_records, + tail_bound_diagnostics=[], ) def test_production_fit_records_design_weight_kind(self) -> None: @@ -326,6 +327,7 @@ def test_records_are_only_emitted_when_a_sink_is_provided(self) -> None: tax_unit_outputs=(), n_estimators=4, seed=0, + tail_bound_diagnostics=[], ) assert imputed.table("person") is not None @@ -367,6 +369,7 @@ def test_production_fit_records_design_kind_under_installed_metadata(self) -> No n_estimators=4, seed=0, fit_records=fit_records, + tail_bound_diagnostics=[], ) assert fit_records == [FitWeightRecord(US_PUF_SUPPORT_FIT_NAME, "design")] diff --git a/tools/spec_engine_coverage.py b/tools/spec_engine_coverage.py index c066d573ce..0322601492 100644 --- a/tools/spec_engine_coverage.py +++ b/tools/spec_engine_coverage.py @@ -41,7 +41,7 @@ REPORT_SCHEMA_VERSION = 3 EXPECTED_POINTER_INVENTORY_SHA256 = ( - "8f655670b9794adae905ede64bdceba6395eeb8d642b6c5896d5c516f1f6a6a7" + "4523cb2c2ccca11f484b6fdd1efb860652b1fb1622a4e39675ba652b9136f4f6" ) DEFAULT_REPORT_PATH = ( Path(__file__).resolve().parents[1] From 37be49066e32d5528c53ebcf2e8e8c1b2e8a847d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 13 Oct 2026 09:22:11 -0400 Subject: [PATCH 3/8] Give the base-builder weights-audit tests a tail-bound sink Co-Authored-By: Claude Opus 5.5 --- .../tests/engine_free/us/test_us_puf_support_base_builder.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_puf_support_base_builder.py b/packages/microcosm-build/tests/engine_free/us/test_us_puf_support_base_builder.py index e6944adaae..cf41dc000f 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_puf_support_base_builder.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_puf_support_base_builder.py @@ -423,6 +423,7 @@ def test_base_build_records_design_weight_kind_in_the_summary(self) -> None: _imputed, weights_audit = builder.impute_and_audit_us_puf_support( clone_us_frame_for_puf_support(_minimal_us_puf_support_frame()), _support_donor(), + tail_bound_diagnostics=[], **_SUPPORT_FIT_KWARGS, ) @@ -439,6 +440,7 @@ def test_base_build_summary_json_carries_the_audit(self) -> None: _imputed, weights_audit = builder.impute_and_audit_us_puf_support( clone_us_frame_for_puf_support(_minimal_us_puf_support_frame()), _support_donor(), + tail_bound_diagnostics=[], **_SUPPORT_FIT_KWARGS, ) round_tripped = json.loads(json.dumps({"weights_audit": weights_audit})) From 8e54ef72b284ea44706e7add21348f5a5eaed4d5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 13 Oct 2026 09:56:29 -0400 Subject: [PATCH 4/8] Address review r1: shared config schema, interim label, edge cases, tests - Stacked publication and the pool loader required execution-config schema 5 while the builder wrote 6, so no current stacked pool could publish or load. All four sites now read PRIMARY_PUF_EXECUTION_CONFIG_SCHEMA_VERSION. - The comment and docstrings now call the bound an interim censor that limits a record to its slice (not a fix for concentration the slice has), name the root fixes (#958/#964, #481), and record the loosened links across columns, the rung dependence and the Social Security re-split. - Slices use positive-weight donors only; non-finite bounded draws are refused before any write; quantile records carry a rule discriminator. - Tests: zero weights on both sides; an LP optimality check replaces the tautological mass test; production-geometry differential test; tax-unit outputs, Social Security total, and per-leg calibration factors through finalization; the identity is pinned to a literal. - Re-pinned the seed protocol and map, coverage evidence, pool-tool spec sha and loader golden. Co-Authored-By: Claude Opus 5.5 --- docs/evidence/spec-engine/us-f0-coverage.json | 16 +- .../build/spec_engine/inventory_coverage.py | 4 +- .../src/microcosm/build/us_runtime/h5_io.py | 8 +- .../microcosm/build/us_runtime/puf_support.py | 47 ++- .../build/us_runtime/stacked_spine.py | 10 +- .../shared/test_spec_engine_loader.py | 2 +- .../us/test_us_puf_recipient_share_bound.py | 287 +++++++++++++++--- .../engine_free/us/test_us_puf_tail_bound.py | 1 + .../us/test_us_multispine_pool_tool.py | 2 +- tools/build_us_multispine_pool.py | 5 +- 10 files changed, 310 insertions(+), 72 deletions(-) diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index 8649930da3..ebe6027ee5 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -1656,13 +1656,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "2eaa714e90df4c8b261741de9b24028a7f1074780d5d177ef74619e2944db63b", - "protocol_sha256": "f114fcb7f01afa54f3bf5ca65dc96342cdd69f5756774f1196ae32e9a71aad3d" + "map_sha256": "1d7cc2ee599758e256d830640ecf01ce49fe35e4970c07cb6cdcfb009f9e2ee4", + "protocol_sha256": "d2d2d7e4994f8970d4803c1e5506d2d8af2a4a606db2a04860ff7db498a7ffd1" }, "failures": [], "observed": { - "map_sha256": "2eaa714e90df4c8b261741de9b24028a7f1074780d5d177ef74619e2944db63b", - "protocol_sha256": "f114fcb7f01afa54f3bf5ca65dc96342cdd69f5756774f1196ae32e9a71aad3d" + "map_sha256": "1d7cc2ee599758e256d830640ecf01ce49fe35e4970c07cb6cdcfb009f9e2ee4", + "protocol_sha256": "d2d2d7e4994f8970d4803c1e5506d2d8af2a4a606db2a04860ff7db498a7ffd1" }, "status": "covered" }, @@ -1677,7 +1677,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "f114fcb7f01afa54f3bf5ca65dc96342cdd69f5756774f1196ae32e9a71aad3d", + "implementation_sha256": "d2d2d7e4994f8970d4803c1e5506d2d8af2a4a606db2a04860ff7db498a7ffd1", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1698,7 +1698,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "f114fcb7f01afa54f3bf5ca65dc96342cdd69f5756774f1196ae32e9a71aad3d", + "implementation_sha256": "d2d2d7e4994f8970d4803c1e5506d2d8af2a4a606db2a04860ff7db498a7ffd1", "protocol": "legacy-v1", "streams": [ "build_model", @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "f56dd06c602f065546c659c353114d2c9a4f6a7cd7e13daadfec14f48b53152a" + "spec_sha256": "c1ac73b48d1468d91812dc49343db45a257fb9f68a176c8203295cc5967f7bff" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "f56dd06c602f065546c659c353114d2c9a4f6a7cd7e13daadfec14f48b53152a" + "spec_sha256": "c1ac73b48d1468d91812dc49343db45a257fb9f68a176c8203295cc5967f7bff" }, "status": "pass" } diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 93238e87bd..2eccf1a70d 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -363,8 +363,8 @@ # RNG rules and the ordered QRF targets are unchanged. microcosm#104 moves # both again: the protocol attests microcosm.calibrate.solve. The # recipient-share bound in puf_support moves both again (#1259). - "seed_map": "2eaa714e90df4c8b261741de9b24028a7f1074780d5d177ef74619e2944db63b", - "seed_protocol": "f114fcb7f01afa54f3bf5ca65dc96342cdd69f5756774f1196ae32e9a71aad3d", + "seed_map": "1d7cc2ee599758e256d830640ecf01ce49fe35e4970c07cb6cdcfb009f9e2ee4", + "seed_protocol": "d2d2d7e4994f8970d4803c1e5506d2d8af2a4a606db2a04860ff7db498a7ffd1", "source_manifest": "f4f8bfeb79268a4043a843b47cec1d1c0e58e9f5dd6bc23417ada95446dd7b8e", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py b/packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py index a22eb4835f..ea03a02978 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/h5_io.py @@ -1050,10 +1050,14 @@ def _current_worker_execution_authentication( ) -> dict[str, object]: """Derive current authentication evidence from the signed nested worker.""" + from microcosm.build.us_runtime.stacked_spine import ( + PRIMARY_PUF_EXECUTION_CONFIG_SCHEMA_VERSION, + ) + config, worker = _stacked_primary_execution_config( manifest, manifest_path=manifest_path ) - if config.get("schema_version") != 5: + if config.get("schema_version") != PRIMARY_PUF_EXECUTION_CONFIG_SCHEMA_VERSION: raise ValueError( f"US stacked pool manifest {manifest_path} primary execution-config " "version changed." @@ -1061,7 +1065,7 @@ def _current_worker_execution_authentication( return worker_identity_runtime.current_worker_execution_authentication_receipt( worker, manifest_schema_version=US_MULTISPINE_POOL_MANIFEST_SCHEMA_VERSION, - execution_config_schema_version=5, + execution_config_schema_version=PRIMARY_PUF_EXECUTION_CONFIG_SCHEMA_VERSION, boundary=f"US stacked pool manifest {manifest_path} primary-QRF worker", ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py index 2a2a85ed15..b97bda4128 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py @@ -403,10 +403,18 @@ def puf_tax_detail_tail_bound_quantiles_identity() -> dict[str, float]: # carries it. The bound caps each raw amount draw at the positive mass, per # unit of slice weight, of the donor's top slice of the recipient's own share # (and floors losses from the bottom slice), so no record carries more mass -# than the most extreme slice of the population it could stand for. Tail -# support below pool resolution is the pool's job (DESIGN.md); this keeps the -# fit from delivering it. Discrete and boolean outputs are not amounts and -# are left alone. +# than the most extreme slice of the population it could stand for. It limits +# a record to its slice; it does not remove concentration the slice itself +# has, and the mass it clips from passthrough columns is dropped. It is an +# interim censor at the finalize seam, like the quantile map above: tail +# support below pool resolution is the pool's job (DESIGN.md), and the root +# fix is a tail stratum carrying PUF donors at their own weights (#958, #964), +# with weight-aware draws (#481) as a complement. Each column is bounded on +# its own, so a clipped record's links to other columns (mortgage balance and +# interest, gains and their subsets) loosen further. Shares depend on the +# recipient population, so a smaller sample rung, whose records stand for +# more of it, is bounded more tightly. Discrete and boolean outputs are not +# amounts and are left alone. PUF_TAX_DETAIL_RECIPIENT_SHARE_BOUND_RULE = "recipient_share_slice_mass" _PUF_TAX_DETAIL_RECIPIENT_SHARE_BOUND_SCHEMA_VERSION = 1 @@ -1979,11 +1987,13 @@ def finalize_us_puf_tax_detail_predictions( explicit ``recipient_share_bound_outputs`` narrows it for tests and must name amount outputs of this invocation. It writes one diagnostics record per bounded output, including outputs it leaves unchanged. It holds on the - raw tax-unit draws and, for snapped outputs, after snapping. Two later - steps rescale a whole column to the donor instrument and so lift every - kept value by one common factor: the donor-rate sparsification of sparse - outputs restores the column's mass after zeroing carriers, and the - signed-mass calibration scales each leg of its outputs. + raw tax-unit draws and, for snapped outputs, after snapping. Three later + steps move bounded values without adding mass. The donor-rate + sparsification of sparse outputs restores the column's mass after zeroing + carriers, and the signed-mass calibration scales each leg of its outputs; + both lift every kept value of the column or leg by one common factor. The + Social Security reconciliation re-splits the four components' bounded + total by the CPS shares, keeping the total. ``absent_cells`` selects the finalization policy for cells outside the PUF clone arm (microcosm#578 audit item 1). The legacy default reproduces the @@ -2088,6 +2098,10 @@ def finalize_us_puf_tax_detail_predictions( ) resolved_share_bounds: dict[str, tuple[np.ndarray, np.ndarray]] = {} for output in share_bound_outputs: + if not np.isfinite(predictions[output].to_numpy(dtype=np.float64)).all(): + raise ValueError( + f"PUF raw predictions for bounded output {output!r} must be finite." + ) if output in snapped_outputs: # A snapped output lives on its snap grid, so it is bounded on the # donor's grid values: a slice inside one grid value caps at that @@ -2118,6 +2132,7 @@ def finalize_us_puf_tax_detail_predictions( tail_bound_diagnostics.append( { "output": output, + "rule": "weighted_positive_donor_quantile", "quantile": quantile, "bound_value": bound, "clipped_row_count": int(clipped.sum()), @@ -2846,9 +2861,11 @@ def _recipient_share_slice_bounds( share ``s`` of donor weight at the top of the column, so a positive value is capped at that mass per unit of slice weight; losses are floored the same way from the bottom slice. Within the positive donors the cap is the - slice's weighted mean, and it falls as the share grows. A side is infinite - only when the donor has no value of that sign. Shares make the bound - independent of either population's weight scale. + slice's weighted mean, and it falls as the share grows. Only donors with + positive weight count. A side is infinite only when no such donor has a + value of that sign, and a zero-weight recipient's cap is the largest + weighted donor value, which binds nothing a weighted draw can reach. + Shares make the bound independent of either population's weight scale. """ values = pd.to_numeric(pd.Series(donor_values), errors="coerce").to_numpy( @@ -2878,6 +2895,12 @@ def _recipient_share_slice_bounds( if donor_total <= 0.0 or recipient_total <= 0.0: raise ValueError("Recipient-share bound weights must have positive mass.") slice_weights = recipients / recipient_total * donor_total + # Only donors with weight stand for anyone, and a zero-weight donor cannot + # be drawn (the forests resample by weight), so slices and the largest + # value come from the weighted donors alone. + weighted = weights > 0.0 + values = values[weighted] + weights = weights[weighted] # Ordering by the positive part orders the top slice exactly as the values # do; donors at or below zero tie at zero and add no positive mass. top = _top_slice_means(np.maximum(values, 0.0), weights, slice_weights) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py b/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py index 340700bff6..a907cb1622 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py @@ -5395,6 +5395,12 @@ def _late_virtual_resource_kind(column: str) -> str: ) from exc +#: The primary PUF execution config's payload schema. Schema 6 adds the +#: recipient-share bound (#1259). The builder, the late-resource validator, +#: stacked publication and the pool loader all read this one value. +PRIMARY_PUF_EXECUTION_CONFIG_SCHEMA_VERSION = 6 + + def _late_resource_binding_schema_version( column: str, *, @@ -5407,7 +5413,7 @@ def _late_resource_binding_schema_version( kind = _late_virtual_resource_kind(column) versions = { "acs_pums_earnings_universe_execution_config": 2, - "primary_puf_execution_config": 6, + "primary_puf_execution_config": PRIMARY_PUF_EXECUTION_CONFIG_SCHEMA_VERSION, "post_clone_source_execution_config": 3, "source_finalizer_execution_config": 2, "late_transfer_model_config": 3, @@ -6122,7 +6128,7 @@ def _late_primary_execution_config_binding( ) return { "resource_kind": "primary_puf_execution_config", - "schema_version": 6, + "schema_version": PRIMARY_PUF_EXECUTION_CONFIG_SCHEMA_VERSION, "clone_attachment": { "fraction": float(clone_attachment_fraction), "seed": clone_attachment_seed, diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py index 48ac736ec4..0031c67a99 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py @@ -12,7 +12,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # Pin the domain separator, normalization rules, schema-set receipt, and # exact normative projection as one reviewable golden vector. assert first.spec_sha256 == ( - "b635bb5527ac9e9fef469e4de9a7ff5478354f3ece7119a7b88be51124c50e23" + "7d1a8e72fd13abb09596f88438b5d56e9d667178b37db9d1b10e086675ecd8ef" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_puf_recipient_share_bound.py b/packages/microcosm-build/tests/engine_free/us/test_us_puf_recipient_share_bound.py index 010ccbeaf6..62801f97a5 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_puf_recipient_share_bound.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_puf_recipient_share_bound.py @@ -2,8 +2,9 @@ A PUF-clone recipient whose weight is the share ``s`` of the recipient population stands for ``s`` of the population. The bound limits its raw draw -to the weighted mean of the donor slice holding the same share at that end of -the distribution, so ``w_r * value <= (W_r / W_d) * mass(slice)``. +to the same-sign mass, per unit of slice weight, of the donor slice holding the +same share at that end of the distribution, so +``w_r * value <= (W_r / W_d) * mass(slice)``. """ from __future__ import annotations @@ -31,18 +32,28 @@ _positive_weights = st.floats( min_value=1e-3, max_value=1e4, allow_nan=False, allow_infinity=False ) +# Zero weights occur on both sides: the donor fill is fillna(0.0), and a +# recipient household can carry no weight. +_weights = st.one_of(st.just(0.0), _positive_weights) + + +def _with_positive_mass(weights: list[float]) -> np.ndarray: + array = np.asarray(weights, dtype=np.float64) + if not (array > 0.0).any(): + array[0] = 1.0 + return array @st.composite def _donor_and_recipients(draw: st.DrawFn) -> tuple[np.ndarray, ...]: donors = draw(st.integers(min_value=1, max_value=40)) values = np.asarray(draw(st.lists(_amounts, min_size=donors, max_size=donors))) - weights = np.asarray( - draw(st.lists(_positive_weights, min_size=donors, max_size=donors)) + weights = _with_positive_mass( + draw(st.lists(_weights, min_size=donors, max_size=donors)) ) recipients = draw(st.integers(min_value=1, max_value=25)) - recipient_weights = np.asarray( - draw(st.lists(_positive_weights, min_size=recipients, max_size=recipients)) + recipient_weights = _with_positive_mass( + draw(st.lists(_weights, min_size=recipients, max_size=recipients)) ) return values, weights, recipient_weights @@ -52,7 +63,12 @@ def _reference_slice_mean( ) -> float: """One slice's same-sign mass per unit weight, donor by donor.""" - order = sorted(range(len(values)), key=lambda i: -values[i] if top else values[i]) + weighted = [i for i in range(len(values)) if weights[i] > 0.0] + order = sorted(weighted, key=lambda i: -values[i] if top else values[i]) + if take <= 0.0: + # An empty slice reads the most extreme weighted value on its side. + extreme = values[order[0]] + return max(extreme, 0.0) if top else min(extreme, 0.0) remaining = take mass = 0.0 for index in order: @@ -91,6 +107,8 @@ def _close(actual: np.ndarray, expected: np.ndarray) -> None: def _top_slice_edge(values: np.ndarray, weights: np.ndarray, take: float) -> float: """The value at the inner edge of the top slice holding ``take`` weight.""" + weighted = weights > 0.0 + values, weights = values[weighted], weights[weighted] order = np.argsort(-values, kind="stable") cumulative = np.cumsum(weights[order]) edge = min(int(np.searchsorted(cumulative, take, side="left")), len(values) - 1) @@ -98,7 +116,8 @@ def _top_slice_edge(values: np.ndarray, weights: np.ndarray, take: float) -> flo def test_bound_is_the_slice_mean_on_a_worked_example() -> None: - # Donor population of weight 1,000: 990 at 10, 9 at 1,000, 1 at 100,000. + # Donor population of weight 1,000: 980 at 10, 9 at 1,000, 1 at 100,000 + # and 10 at -50. values = np.asarray([10.0, 1_000.0, 100_000.0, -50.0]) weights = np.asarray([980.0, 9.0, 1.0, 10.0]) # Recipients of share 0.001, 0.01 and 0.989 of their own population. @@ -201,11 +220,17 @@ def test_vectorized_bound_matches_the_donor_by_donor_reference( _close(floor, expected_floor) -@_PROPERTY_SETTINGS +@settings(max_examples=150, deadline=None, suppress_health_check=[HealthCheck.too_slow]) @given(_donor_and_recipients()) -def test_one_record_never_carries_more_than_its_slice_mass( +def test_cap_mass_is_the_most_any_donor_subpopulation_of_that_size_holds( case: tuple[np.ndarray, ...], ) -> None: + # Independent of the slice construction: a linear program picks any + # fractional subpopulation of donor weight `take` (0 <= x_d <= w_d) to + # maximize its positive mass (or minimize its negative mass). The bound's + # mass, w_r * cap scaled to donor weight, must equal that optimum. + from scipy.optimize import linprog + values, weights, recipient_weights = case donor_total = weights.sum() recipient_total = recipient_weights.sum() @@ -214,13 +239,25 @@ def test_one_record_never_carries_more_than_its_slice_mass( for index, weight in enumerate(recipient_weights): take = weight / recipient_total * donor_total - scale = recipient_total / donor_total - if np.isfinite(cap[index]): - top_mass = take * _reference_slice_mean(values, weights, take, top=True) - assert weight * cap[index] == pytest.approx(scale * top_mass, rel=1e-9) - if np.isfinite(floor[index]): - bottom_mass = take * _reference_slice_mean(values, weights, take, top=False) - assert weight * floor[index] == pytest.approx(scale * bottom_mass, rel=1e-9) + if take <= 0.0: + continue + for side, bound in ((1.0, cap[index]), (-1.0, floor[index])): + part = np.maximum(side * values, 0.0) + if not (part[weights > 0.0] > 0.0).any(): + assert np.isinf(bound) + continue + result = linprog( + -part, + A_eq=np.ones((1, len(values))), + b_eq=[take], + bounds=[(0.0, w) for w in weights], + method="highs", + ) + assert result.success + best_mass = -result.fun + assert side * bound * take == pytest.approx( + best_mass, rel=1e-7, abs=1e-6 * max(1.0, best_mass) + ) @_PROPERTY_SETTINGS @@ -236,13 +273,14 @@ def test_bound_stays_inside_the_donor_support_and_slice( for index, weight in enumerate(recipient_weights): take = weight / recipient_total * donor_total + weighted_values = values[weights > 0.0] if np.isfinite(cap[index]): - assert 0.0 < cap[index] <= values.max() + assert 0.0 < cap[index] <= weighted_values.max() # The mean of the slice is at least its inner edge, exactly. edge = _top_slice_edge(values, weights, take) assert cap[index] >= edge if np.isfinite(floor[index]): - assert values.min() <= floor[index] < 0.0 + assert weighted_values.min() <= floor[index] < 0.0 edge = -_top_slice_edge(-values, weights, take) assert floor[index] <= edge @@ -262,10 +300,11 @@ def test_heavier_recipients_get_tighter_bounds(case: tuple[np.ndarray, ...]) -> assert (np.diff(ordered_floor[np.isfinite(ordered_floor)]) >= -tolerance).all() # A side applies to every recipient or to none: it is infinite only when # the donor has no value of that sign. - assert np.isfinite(cap).all() == bool((values > 0.0).any()) - assert np.isinf(cap).all() == (not (values > 0.0).any()) - assert np.isfinite(floor).all() == bool((values < 0.0).any()) - assert np.isinf(floor).all() == (not (values < 0.0).any()) + weighted_values = values[weights > 0.0] + assert np.isfinite(cap).all() == bool((weighted_values > 0.0).any()) + assert np.isinf(cap).all() == (not (weighted_values > 0.0).any()) + assert np.isfinite(floor).all() == bool((weighted_values < 0.0).any()) + assert np.isinf(floor).all() == (not (weighted_values < 0.0).any()) @_PROPERTY_SETTINGS @@ -445,12 +484,14 @@ def _finalize( raw: dict[str, np.ndarray], *, diagnostics: list[dict[str, object]] | None, + tax_unit_raw: dict[str, np.ndarray] | None = None, **kwargs: object, ) -> Frame: + tax_unit_raw = tax_unit_raw or {} predictions = pd.DataFrame( { column: np.asarray(values, dtype=np.float64) - for column, values in raw.items() + for column, values in {**raw, **tax_unit_raw}.items() }, index=_recipient_index(frame), ) @@ -459,12 +500,20 @@ def _finalize( donor, predictions, person_outputs=tuple(raw), - tax_unit_outputs=(), + tax_unit_outputs=tuple(tax_unit_raw), tail_bound_diagnostics=diagnostics, **kwargs, ) +def _puf_tax_unit_values(frame: Frame, column: str) -> np.ndarray: + tax_unit = frame.table("tax_unit") + puf = tax_unit[ + tax_unit[support_channel_column("tax_unit")] == PUF_TAX_DETAIL_SUPPORT_CHANNEL + ].sort_values(support_source_id_column("tax_unit")) + return puf[column].to_numpy(dtype=np.float64) + + def _heavy_tail_donor(column: str, *, losses: bool = True) -> pd.DataFrame: # 1,000 donor units: most small, nine large, one extreme record of weight # 0.13 (the Forbes-tail synthetic records' weight), and a loss tail. @@ -675,7 +724,7 @@ def test_sparsification_rescales_bounded_draws_by_one_common_factor() -> None: ) -def test_signed_mass_calibration_still_pins_both_legs_after_the_bound() -> None: +def test_signed_mass_calibration_rescales_each_bounded_leg_by_one_factor() -> None: frame = _recipient_frame([20.0, 400.0, 600.0, 980.0, 300.0]) donor = _heavy_tail_donor(_CALIBRATED) raw = np.asarray([150.0, 2.0e8, -40_000.0, 4_000.0, -100.0]) @@ -683,10 +732,20 @@ def test_signed_mass_calibration_still_pins_both_legs_after_the_bound() -> None: finalized = _finalize(frame, donor, {_CALIBRATED: raw}, diagnostics=[]) weights = _recipient_weights(frame) + floor, cap = _bounds(donor[_CALIBRATED], donor["weight"], weights) + bounded = np.minimum(np.maximum(raw, floor), cap) + assert bounded[1] < raw[1] actual = _puf_values(finalized, _CALIBRATED) values = donor[_CALIBRATED].to_numpy() donor_weights = donor["weight"].to_numpy() - for leg, donor_leg in ((actual > 0.0, values > 0.0), (actual < 0.0, values < 0.0)): + for leg, donor_leg in ( + (bounded > 0.0, values > 0.0), + (bounded < 0.0, values < 0.0), + ): + # Each leg is the bounded draws times one factor ... + factors = actual[leg] / bounded[leg] + np.testing.assert_allclose(factors, factors[0], rtol=1e-12) + # ... chosen so the leg's mass per unit weight is the donor's. assert float((actual[leg] * weights[leg]).sum()) / weights.sum() == ( pytest.approx( float((values[donor_leg] * donor_weights[donor_leg]).sum()) @@ -694,11 +753,6 @@ def test_signed_mass_calibration_still_pins_both_legs_after_the_bound() -> None: rel=1e-9, ) ) - # The calibration rescales whole legs, so the bound's effect survives it: - # the extreme draw no longer holds nearly all of the positive leg. - positive = actual > 0.0 - leg_mass = actual[positive] * weights[positive] - assert leg_mass.max() < 0.99 * leg_mass.sum() def test_boolean_outputs_are_not_bounded() -> None: @@ -748,19 +802,168 @@ def test_explicit_narrowing_is_validated_and_can_empty_the_bound() -> None: ) -def test_bound_identity_is_exact_and_names_its_rule() -> None: - identity = puf_support_module.puf_tax_detail_recipient_share_bound_identity() - assert identity["rule"] == ( - puf_support_module.PUF_TAX_DETAIL_RECIPIENT_SHARE_BOUND_RULE - ) - assert ( - identity == puf_support_module.puf_tax_detail_recipient_share_bound_identity() - ) +def test_bound_identity_is_pinned() -> None: + assert puf_support_module.puf_tax_detail_recipient_share_bound_identity() == { + "rule": "recipient_share_slice_mass", + "schema_version": 1, + "scope": "every_amount_output_of_the_invocation", + "excluded_output_classes": [ + "boolean_person_outputs", + "discrete_tax_unit_outputs", + ], + "recipient_share": "household_design_weight_over_puf_recipient_total", + "cap": "top_slice_positive_mass_per_slice_weight", + "floor": "bottom_slice_negative_mass_per_slice_weight", + "snapped_outputs": "bounded_on_snap_grid_values_cap_lowered_onto_grid", + "seam": "raw_tax_unit_draws_after_tail_bound_quantiles", + } + + +def test_every_canonical_amount_output_is_bounded_and_nothing_else() -> None: canonical = ( *puf_support_module.PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS, *puf_support_module.PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS, ) bounded = puf_support_module._recipient_share_bound_amount_outputs(canonical) + excluded = set(canonical) - set(bounded) + assert excluded == set(puf_support_module.US_QBI_BOOLEAN_OUTPUT_COLUMNS) | { + "first_home_mortgage_origination_year", + "second_home_mortgage_origination_year", + } assert "partnership_income" in bounded - assert "first_home_mortgage_origination_year" not in bounded - assert not set(bounded) & set(puf_support_module.US_QBI_BOOLEAN_OUTPUT_COLUMNS) + assert "first_home_mortgage_balance" in bounded + assert list(bounded) == [output for output in canonical if output in bounded] + + +def test_realistic_weight_ratios_cap_a_forbes_tail_draw() -> None: + # The production geometry: ~10^5 donors including 3,900 synthetic records + # of weight 0.13 with Pareto values, and recipients of weight ~600 among + # ~9e7 of recipient weight. Vectorized bounds match the donor-by-donor + # reference, and a 218.9M draw on a weight-644.7 unit is capped at the + # mass of the slice it stands for. + rng = np.random.default_rng(1259) + regular = 100_000 + values = np.concatenate( + [ + np.where( + rng.random(regular) < 0.06, rng.lognormal(10.0, 2.0, regular), 0.0 + ), + (rng.pareto(1.2, 3_900) + 1.0) * 2.0e6, + ] + ) + values[-1] = 2.189e8 + weights = np.concatenate( + [rng.uniform(200.0, 2_400.0, regular), np.full(3_900, 0.13)] + ) + recipient_weights = np.concatenate( + [rng.uniform(300.0, 1_000.0, 4_000), [644.7], [8.97e7]] + ) + + floor, cap = _bounds(values, weights, recipient_weights) + + probe = np.r_[np.arange(0, 4_000, 400), 4_000, 4_001] + reference_cap = np.asarray( + [ + _reference_slice_mean( + values, + weights, + recipient_weights[i] / recipient_weights.sum() * weights.sum(), + top=True, + ) + for i in probe + ] + ) + np.testing.assert_allclose(cap[probe], reference_cap, rtol=1e-9) + assert np.isinf(floor).all() + heavy = 4_000 + assert cap[heavy] < 2.189e8 + take = recipient_weights[heavy] / recipient_weights.sum() * weights.sum() + assert recipient_weights[heavy] * cap[heavy] == pytest.approx( + recipient_weights.sum() / weights.sum() * take * reference_cap[-2], rel=1e-9 + ) + + +def test_non_finite_bounded_draws_are_refused_before_any_write() -> None: + frame = _recipient_frame([20.0, 400.0]) + raw = np.asarray([np.nan, 1.0]) + predictions_before = raw.copy() + with pytest.raises(ValueError, match="must be finite"): + _finalize( + frame, + _heavy_tail_donor(_PASSTHROUGH), + {_PASSTHROUGH: raw}, + diagnostics=[], + ) + np.testing.assert_array_equal(raw, predictions_before) + + +def test_tax_unit_amount_outputs_are_bounded_on_the_tax_unit_table() -> None: + # A passthrough tax-unit amount (mortgage balance) and a snapped, + # sparsified one (domestic production ALD) through the tax-unit write path. + frame = _recipient_frame([20.0, 400.0, 600.0, 980.0]) + balance = "first_home_mortgage_balance" + dpad = "domestic_production_ald" + donor = pd.DataFrame( + { + balance: [200_000.0, 900_000.0, 5.0e7, 0.0], + dpad: [0.0, 1_000.0, 3.0e6, 0.0], + "weight": [900.0, 9.0, 0.13, 90.87], + } + ) + raw_balance = np.asarray([150_000.0, 5.0e7, 0.0, 300_000.0]) + raw_dpad = np.asarray([0.0, 3.0e6, 0.0, 0.0]) + diagnostics: list[dict[str, object]] = [] + + finalized = _finalize( + frame, + donor, + {}, + tax_unit_raw={balance: raw_balance, dpad: raw_dpad}, + diagnostics=diagnostics, + ) + + weights = _recipient_weights(frame) + _, balance_cap = _bounds(donor[balance], donor["weight"], weights) + actual_balance = _puf_tax_unit_values(finalized, balance) + np.testing.assert_array_equal(actual_balance, np.minimum(raw_balance, balance_cap)) + assert actual_balance[1] < raw_balance[1] + _, dpad_cap = _bounds( + puf_support_module._snap_grid_values(donor[dpad]), donor["weight"], weights + ) + actual_dpad = _puf_tax_unit_values(finalized, dpad) + assert actual_dpad[1] < raw_dpad[1] + assert actual_dpad[1] <= dpad_cap[1] + assert [record["output"] for record in diagnostics] == [balance, dpad] + + +def test_social_security_reconciliation_keeps_the_bounded_total() -> None: + # The four components are re-split by CPS shares after the bound, so a + # component can move, but each unit's total is the bounded draws' total. + frame = _recipient_frame([20.0, 980.0, 300.0]) + components = puf_support_module.PUF_TAX_DETAIL_SOCIAL_SECURITY_COMPONENT_OUTPUTS + donor = pd.DataFrame( + { + "social_security_retirement": [20_000.0, 60_000.0, 4.0e6, 0.0], + "social_security_disability": [0.0, 15_000.0, 0.0, 0.0], + "social_security_dependents": [0.0, 0.0, 0.0, 0.0], + "social_security_survivors": [0.0, 0.0, 0.0, 0.0], + "weight": [600.0, 300.0, 0.13, 99.87], + } + ) + raw = { + "social_security_retirement": np.asarray([20_000.0, 4.0e6, 60_000.0]), + "social_security_disability": np.asarray([0.0, 15_000.0, 0.0]), + "social_security_dependents": np.zeros(3), + "social_security_survivors": np.zeros(3), + } + + finalized = _finalize(frame, donor, raw, diagnostics=[]) + + weights = _recipient_weights(frame) + bounded_total = np.zeros(3) + for component in components: + floor, cap = _bounds(donor[component], donor["weight"], weights) + bounded_total += np.minimum(np.maximum(raw[component], floor), cap) + finalized_total = sum(_puf_values(finalized, component) for component in components) + np.testing.assert_allclose(finalized_total, bounded_total, rtol=1e-12) + assert finalized_total[1] < 4.0e6 + 15_000.0 diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_puf_tail_bound.py b/packages/microcosm-build/tests/engine_free/us/test_us_puf_tail_bound.py index d1919c2009..fae4e7dd62 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_puf_tail_bound.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_puf_tail_bound.py @@ -154,6 +154,7 @@ def test_finalizer_clips_tail_without_changing_participation_or_lower_bits() -> np.testing.assert_array_equal(actual[:2].view(np.uint64), raw[:2].view(np.uint64)) assert diagnostics[0] == { "output": _TARGET, + "rule": "weighted_positive_donor_quantile", "quantile": 0.999, "bound_value": 100.0, "clipped_row_count": 2, diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py index dfbb127429..b38fb180d5 100644 --- a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py @@ -338,7 +338,7 @@ def capture_equality(expected: object, actual: object) -> None: "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "f56dd06c602f065546c659c353114d2c9a4f6a7cd7e13daadfec14f48b53152a", + "spec_sha256": "c1ac73b48d1468d91812dc49343db45a257fb9f68a176c8203295cc5967f7bff", }, } diff --git a/tools/build_us_multispine_pool.py b/tools/build_us_multispine_pool.py index 8c852c985b..68abde6c13 100644 --- a/tools/build_us_multispine_pool.py +++ b/tools/build_us_multispine_pool.py @@ -199,6 +199,7 @@ CANONICAL_STACKED_GAP_FILL_SURFACE, CANONICAL_STACKED_POST_PUF_TRANSFER_SURFACE, DEFAULT_STACKED_HOUSEHOLD_MASS_SHARES, + PRIMARY_PUF_EXECUTION_CONFIG_SCHEMA_VERSION, assemble_stacked_spine, assert_stacked_tail_cells_preserved, by_origin_battery, @@ -4221,14 +4222,14 @@ def _stacked_worker_execution_authentication( qrf = config.get("qrf") if isinstance(config, Mapping) else None if ( not isinstance(config, Mapping) - or config.get("schema_version") != 5 + or config.get("schema_version") != PRIMARY_PUF_EXECUTION_CONFIG_SCHEMA_VERSION or not isinstance(qrf, Mapping) ): raise ValueError("Stacked publication primary execution config changed.") return current_worker_execution_authentication_receipt( qrf.get("worker_execution"), manifest_schema_version=POOL_MANIFEST_SCHEMA_VERSION, - execution_config_schema_version=5, + execution_config_schema_version=PRIMARY_PUF_EXECUTION_CONFIG_SCHEMA_VERSION, boundary="stacked publication primary-QRF worker", ) From 635ae0a11e3e007c9a539b0c4f5724ad5730dc3b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 13 Oct 2026 10:23:19 -0400 Subject: [PATCH 5/8] Address review r2: spec-bundle schema assert, claims, tests, diagnostics - test_us_spec_bundle asserted execution-config schema 5; it now reads the shared constant. The operator-ordering doc and a pass-through fixture said 5 too. - Share-bound diagnostics records carry the recipient and donor weight totals, so rungs can be compared. - Tests: the non-finite check is shown to stop before the quantile cap's write; the LP oracle is scaled to a unit objective so solver tolerance cannot flake; a restated assertion is removed; the base-builder fake record carries the rule field. - Changelog: interim label; schema-5 stacked pools are no longer loadable. - Re-pinned the seed protocol and map, coverage evidence, pool-tool spec sha and loader golden. Co-Authored-By: Claude Opus 5.5 --- .../us-puf-recipient-share-bound.fixed.md | 2 +- docs/evidence/spec-engine/us-f0-coverage.json | 16 ++--- docs/us-multispine-operator-ordering.md | 3 +- .../build/spec_engine/inventory_coverage.py | 4 +- .../microcosm/build/us_runtime/puf_support.py | 4 ++ .../engine_contract/us/test_us_spec_bundle.py | 7 ++- .../shared/test_spec_engine_loader.py | 2 +- .../us/test_us_fiscal_refresh_builder.py | 2 +- .../us/test_us_puf_recipient_share_bound.py | 62 ++++++++++++------- .../us/test_us_puf_support_base_builder.py | 1 + .../us/test_us_multispine_pool_tool.py | 2 +- 11 files changed, 68 insertions(+), 37 deletions(-) diff --git a/changelog.d/us-puf-recipient-share-bound.fixed.md b/changelog.d/us-puf-recipient-share-bound.fixed.md index 404d224e6e..58487e144f 100644 --- a/changelog.d/us-puf-recipient-share-bound.fixed.md +++ b/changelog.d/us-puf-recipient-share-bound.fixed.md @@ -1 +1 @@ -A single PUF-channel QRF draw can no longer carry more mass than the population slice its recipient stands for. The #982 income-rank predictor sends the top-ranked survey tax units, each standing for several hundred units, onto the PUF's top-ranked returns, which stand for one return or less: the synthetic Forbes-tail records weigh about 0.13 each. One such draw put $141B of line-32 mass on a single weight-644 unit in the #1251 rebuild. QRF finalization now caps each raw amount draw at the positive mass, per unit of slice weight, of the donor's top slice of the recipient's own weight share, and floors losses from the bottom slice, on every amount output (booleans and origination years excepted). The rule is bound into the primary PUF execution config, which moves to schema 6; every bounded output writes a tail-bound diagnostics record. +A single PUF-channel QRF draw can no longer carry more mass than the population slice its recipient stands for. The #982 income-rank predictor sends the top-ranked survey tax units, each standing for several hundred units, onto the PUF's top-ranked returns, which stand for one return or less: the synthetic Forbes-tail records weigh about 0.13 each. One such draw put $141B of line-32 mass on a single weight-644 unit in the #1251 rebuild. As an interim censor, until a tail stratum carries those donors at their own weights (#958, #964), QRF finalization now caps each raw amount draw at the positive mass, per unit of slice weight, of the donor's top slice of the recipient's own weight share, and floors losses from the bottom slice, on every amount output (booleans and origination years excepted). The cap limits a record to the mass of its slice; it does not remove concentration the slice itself has. The rule is bound into the primary PUF execution config, which moves to schema 6, and stacked pools recorded at schema 5 are no longer loadable. Every bounded output writes a tail-bound diagnostics record. diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index ebe6027ee5..0f53c9af47 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -1656,13 +1656,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "1d7cc2ee599758e256d830640ecf01ce49fe35e4970c07cb6cdcfb009f9e2ee4", - "protocol_sha256": "d2d2d7e4994f8970d4803c1e5506d2d8af2a4a606db2a04860ff7db498a7ffd1" + "map_sha256": "06356affd52a29c92def0549db758e3ac4cb40eb3aa04e5040d1ac10951859e0", + "protocol_sha256": "027c075e17c872780fad2dc6fb349d70a2bf9ec3e98811f09a460f070f6bc608" }, "failures": [], "observed": { - "map_sha256": "1d7cc2ee599758e256d830640ecf01ce49fe35e4970c07cb6cdcfb009f9e2ee4", - "protocol_sha256": "d2d2d7e4994f8970d4803c1e5506d2d8af2a4a606db2a04860ff7db498a7ffd1" + "map_sha256": "06356affd52a29c92def0549db758e3ac4cb40eb3aa04e5040d1ac10951859e0", + "protocol_sha256": "027c075e17c872780fad2dc6fb349d70a2bf9ec3e98811f09a460f070f6bc608" }, "status": "covered" }, @@ -1677,7 +1677,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "d2d2d7e4994f8970d4803c1e5506d2d8af2a4a606db2a04860ff7db498a7ffd1", + "implementation_sha256": "027c075e17c872780fad2dc6fb349d70a2bf9ec3e98811f09a460f070f6bc608", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1698,7 +1698,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "d2d2d7e4994f8970d4803c1e5506d2d8af2a4a606db2a04860ff7db498a7ffd1", + "implementation_sha256": "027c075e17c872780fad2dc6fb349d70a2bf9ec3e98811f09a460f070f6bc608", "protocol": "legacy-v1", "streams": [ "build_model", @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "c1ac73b48d1468d91812dc49343db45a257fb9f68a176c8203295cc5967f7bff" + "spec_sha256": "60b8228598fdf52431e6535a69947bad4f4cb5148f898e275586023780ef850f" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "c1ac73b48d1468d91812dc49343db45a257fb9f68a176c8203295cc5967f7bff" + "spec_sha256": "60b8228598fdf52431e6535a69947bad4f4cb5148f898e275586023780ef850f" }, "status": "pass" } diff --git a/docs/us-multispine-operator-ordering.md b/docs/us-multispine-operator-ordering.md index f3f62e66aa..058b4e887f 100644 --- a/docs/us-multispine-operator-ordering.md +++ b/docs/us-multispine-operator-ordering.md @@ -252,7 +252,8 @@ by_origin_battery QBI reconciliation contracts, the tail schema and support contract, and late-producer registry schema version 17, including execution-receipt contract 4, transition authority 2, and resource-semantics receipt 2. The - primary execution config is version 5, its portable worker identity is + primary execution config is version 6 (it binds the recipient-share bound, + #1259), its portable worker identity is version 1, and its checkpoint input sidecar is version 2. The companion pool manifest uses schema version 10. Older outer authority or materializer payloads are stale; primary-QRF diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 2eccf1a70d..719a7f4dc0 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -363,8 +363,8 @@ # RNG rules and the ordered QRF targets are unchanged. microcosm#104 moves # both again: the protocol attests microcosm.calibrate.solve. The # recipient-share bound in puf_support moves both again (#1259). - "seed_map": "1d7cc2ee599758e256d830640ecf01ce49fe35e4970c07cb6cdcfb009f9e2ee4", - "seed_protocol": "d2d2d7e4994f8970d4803c1e5506d2d8af2a4a606db2a04860ff7db498a7ffd1", + "seed_map": "06356affd52a29c92def0549db758e3ac4cb40eb3aa04e5040d1ac10951859e0", + "seed_protocol": "027c075e17c872780fad2dc6fb349d70a2bf9ec3e98811f09a460f070f6bc608", "source_manifest": "f4f8bfeb79268a4043a843b47cec1d1c0e58e9f5dd6bc23417ada95446dd7b8e", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py index b97bda4128..d9ca972877 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py @@ -2164,6 +2164,10 @@ def finalize_us_puf_tax_detail_predictions( ), "largest_weighted_draw_before": float(weighted_before.max(initial=0.0)), "largest_weighted_draw_after": float(weighted_after.max(initial=0.0)), + "recipient_weight_total": float(recipient_weights.sum()), + "donor_weight_total": float( + pd.to_numeric(donor["weight"], errors="coerce").fillna(0.0).sum() + ), } ) diff --git a/packages/microcosm-build/tests/engine_contract/us/test_us_spec_bundle.py b/packages/microcosm-build/tests/engine_contract/us/test_us_spec_bundle.py index 01b2e92023..4ea8a8e3cc 100644 --- a/packages/microcosm-build/tests/engine_contract/us/test_us_spec_bundle.py +++ b/packages/microcosm-build/tests/engine_contract/us/test_us_spec_bundle.py @@ -1,6 +1,9 @@ """Tests split from packages/microcosm-build/tests/test_us_spec_bundle.py.""" # ruff: noqa: F403, F405 +from microcosm.build.us_runtime.stacked_spine import ( + PRIMARY_PUF_EXECUTION_CONFIG_SCHEMA_VERSION, +) from test_support.microcosm_build.us_spec_bundle import * @@ -291,7 +294,9 @@ def collect_sha256(value: object, path: tuple[str, ...] = ()) -> None: assert graph["execution_receipt_contract"]["version"] == 4 assert graph["execution_receipt_contract"]["transition_authority"]["version"] == 2 assert graph["resource_semantics"]["schema_version"] == 2 - assert primary_binding["schema_version"] == 5 + assert primary_binding["schema_version"] == ( + PRIMARY_PUF_EXECUTION_CONFIG_SCHEMA_VERSION + ) assert primary_binding["qrf"]["worker_execution"] == { "surface": "execution_profile", "resolve_as": "worker_execution", diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py index 0031c67a99..38207c32d6 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py @@ -12,7 +12,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # Pin the domain separator, normalization rules, schema-set receipt, and # exact normative projection as one reviewable golden vector. assert first.spec_sha256 == ( - "7d1a8e72fd13abb09596f88438b5d56e9d667178b37db9d1b10e086675ecd8ef" + "be3c6e2381c43dd51573391fee122d15cde2168733e8adb0b8fa938f2be5a706" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py index a1d494147c..be179e136d 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py @@ -12264,7 +12264,7 @@ def test_exact_k_receipt_carries_current_worker_authentication() -> None: inputs = _gate_failed_exact_k_inputs(builder) worker_authentication = { "manifest_schema_version": 10, - "execution_config_schema_version": 5, + "execution_config_schema_version": 6, "worker_execution_schema_version": 1, "semantic_identity_sha256": "6" * 64, "audit_aliases": { diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_puf_recipient_share_bound.py b/packages/microcosm-build/tests/engine_free/us/test_us_puf_recipient_share_bound.py index 62801f97a5..96b3629323 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_puf_recipient_share_bound.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_puf_recipient_share_bound.py @@ -226,14 +226,17 @@ def test_cap_mass_is_the_most_any_donor_subpopulation_of_that_size_holds( case: tuple[np.ndarray, ...], ) -> None: # Independent of the slice construction: a linear program picks any - # fractional subpopulation of donor weight `take` (0 <= x_d <= w_d) to - # maximize its positive mass (or minimize its negative mass). The bound's - # mass, w_r * cap scaled to donor weight, must equal that optimum. + # fractional subpopulation of donor weight `take` to maximize its positive + # mass (or minimize its negative mass). The problem is scaled to unit + # weight and unit value (y_d = x_d / take, values / max |value|), so the + # solver's tolerances apply to an objective in [0, 1]. The bound per unit + # of slice weight must equal the optimum. from scipy.optimize import linprog values, weights, recipient_weights = case donor_total = weights.sum() recipient_total = recipient_weights.sum() + scale = max(float(np.abs(values).max()), 1.0) floor, cap = _bounds(values, weights, recipient_weights) @@ -242,22 +245,19 @@ def test_cap_mass_is_the_most_any_donor_subpopulation_of_that_size_holds( if take <= 0.0: continue for side, bound in ((1.0, cap[index]), (-1.0, floor[index])): - part = np.maximum(side * values, 0.0) + part = np.maximum(side * values, 0.0) / scale if not (part[weights > 0.0] > 0.0).any(): assert np.isinf(bound) continue result = linprog( -part, A_eq=np.ones((1, len(values))), - b_eq=[take], - bounds=[(0.0, w) for w in weights], + b_eq=[1.0], + bounds=[(0.0, w / take) for w in weights], method="highs", ) assert result.success - best_mass = -result.fun - assert side * bound * take == pytest.approx( - best_mass, rel=1e-7, abs=1e-6 * max(1.0, best_mass) - ) + assert side * bound / scale == pytest.approx(-result.fun, abs=1e-6) @_PROPERTY_SETTINGS @@ -552,6 +552,8 @@ def test_one_extreme_draw_is_bounded_by_its_slice_and_recorded() -> None: "clipped_mass_after": float(expected[1] * weights[1]), "largest_weighted_draw_before": float(np.abs(raw * weights).max()), "largest_weighted_draw_after": float(np.abs(expected * weights).max()), + "recipient_weight_total": float(weights.sum()), + "donor_weight_total": float(donor["weight"].sum()), } ] @@ -877,24 +879,42 @@ def test_realistic_weight_ratios_cap_a_forbes_tail_draw() -> None: assert np.isinf(floor).all() heavy = 4_000 assert cap[heavy] < 2.189e8 - take = recipient_weights[heavy] / recipient_weights.sum() * weights.sum() - assert recipient_weights[heavy] * cap[heavy] == pytest.approx( - recipient_weights.sum() / weights.sum() * take * reference_cap[-2], rel=1e-9 - ) def test_non_finite_bounded_draws_are_refused_before_any_write() -> None: + # non_sch_d_capital_gains has the q0.999 quantile cap, whose write runs + # before the recipient-share bound; a NaN in another bounded output must + # stop finalization before that write or any other. frame = _recipient_frame([20.0, 400.0]) - raw = np.asarray([np.nan, 1.0]) - predictions_before = raw.copy() + donor = pd.DataFrame( + { + "non_sch_d_capital_gains": [100.0, 200.0, 300.0], + _PASSTHROUGH: [100.0, 5_000.0, -10.0], + "weight": [999.0, 0.5, 0.5], + } + ) + predictions = pd.DataFrame( + { + "non_sch_d_capital_gains": [50.0, 400.0], + _PASSTHROUGH: [np.nan, 1.0], + }, + index=_recipient_index(frame), + ) + before = predictions.copy(deep=True) + diagnostics: list[dict[str, object]] = [] + with pytest.raises(ValueError, match="must be finite"): - _finalize( + puf_support_module.finalize_us_puf_tax_detail_predictions( frame, - _heavy_tail_donor(_PASSTHROUGH), - {_PASSTHROUGH: raw}, - diagnostics=[], + donor, + predictions, + person_outputs=("non_sch_d_capital_gains", _PASSTHROUGH), + tax_unit_outputs=(), + tail_bound_diagnostics=diagnostics, ) - np.testing.assert_array_equal(raw, predictions_before) + + pd.testing.assert_frame_equal(predictions, before, check_exact=True) + assert diagnostics == [] def test_tax_unit_amount_outputs_are_bounded_on_the_tax_unit_table() -> None: diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_puf_support_base_builder.py b/packages/microcosm-build/tests/engine_free/us/test_us_puf_support_base_builder.py index cf41dc000f..bbe8d2d00a 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_puf_support_base_builder.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_puf_support_base_builder.py @@ -457,6 +457,7 @@ def test_qrf_finalization_stage_records_tail_bound_diagnostics( builder = _load_support_builder_module() record = { "output": "non_sch_d_capital_gains", + "rule": "weighted_positive_donor_quantile", "quantile": 0.999, "bound_value": 100.0, "clipped_row_count": 2, diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py index b38fb180d5..e64843fb6b 100644 --- a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py @@ -338,7 +338,7 @@ def capture_equality(expected: object, actual: object) -> None: "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "c1ac73b48d1468d91812dc49343db45a257fb9f68a176c8203295cc5967f7bff", + "spec_sha256": "60b8228598fdf52431e6535a69947bad4f4cb5148f898e275586023780ef850f", }, } From a3d61e9e8ab7c22c60fbeb1275057c4d2fa4d335 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 13 Oct 2026 10:37:49 -0400 Subject: [PATCH 6/8] Keep the LP oracle well conditioned: no share exceeds one (review r3) Co-Authored-By: Claude Opus 5.5 --- .../tests/engine_free/us/test_us_puf_recipient_share_bound.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_puf_recipient_share_bound.py b/packages/microcosm-build/tests/engine_free/us/test_us_puf_recipient_share_bound.py index 96b3629323..c9a9f02718 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_puf_recipient_share_bound.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_puf_recipient_share_bound.py @@ -253,7 +253,9 @@ def test_cap_mass_is_the_most_any_donor_subpopulation_of_that_size_holds( -part, A_eq=np.ones((1, len(values))), b_eq=[1.0], - bounds=[(0.0, w / take) for w in weights], + # The shares sum to one, so no share exceeds one; capping the + # bound there keeps the problem well conditioned. + bounds=[(0.0, min(1.0, w / take)) for w in weights], method="highs", ) assert result.success From 0260d6404fbb23791495af3dc939bff497c66a72 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 13 Oct 2026 12:15:30 -0400 Subject: [PATCH 7/8] Never zero a snapped carrier; fix tests the Modal CI run exposed GitHub's runner queue for the repository was stalled, so the engine-free and engine-us suites were run on Modal against this branch. They found: - For a snapped output the cap was lowered to the largest grid value within it, which could be zero: a carrier whose slice mass sits below the smallest positive donor value was erased. The cap is now lowered onto the positive grid only, so the bound moves no draw to zero; how many records carry a sparse output stays the donor-rate sparsification's job. - Two tests pinned the old field counts as literals. - Non-production stacked passes that ask for no diagnostics now hand finalization a discarded sink; production still requires one. - Four tests that isolate another mechanism on two- to twenty-unit frames (snapping control column, educator sparsification, farm signed-mass calibration, Social Security preservation) narrow the bound explicitly, through a new test-only passthrough on impute_us_puf_tax_detail_support. - Share-bound diagnostics records carry the recipient count, which is what differs between sample rungs. - Re-pinned the regenerated bundle and every digest derived from it. Co-Authored-By: Claude Opus 5.5 --- docs/evidence/spec-engine/us-f0-coverage.json | 30 +++++------ .../build/spec_engine/inventory_coverage.py | 10 ++-- .../microcosm/build/us/spec/imputation.yaml | 2 +- .../microcosm/build/us_runtime/puf_support.py | 28 +++++++--- .../build/us_runtime/stacked_spine.py | 11 +++- .../us/test_spec_engine_coverage_tool.py | 10 ++-- .../shared/test_spec_engine_field_usage.py | 10 ++-- .../shared/test_spec_engine_loader.py | 2 +- .../us/test_us_puf_recipient_share_bound.py | 51 ++++++++++++++++--- .../us/test_us_puf_social_security.py | 6 ++- .../engine_free/us/test_us_puf_support.py | 11 +++- .../us/test_us_multispine_pool_tool.py | 2 +- .../us_farm_business_income.py | 6 ++- 13 files changed, 126 insertions(+), 53 deletions(-) diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index 0f53c9af47..bc7c7805bc 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -1338,12 +1338,12 @@ ], "expected": { "nodes": 38, - "sha256": "e74fb1e6017cf77397708cbf2ad6e323dffd6be89425bc6544b5679e883cb674" + "sha256": "4e0b91fe9010f9cbb54648c024fce2808b1cd4b283005bf9e04d626f4281f878" }, "failures": [], "observed": { "nodes": 38, - "sha256": "e74fb1e6017cf77397708cbf2ad6e323dffd6be89425bc6544b5679e883cb674" + "sha256": "4e0b91fe9010f9cbb54648c024fce2808b1cd4b283005bf9e04d626f4281f878" }, "status": "covered" }, @@ -1361,12 +1361,12 @@ ], "expected": { "producer_count": 38, - "sha256": "b2580f8725297ea535191416d37b77ccf891003b76db8e30d3c949e986b5cef6" + "sha256": "7bc8373a649e36eacfa8e1f0fe5e57179979913c0ad2ebba2c4b29e4bd81684a" }, "failures": [], "observed": { "producer_count": 38, - "sha256": "b2580f8725297ea535191416d37b77ccf891003b76db8e30d3c949e986b5cef6" + "sha256": "7bc8373a649e36eacfa8e1f0fe5e57179979913c0ad2ebba2c4b29e4bd81684a" }, "status": "covered" }, @@ -1656,13 +1656,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "06356affd52a29c92def0549db758e3ac4cb40eb3aa04e5040d1ac10951859e0", - "protocol_sha256": "027c075e17c872780fad2dc6fb349d70a2bf9ec3e98811f09a460f070f6bc608" + "map_sha256": "96ed99b8bebef1561727af392082e2f4ec52fb3d466bfaa1a298bd1697d103ae", + "protocol_sha256": "57a75eba1a98e3006732122441f6e59c897201cb48e7dbdcc401ea48a1ef89d7" }, "failures": [], "observed": { - "map_sha256": "06356affd52a29c92def0549db758e3ac4cb40eb3aa04e5040d1ac10951859e0", - "protocol_sha256": "027c075e17c872780fad2dc6fb349d70a2bf9ec3e98811f09a460f070f6bc608" + "map_sha256": "96ed99b8bebef1561727af392082e2f4ec52fb3d466bfaa1a298bd1697d103ae", + "protocol_sha256": "57a75eba1a98e3006732122441f6e59c897201cb48e7dbdcc401ea48a1ef89d7" }, "status": "covered" }, @@ -1677,7 +1677,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "027c075e17c872780fad2dc6fb349d70a2bf9ec3e98811f09a460f070f6bc608", + "implementation_sha256": "57a75eba1a98e3006732122441f6e59c897201cb48e7dbdcc401ea48a1ef89d7", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1698,7 +1698,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "027c075e17c872780fad2dc6fb349d70a2bf9ec3e98811f09a460f070f6bc608", + "implementation_sha256": "57a75eba1a98e3006732122441f6e59c897201cb48e7dbdcc401ea48a1ef89d7", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1887,7 +1887,7 @@ "alpha", "zeta" ], - "sha256": "d7b84ec565db19067d984e0596ca1805110038c38bec6b5d02ac89fde7f261cf" + "sha256": "5d06e498c2167a4b008db07bca3f82294f494fe071ab8de874094aaf6c59643f" }, "failures": [], "observed": { @@ -1911,7 +1911,7 @@ "alpha", "zeta" ], - "sha256": "d7b84ec565db19067d984e0596ca1805110038c38bec6b5d02ac89fde7f261cf" + "sha256": "5d06e498c2167a4b008db07bca3f82294f494fe071ab8de874094aaf6c59643f" }, "status": "covered" }, @@ -1975,7 +1975,7 @@ "take_up_contract", "us_qbi_reconciliation_contract" ], - "sha256": "fcb03fb129f808ee52a27c742a7d9a4f0f1fd9f1729f4be68dcfd7b8fdaca372" + "sha256": "f02a16ff1365b1797ad34725980ec73155f613f72ce45f7a8cc8fd2a259615d0" }, "status": "covered" }, @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "60b8228598fdf52431e6535a69947bad4f4cb5148f898e275586023780ef850f" + "spec_sha256": "6feff01e7b35a1846e81663d6aca8ae7c34d25d24d8b1bcf259ff87731c9d655" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "60b8228598fdf52431e6535a69947bad4f4cb5148f898e275586023780ef850f" + "spec_sha256": "6feff01e7b35a1846e81663d6aca8ae7c34d25d24d8b1bcf259ff87731c9d655" }, "status": "pass" } diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 719a7f4dc0..0b52af0cc9 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -350,12 +350,12 @@ "acs_person_predictors": "878c788a6f037d7aca12b3586ea034eff04f3034ffa11935a736493042551f25", "authority": "9d4a9672a0f03039b1fe874b9fe21ed575be0d29f14afc396d03cdf5c809bdd2", "early_families": "4aa9f736fd76e83955477ad1667e58f48f264783f05bdc7f0102cd32d61323bd", - "full_checkpoint": "d7b84ec565db19067d984e0596ca1805110038c38bec6b5d02ac89fde7f261cf", + "full_checkpoint": "5d06e498c2167a4b008db07bca3f82294f494fe071ab8de874094aaf6c59643f", "gap_fill_schedule": "1c31f9868f7884347cc19cf1ff65da43f950b9114941a715bab168246db414a7", - "graph_nodes": "e74fb1e6017cf77397708cbf2ad6e323dffd6be89425bc6544b5679e883cb674", + "graph_nodes": "4e0b91fe9010f9cbb54648c024fce2808b1cd4b283005bf9e04d626f4281f878", "geography_assignment": "44e7f5f6cd7ceeef85d532c3d46306d03de4f4ee811376e6d29920742746b65c", "late_families": "d91f9ff0eb52f43e7b6eed3d5c58c37abe1620c3a11021da15dae9c10e16d382", - "late_resource_semantics": "b2580f8725297ea535191416d37b77ccf891003b76db8e30d3c949e986b5cef6", + "late_resource_semantics": "7bc8373a649e36eacfa8e1f0fe5e57179979913c0ad2ebba2c4b29e4bd81684a", "late_schedule": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "fdf23da429f8c501e198a2f5b719763d523a41565c2153fdf17b0344f21bcd7c", @@ -363,8 +363,8 @@ # RNG rules and the ordered QRF targets are unchanged. microcosm#104 moves # both again: the protocol attests microcosm.calibrate.solve. The # recipient-share bound in puf_support moves both again (#1259). - "seed_map": "06356affd52a29c92def0549db758e3ac4cb40eb3aa04e5040d1ac10951859e0", - "seed_protocol": "027c075e17c872780fad2dc6fb349d70a2bf9ec3e98811f09a460f070f6bc608", + "seed_map": "96ed99b8bebef1561727af392082e2f4ec52fb3d466bfaa1a298bd1697d103ae", + "seed_protocol": "57a75eba1a98e3006732122441f6e59c897201cb48e7dbdcc401ea48a1ef89d7", "source_manifest": "f4f8bfeb79268a4043a843b47cec1d1c0e58e9f5dd6bc23417ada95446dd7b8e", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/imputation.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/imputation.yaml index 445d7cf5b6..62df128c83 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/imputation.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/imputation.yaml @@ -4415,7 +4415,7 @@ producer_graph: recipient_share: household_design_weight_over_puf_recipient_total cap: top_slice_positive_mass_per_slice_weight floor: bottom_slice_negative_mass_per_slice_weight - snapped_outputs: bounded_on_snap_grid_values_cap_lowered_onto_grid + snapped_outputs: bounded_on_snap_grid_values_cap_lowered_onto_positive_grid seam: raw_tax_unit_draws_after_tail_bound_quantiles worker_execution: surface: execution_profile diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py index d9ca972877..eae23b7353 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py @@ -433,7 +433,7 @@ def puf_tax_detail_recipient_share_bound_identity() -> dict[str, object]: "recipient_share": "household_design_weight_over_puf_recipient_total", "cap": "top_slice_positive_mass_per_slice_weight", "floor": "bottom_slice_negative_mass_per_slice_weight", - "snapped_outputs": "bounded_on_snap_grid_values_cap_lowered_onto_grid", + "snapped_outputs": "bounded_on_snap_grid_values_cap_lowered_onto_positive_grid", "seam": "raw_tax_unit_draws_after_tail_bound_quantiles", } @@ -1735,6 +1735,7 @@ def impute_us_puf_tax_detail_support( raw_predictions_callback: Callable[[pd.DataFrame], None] | None = None, tail_bound_diagnostics: list[dict[str, object]] | None = None, tail_bound_quantiles: Mapping[str, float] | None = None, + recipient_share_bound_outputs: Sequence[str] | None = None, predictor_universe_receipts: list[dict[str, object]] | None = None, require_complete_recipient_predictors: bool = False, absent_cells: str = PUF_ABSENT_CELLS_LEGACY_ZERO_FILL, @@ -1766,6 +1767,10 @@ def impute_us_puf_tax_detail_support( tail_bound_diagnostics: Optional output sink for the per-target tail-bound records produced during finalization. Build callers publish these records with the QRF-finalization telemetry. + recipient_share_bound_outputs: Test-only narrowing of the + recipient-share bound (see + :func:`finalize_us_puf_tax_detail_predictions`). Production callers + leave it unset, which bounds every amount output. predictor_universe_receipts: Optional output sink for the exact recipient predictor source-universe receipt. Stacked callers bind this receipt into the PUF-pass provenance; legacy callers omit it. @@ -1863,6 +1868,7 @@ def impute_us_puf_tax_detail_support( tax_unit_outputs=tax_unit_outputs, tail_bound_diagnostics=tail_bound_diagnostics, tail_bound_quantiles=tail_bound_quantiles, + recipient_share_bound_outputs=recipient_share_bound_outputs, absent_cells=absent_cells, ) @@ -1987,7 +1993,9 @@ def finalize_us_puf_tax_detail_predictions( explicit ``recipient_share_bound_outputs`` narrows it for tests and must name amount outputs of this invocation. It writes one diagnostics record per bounded output, including outputs it leaves unchanged. It holds on the - raw tax-unit draws and, for snapped outputs, after snapping. Three later + raw tax-unit draws and, for snapped outputs, after snapping; a snapped + output's cap is never below its smallest positive donor value, so the + bound moves no draw to zero. Three later steps move bounded values without adding mass. The donor-rate sparsification of sparse outputs restores the column's mass after zeroing carriers, and the signed-mass calibration scales each leg of its outputs; @@ -2164,6 +2172,7 @@ def finalize_us_puf_tax_detail_predictions( ), "largest_weighted_draw_before": float(weighted_before.max(initial=0.0)), "largest_weighted_draw_after": float(weighted_after.max(initial=0.0)), + "recipient_count": int(len(recipient_weights)), "recipient_weight_total": float(recipient_weights.sum()), "donor_weight_total": float( pd.to_numeric(donor["weight"], errors="coerce").fillna(0.0).sum() @@ -2915,20 +2924,23 @@ def _recipient_share_slice_bounds( def _largest_grid_values_within(caps: np.ndarray, grid: np.ndarray) -> np.ndarray: - """Lower each finite cap to the largest ``grid`` value not above it. + """Lower each finite cap to the largest positive ``grid`` value not above it. A capped value that is itself a grid value snaps to itself, and every value at or below it snaps to a grid value at or below it, so snapping after the - bound cannot carry a value past it. ``grid`` is sorted and starts at its - smallest value; a cap below it keeps that smallest value. + bound cannot carry a value past it. Only positive grid values are used: a + cap below the smallest of them keeps that smallest value, so the bound + never turns a positive draw into a zero. How many records carry a sparse + output is the donor-rate sparsification's job, not this bound's. """ lowered = np.array(caps, dtype=np.float64, copy=True) + positive_grid = grid[grid > 0.0] finite = np.isfinite(lowered) - if not finite.any(): + if not finite.any() or len(positive_grid) == 0: return lowered - position = np.searchsorted(grid, lowered[finite], side="right") - 1 - lowered[finite] = grid[np.maximum(position, 0)] + position = np.searchsorted(positive_grid, lowered[finite], side="right") - 1 + lowered[finite] = positive_grid[np.maximum(position, 0)] return lowered diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py b/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py index a907cb1622..1ef69708c9 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py @@ -11936,6 +11936,13 @@ def _run_stacked_puf_pass_evaluate( if person_outputs is None and tax_unit_outputs is None else None ) + # Finalization always needs a sink, because the recipient-share bound acts + # on every amount output. Production has supplied one (checked above); a + # non-production pass that asked for no diagnostics discards the records, + # and its execution config still records the sink as disabled. + finalization_diagnostics: list[dict[str, object]] = ( + [] if tail_bound_diagnostics is None else tail_bound_diagnostics + ) tail_spec, tail_agi_bands = resolve_puf_capital_gains_tail_execution_inputs() validate_stacked_spine_frame(frame, boundary="stacked PUF pass entry") person_clone = frame.table("person")[support_clone_index_column("person")] @@ -12000,7 +12007,7 @@ def _run_stacked_puf_pass_evaluate( seed=seed, n_estimators=n_estimators, fit_records=fit_records, - tail_bound_diagnostics=tail_bound_diagnostics, + tail_bound_diagnostics=finalization_diagnostics, tail_bound_quantiles=canonical_tail_bounds, predictor_universe_receipts=predictor_universe_receipts, require_complete_recipient_predictors=True, @@ -12143,7 +12150,7 @@ def _run_stacked_puf_pass_evaluate( imputed, weight_kind = finalize_primary_puf_qrf_chain( cloned, checkpoint_dir, - tail_bound_diagnostics=tail_bound_diagnostics, + tail_bound_diagnostics=finalization_diagnostics, tail_bound_quantiles=canonical_tail_bounds, ) if fit_records is not None: diff --git a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py index 7be8062b02..29f0a62c85 100644 --- a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py +++ b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py @@ -82,21 +82,21 @@ def test_us_coverage_is_exact_complete_and_honest( assert_coverage_complete(coverage_report) assert coverage_report["status"] == "pass" fields = coverage_report["field_usage"] - assert fields["configuration_field_count"] == 42_184 - assert fields["authored_normative_field_count"] == 32_404 + assert fields["configuration_field_count"] == 42_194 + assert fields["authored_normative_field_count"] == 32_414 assert fields["resolved_binding_field_count"] == 9_780 - assert fields["consumed_field_count"] == 42_184 + assert fields["consumed_field_count"] == 42_194 assert fields["unused_field_count"] == 0 assert fields["multiple_primary_use_field_count"] == 0 assert fields["claim_count"] == 49 assert fields["mode_counts"] == { "legacy_behavior": 14_010, - "compiler_semantic": 27_717, + "compiler_semantic": 27_727, "front_end_validation": 354, "identity_only": 103, } assert fields["generation0_effect_counts"] == { - "legacy_behavior": 38_498, + "legacy_behavior": 38_508, "no_generation0_effect": 3_686, } diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py index c5ec18bb51..7475c214c0 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py @@ -95,22 +95,22 @@ def _mutated_bundle( def test_exact_complete_ledger_has_one_primary_mode_per_pointer(field_ledger) -> None: - assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_184 + assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_194 assert field_ledger.source_counts == { - "authored": 32_404, + "authored": 32_414, "resolved_bindings": 9_780, } assert field_ledger.mode_counts == { "legacy_behavior": 14_010, - "compiler_semantic": 27_717, + "compiler_semantic": 27_727, "front_end_validation": 354, "identity_only": 103, } assert field_ledger.generation0_effect_counts == { - "legacy_behavior": 38_498, + "legacy_behavior": 38_508, "no_generation0_effect": 3_686, } - assert len({field.pointer for field in field_ledger.fields}) == 42_184 + assert len({field.pointer for field in field_ledger.fields}) == 42_194 def test_eligibility_concepts_are_validation_not_generation0_behavior( diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py index 38207c32d6..b7ab6474bf 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py @@ -12,7 +12,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # Pin the domain separator, normalization rules, schema-set receipt, and # exact normative projection as one reviewable golden vector. assert first.spec_sha256 == ( - "be3c6e2381c43dd51573391fee122d15cde2168733e8adb0b8fa938f2be5a706" + "aa3257d7f377122660229e1db0ddfad8a2ed2807e703dd2f0ebf6b12fdb1fa88" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_puf_recipient_share_bound.py b/packages/microcosm-build/tests/engine_free/us/test_us_puf_recipient_share_bound.py index c9a9f02718..08d211f6f5 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_puf_recipient_share_bound.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_puf_recipient_share_bound.py @@ -385,21 +385,34 @@ def test_snapping_after_the_lowered_cap_never_crosses_it( observed: list[float], draws: list[float], cap: float ) -> None: grid = np.unique(np.rint(np.asarray(observed, dtype=np.float64)).clip(min=0.0)) + positive_grid = grid[grid > 0.0] caps = np.full(len(draws), cap) lowered = _grid_within(caps, grid) - bounded = np.minimum(np.asarray(draws, dtype=np.float64), lowered) + raw = np.asarray(draws, dtype=np.float64) + bounded = np.minimum(raw, lowered) snapped = _snap(bounded, observed) - - assert (lowered <= np.maximum(cap, grid[0])).all() - assert np.isin(lowered, grid).all() + unbounded = _snap(raw, observed) + + if len(positive_grid) == 0: + # Nothing positive to snap onto: the cap is left alone and every draw + # snaps to zero with or without it. + np.testing.assert_array_equal(lowered, caps) + assert (snapped == 0.0).all() + return + # The lowered cap is a positive grid value: the largest within the cap, or + # the smallest positive one when the cap is below them all. + assert np.isin(lowered, positive_grid).all() + assert (lowered <= np.maximum(cap, positive_grid[0])).all() assert (snapped <= lowered).all() + # The bound never turns a draw that would have snapped positive into zero. + assert (snapped[unbounded > 0.0] > 0.0).all() -def test_lowering_keeps_infinite_caps_infinite() -> None: +def test_lowering_keeps_infinite_caps_infinite_and_stays_positive() -> None: grid = np.asarray([0.0, 5.0, 10.0]) lowered = _grid_within(np.asarray([np.inf, 7.0, 10.0, 3.0]), grid) - np.testing.assert_array_equal(lowered, [np.inf, 5.0, 10.0, 0.0]) + np.testing.assert_array_equal(lowered, [np.inf, 5.0, 10.0, 5.0]) # --------------------------------------------------------------------------- # @@ -554,6 +567,7 @@ def test_one_extreme_draw_is_bounded_by_its_slice_and_recorded() -> None: "clipped_mass_after": float(expected[1] * weights[1]), "largest_weighted_draw_before": float(np.abs(raw * weights).max()), "largest_weighted_draw_after": float(np.abs(expected * weights).max()), + "recipient_count": 4, "recipient_weight_total": float(weights.sum()), "donor_weight_total": float(donor["weight"].sum()), } @@ -677,6 +691,29 @@ def test_snapped_outputs_stay_on_observed_values_within_the_bound() -> None: assert actual[1] == grid[grid <= cap[1]].max() +def test_the_bound_never_zeroes_a_carrier_of_a_snapped_output() -> None: + # Half the donor holds 10 and half holds nothing, so a recipient standing + # for two thirds of the population has a slice mass of 7.5 per unit, below + # the only positive donor value. Lowering that cap to the grid value 0 + # would erase the carrier; the cap stays at the smallest positive value and + # the carrier count is left to the donor-rate sparsification. + frame = _recipient_frame([20.0, 40.0]) + donor = pd.DataFrame({_SNAPPED: [0.0, 10.0], "weight": [1.0, 1.0]}) + raw = np.asarray([0.25, 9.6]) + diagnostics: list[dict[str, object]] = [] + + finalized = _finalize(frame, donor, {_SNAPPED: raw}, diagnostics=diagnostics) + + _, cap = _bounds( + puf_support_module._snap_grid_values(donor[_SNAPPED]), + donor["weight"], + _recipient_weights(frame), + ) + assert cap[1] < 10.0 + np.testing.assert_array_equal(_puf_values(finalized, _SNAPPED), [0.0, 10.0]) + assert diagnostics[0]["clipped_row_count"] == 0 + + def test_draws_at_a_tied_maximum_on_the_snap_grid_are_not_clipped() -> None: # Statutory caps aged by a factor leave the donor's maximum off the integer # grid (4,504.8 here) yet tied across many returns. The bound is computed @@ -818,7 +855,7 @@ def test_bound_identity_is_pinned() -> None: "recipient_share": "household_design_weight_over_puf_recipient_total", "cap": "top_slice_positive_mass_per_slice_weight", "floor": "bottom_slice_negative_mass_per_slice_weight", - "snapped_outputs": "bounded_on_snap_grid_values_cap_lowered_onto_grid", + "snapped_outputs": "bounded_on_snap_grid_values_cap_lowered_onto_positive_grid", "seam": "raw_tax_unit_draws_after_tail_bound_quantiles", } diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_puf_social_security.py b/packages/microcosm-build/tests/engine_free/us/test_us_puf_social_security.py index d4d2e7ddc5..815a92d8e9 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_puf_social_security.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_puf_social_security.py @@ -60,7 +60,11 @@ def _finalize( person_outputs=components, tax_unit_outputs=(), absent_cells=absent_cells, - tail_bound_diagnostics=[], + # These tests isolate CPS Social Security preservation on two-unit + # frames, where every draw exceeds its recipient's slice. The + # recipient-share bound is covered, with this reconciliation, in + # test_us_puf_recipient_share_bound.py. + recipient_share_bound_outputs=(), ) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_puf_support.py b/packages/microcosm-build/tests/engine_free/us/test_us_puf_support.py index 205e2e43a6..c10b59c96c 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_puf_support.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_puf_support.py @@ -1174,6 +1174,10 @@ def predict( n_estimators=4, seed=0, tail_bound_diagnostics=[], + # The snapped column keeps the recipient-share bound, which must not + # zero its carrier. The dividends column is this test's untouched + # passthrough control, and on a two-unit frame the bound would cap it. + recipient_share_bound_outputs=("taxable_interest_income",), ) puf_people = imputed.table("person")[ @@ -1244,7 +1248,12 @@ def predict( tax_unit_outputs=(), n_estimators=4, seed=0, - tail_bound_diagnostics=[], + # Isolates the donor-rate sparsification and the earnings allocation on + # a two-unit frame, where a unit standing for a quarter of the + # population would have its 300 draw bounded to 200. The bound and + # the sparsification together are covered in + # test_us_puf_recipient_share_bound.py. + recipient_share_bound_outputs=(), ) person = imputed.table("person") diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py index e64843fb6b..0735f98d19 100644 --- a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py @@ -338,7 +338,7 @@ def capture_equality(expected: object, actual: object) -> None: "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "60b8228598fdf52431e6535a69947bad4f4cb5148f898e275586023780ef850f", + "spec_sha256": "6feff01e7b35a1846e81663d6aca8ae7c34d25d24d8b1bcf259ff87731c9d655", }, } diff --git a/test_support/microcosm_build/us_farm_business_income.py b/test_support/microcosm_build/us_farm_business_income.py index a73315a8e8..005fc325b8 100644 --- a/test_support/microcosm_build/us_farm_business_income.py +++ b/test_support/microcosm_build/us_farm_business_income.py @@ -133,7 +133,11 @@ def fit( person_outputs=US_FARM_BUSINESS_INCOME_OUTPUT_COLUMNS, tax_unit_outputs=(), seed=11, - tail_bound_diagnostics=[], + # The fake draws (hundreds) lie far outside this donor's support (at + # most 30), which the recipient-share bound would cap. These tests + # isolate placement and the signed-mass calibration; the bound with + # that calibration is covered in test_us_puf_recipient_share_bound.py. + recipient_share_bound_outputs=(), ) From 683b644289838c3528fe498085f4ba4bad9fb128 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 13 Oct 2026 13:49:21 -0400 Subject: [PATCH 8/8] State the snapped-output exception beside the bound's guarantee (review r4) A snapped output is bounded by the larger of its slice cap and its smallest positive donor value, so the bound never erases a carrier. The module comment promised the slice-mass bound without that exception. Comment only; re-pinned the seed protocol and map, coverage evidence, pool-tool spec sha and loader golden, which attest puf_support.py's bytes. Co-Authored-By: Claude Opus 5.5 --- docs/evidence/spec-engine/us-f0-coverage.json | 16 ++++++++-------- .../build/spec_engine/inventory_coverage.py | 4 ++-- .../microcosm/build/us_runtime/puf_support.py | 4 +++- .../shared/test_spec_engine_loader.py | 2 +- .../us/test_us_multispine_pool_tool.py | 2 +- 5 files changed, 15 insertions(+), 13 deletions(-) diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index bc7c7805bc..6d498ecca6 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -1656,13 +1656,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "96ed99b8bebef1561727af392082e2f4ec52fb3d466bfaa1a298bd1697d103ae", - "protocol_sha256": "57a75eba1a98e3006732122441f6e59c897201cb48e7dbdcc401ea48a1ef89d7" + "map_sha256": "1cf6f86c3c7541bf2f8dd8909a9f42a5b4d2a2cf09bd72c0fe330ff8be12fa3d", + "protocol_sha256": "d56ecc1e77ccca082f76a06dafb8e2e83d1467a5883ab74d8e75d4265e3586a9" }, "failures": [], "observed": { - "map_sha256": "96ed99b8bebef1561727af392082e2f4ec52fb3d466bfaa1a298bd1697d103ae", - "protocol_sha256": "57a75eba1a98e3006732122441f6e59c897201cb48e7dbdcc401ea48a1ef89d7" + "map_sha256": "1cf6f86c3c7541bf2f8dd8909a9f42a5b4d2a2cf09bd72c0fe330ff8be12fa3d", + "protocol_sha256": "d56ecc1e77ccca082f76a06dafb8e2e83d1467a5883ab74d8e75d4265e3586a9" }, "status": "covered" }, @@ -1677,7 +1677,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "57a75eba1a98e3006732122441f6e59c897201cb48e7dbdcc401ea48a1ef89d7", + "implementation_sha256": "d56ecc1e77ccca082f76a06dafb8e2e83d1467a5883ab74d8e75d4265e3586a9", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1698,7 +1698,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "57a75eba1a98e3006732122441f6e59c897201cb48e7dbdcc401ea48a1ef89d7", + "implementation_sha256": "d56ecc1e77ccca082f76a06dafb8e2e83d1467a5883ab74d8e75d4265e3586a9", "protocol": "legacy-v1", "streams": [ "build_model", @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "6feff01e7b35a1846e81663d6aca8ae7c34d25d24d8b1bcf259ff87731c9d655" + "spec_sha256": "fb7990d69e72cc1d333e7589e744e43eea7e938e16e99447819b3a4d1a96c4c8" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "6feff01e7b35a1846e81663d6aca8ae7c34d25d24d8b1bcf259ff87731c9d655" + "spec_sha256": "fb7990d69e72cc1d333e7589e744e43eea7e938e16e99447819b3a4d1a96c4c8" }, "status": "pass" } diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 0b52af0cc9..687cafa80a 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -363,8 +363,8 @@ # RNG rules and the ordered QRF targets are unchanged. microcosm#104 moves # both again: the protocol attests microcosm.calibrate.solve. The # recipient-share bound in puf_support moves both again (#1259). - "seed_map": "96ed99b8bebef1561727af392082e2f4ec52fb3d466bfaa1a298bd1697d103ae", - "seed_protocol": "57a75eba1a98e3006732122441f6e59c897201cb48e7dbdcc401ea48a1ef89d7", + "seed_map": "1cf6f86c3c7541bf2f8dd8909a9f42a5b4d2a2cf09bd72c0fe330ff8be12fa3d", + "seed_protocol": "d56ecc1e77ccca082f76a06dafb8e2e83d1467a5883ab74d8e75d4265e3586a9", "source_manifest": "f4f8bfeb79268a4043a843b47cec1d1c0e58e9f5dd6bc23417ada95446dd7b8e", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py index eae23b7353..12270c682d 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py @@ -403,7 +403,9 @@ def puf_tax_detail_tail_bound_quantiles_identity() -> dict[str, float]: # carries it. The bound caps each raw amount draw at the positive mass, per # unit of slice weight, of the donor's top slice of the recipient's own share # (and floors losses from the bottom slice), so no record carries more mass -# than the most extreme slice of the population it could stand for. It limits +# than the most extreme slice of the population it could stand for. One +# exception: a snapped output is bounded by the larger of that cap and its +# smallest positive donor value, so the bound never erases a carrier. It limits # a record to its slice; it does not remove concentration the slice itself # has, and the mass it clips from passthrough columns is dropped. It is an # interim censor at the finalize seam, like the quantile map above: tail diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py index b7ab6474bf..0a1b1f79f8 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py @@ -12,7 +12,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # Pin the domain separator, normalization rules, schema-set receipt, and # exact normative projection as one reviewable golden vector. assert first.spec_sha256 == ( - "aa3257d7f377122660229e1db0ddfad8a2ed2807e703dd2f0ebf6b12fdb1fa88" + "a87331c733b013ff6fa8a53e0474e8e98a6f27e53322fa35afff6291635a3897" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py index 0735f98d19..f6e52b3f2b 100644 --- a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py @@ -338,7 +338,7 @@ def capture_equality(expected: object, actual: object) -> None: "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "6feff01e7b35a1846e81663d6aca8ae7c34d25d24d8b1bcf259ff87731c9d655", + "spec_sha256": "fb7990d69e72cc1d333e7589e744e43eea7e938e16e99447819b3a4d1a96c4c8", }, }