From 265b3e204b39cfeed68ee36e0224e72c5f85ba6f Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 06:06:15 +0000 Subject: [PATCH 1/3] Give a person with no household no household's row in encode ConceptMapping.encode runs no frame validator, so a person whose household id is null or names no household can reach _Context. get_indexer gave such a person row -1, and the gathers in reference_rows and household_amount_for_persons read -1 as the household table's last row. A null person household id also found a household row whose own id was null. _Context now gives a null household id row -1 whatever the household table holds, and both gathers read row -1 as an appended sentinel: reference row -1 and amount NaN. So allocation gives such a person 0, and no role flags them as a reference person or as a reference person's partner. Frames whose person household ids all name a household encode as before. The row-by-row reference applies the same rule instead of raising KeyError, and the differential test now draws null and unknown person household ids. New example tests cover the probe cases on Int64 and UInt64 ids, a null id in the household table and an empty household table, and Hypothesis properties check the no-household, row-order and conservation invariants. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_014eCgZkFeeyUZcCZLNXec1k --- .../src/microcosm/frame/concept_mapping.py | 17 +- .../shared/test_concept_mapping.py | 294 +++++++++++++++++- .../microcosm_frame/concept_reference.py | 23 +- 3 files changed, 320 insertions(+), 14 deletions(-) diff --git a/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py b/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py index 1379070064..ae5a673f9c 100644 --- a/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py +++ b/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py @@ -2093,6 +2093,9 @@ def __init__(self, person: pd.DataFrame, household: pd.DataFrame) -> None: self._household_rows = pd.Index( _matched_ids(household[_HOUSEHOLD_ID]) ).get_indexer(_matched_ids(person[_PERSON_HOUSEHOLD_ID])) + # A null household id names no household, even where the household + # table holds a null id of its own, which get_indexer would match. + self._household_rows[person[_PERSON_HOUSEHOLD_ID].isna().to_numpy()] = -1 def has(self, concept_id: str) -> bool: item = CONCEPT_BY_ID[concept_id] @@ -2116,15 +2119,21 @@ def person_rows(self, pointer: pd.Series) -> np.ndarray: return rows def household_amount_for_persons(self, concept_id: str) -> np.ndarray: - """A household amount on each person's row.""" + """A household amount on each person's row; NaN with no household.""" amounts = self.values(concept_id).to_numpy(dtype=np.float64) - return amounts[self._household_rows] + # Row -1 reads the appended NaN, never the last household's amount. + return np.append(amounts, np.nan)[self._household_rows] def reference_rows(self) -> np.ndarray: - """For each person, the row of their household's reference person.""" + """For each person, the row of their household's reference person. + + -1 for a person with no household, as for a household whose + reference person is null or unknown. + """ _require(self, _REFERENCE_PERSON, "Placing values on the reference person") by_household = self.person_rows(self.household["reference_person_id"]) - return by_household[self._household_rows] + # Row -1 reads the appended -1, never the last household's reference. + return np.append(by_household, -1)[self._household_rows] def _matched_ids(values: pd.Series) -> np.ndarray: diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py b/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py index e8b155d5ad..e8b476946c 100644 --- a/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py +++ b/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py @@ -15,11 +15,12 @@ from contextlib import contextmanager from dataclasses import replace from types import MappingProxyType +from typing import NamedTuple import numpy as np import pandas as pd import pytest -from hypothesis import assume, given, settings +from hypothesis import assume, example, given, settings from hypothesis import strategies as st from microcosm.frame import concept_mapping as concept_mapping_module @@ -160,6 +161,27 @@ def _parameters(mapping: ConceptMapping, data) -> dict[str, dict[str, float]]: } +def _detached(tables: dict[str, pd.DataFrame], data) -> dict[str, pd.DataFrame]: + """``tables`` with some persons' household ids made null or unknown. + + ``encode`` runs no frame validator, so a person may name no household. + """ + + person = tables["person"].copy() + pointers = person["person_household_id"].tolist() + known = set(tables["household"]["household_id"].tolist()) + unknown = st.integers(0, 2**40).filter(lambda value: value not in known) + detached = st.lists(st.sampled_from(range(len(pointers))), unique=True) + for row in data.draw(detached, label="detached rows"): + pointers[row] = data.draw(st.none() | unknown, label=f"household of {row}") + person["person_household_id"] = ( + pd.array(pointers, dtype="Int64") + if None in pointers + else np.asarray(pointers, dtype=np.int64) + ) + return {**tables, "person": person} + + def _assert_same_values(actual: pd.Series, expected: pd.Series) -> None: if actual.dtype.kind in "fiub" and expected.dtype.kind in "fiub": np.testing.assert_allclose( @@ -291,6 +313,7 @@ def test_source_household_lineage_changes_no_engine_input( def test_encode_matches_a_row_by_row_reference(self, name, tables, data) -> None: mapping = MAPPINGS[name] parameters = _parameters(mapping, data) + tables = _detached(tables, data) compared = assert_encode_matches_reference( mapping, tables, parameters["shares"], parameters["take_up_rates"] ) @@ -2200,3 +2223,272 @@ def test_a_null_household_id_shares_a_household_with_nobody(self) -> None: } out = _adult_mapping(18).encode(tables).tables["person"] assert out[_ADULT_INPUT].tolist() == [False, False, False] + + +# --- A person with no household -------------------------------------------- + +_POINTER_DTYPES = ("Int64", "UInt64") +_RENT = "fact:household.rent" +#: P3's household id in the probe: null, or an id no household carries. +_DETACHED = pytest.mark.parametrize("detached", [None, 99], ids=["null", "unknown"]) +_REFERENCE_ROLES = ( + Role.REFERENCE_PERSON, + Role.REFERENCE_PERSON_PARTNER, + Role.UNMARRIED_PARTNER_OF_REFERENCE_PERSON, +) + + +def _probe_tables( + dtype: str, + detached: int | None, + *, + references=(1, 2), + partners=(None, None, None, None), + household_ids=(10, 20), +) -> dict[str, pd.DataFrame]: + """P1 in household 10, P2 and P4 in household 20, P3 in ``detached``.""" + + dtype = pd.api.types.pandas_dtype(dtype) + person = pd.DataFrame( + { + "person_id": np.array([1, 2, 3, 4], dtype=np.int64), + "person_household_id": typed([10, 20, detached, 20], dtype), + "partner_person_id": pd.array(list(partners), dtype="Int64"), + "parent_1_person_id": pd.array([None] * 4, dtype="Int64"), + "parent_2_person_id": pd.array([None] * 4, dtype="Int64"), + "legal_marital_status": ["never_married"] * 4, + } + ) + household = pd.DataFrame( + { + "household_id": typed(list(household_ids), dtype), + "reference_person_id": pd.array(list(references), dtype="Int64"), + "rent": [100.0, 900.0, 500.0][: len(household_ids)], + } + ) + return {"person": person, "household": household} + + +def _context(tables: dict[str, pd.DataFrame]): + return concept_mapping_module._Context(tables["person"], tables["household"]) + + +class _Strays(NamedTuple): + """A frame in which some persons may name no household, and a row order + for each of its tables.""" + + tables: dict[str, pd.DataFrame] + household_order: tuple[int, ...] + person_order: tuple[int, ...] + + +def _reversed(tables: dict[str, pd.DataFrame]) -> _Strays: + return _Strays( + tables, + tuple(reversed(range(len(tables["household"])))), + tuple(reversed(range(len(tables["person"])))), + ) + + +@st.composite +def _strays(draw) -> _Strays: + """A valid frame, then some persons' household ids made null or unknown. + + Ids are typed Int64 or UInt64. A household row whose own id is null may + join, led by a person whose household id is null: get_indexer matches + a null with a null. + """ + + tables = draw(concept_frames(max_households=3, max_members=4)) + dtype = pd.api.types.pandas_dtype(draw(st.sampled_from(_POINTER_DTYPES))) + person = tables["person"].copy() + household = tables["household"].copy() + known = household["household_id"].tolist() + pointers = person["person_household_id"].tolist() + unknown = st.integers(0, 2**40).filter(lambda value: value not in known) + for row in draw(st.lists(st.sampled_from(range(len(pointers))), unique=True)): + pointers[row] = draw(st.none() | unknown) + household_ids = list(known) + if draw(st.booleans()): + person_ids = person["person_id"].tolist() + nulls = [ + id_ for id_, at in zip(person_ids, pointers, strict=True) if at is None + ] + extra = household.iloc[[0]].copy() + extra["reference_person_id"] = pd.array( + [draw(st.sampled_from(nulls or person_ids))], dtype="Int64" + ) + household = pd.concat([household, extra], ignore_index=True) + household_ids.append(None) + person["person_household_id"] = typed(pointers, dtype) + household["household_id"] = typed(household_ids, dtype) + return _Strays( + {"person": person, "household": household}, + tuple(draw(st.permutations(range(len(household))))), + tuple(draw(st.permutations(range(len(person))))), + ) + + +def _homes(tables: dict[str, pd.DataFrame]) -> list[int | None]: + """Each person's household row, looked up by id; None for no household.""" + + rows = { + int(value): row + for row, value in enumerate(tables["household"]["household_id"]) + if not pd.isna(value) + } + return [ + None if pd.isna(value) else rows.get(int(value)) + for value in tables["person"]["person_household_id"] + ] + + +_STRAYS = settings(max_examples=50, deadline=None) +#: The probe cases, each with both tables' rows reversed. +_PROBES = ( + _reversed(_probe_tables("Int64", None)), + _reversed(_probe_tables("UInt64", 99, partners=(None, 3, 2, None))), + _reversed(_probe_tables("Int64", None, partners=(None, 3, 2, None))), + _reversed(_probe_tables("UInt64", None, references=(1, 3))), + _reversed(_probe_tables("Int64", 99, references=(1, 3))), + _reversed( + _probe_tables( + "UInt64", None, references=(1, 2, 3), household_ids=(10, 20, None) + ) + ), +) + + +def _with_probes(test): + for probe in _PROBES: + test = example(strays=probe)(test) + return test + + +class TestNoHousehold: + """A person whose household id is null or names no household has none. + + ``encode`` runs no frame validator, so such a person can reach it. + + - I1: they read no household's row. Their reference row is -1 and their + household amount NaN; no role flags them as a reference person or a + reference person's partner, and allocation gives them 0. + - I2: permuting the household rows changes no person's values, and + permuting the person rows permutes them alike. + - I3: each household whose reference person is a member keeps its + amount: its members' allocations sum to it. + """ + + @_DETACHED + @pytest.mark.parametrize("dtype", _POINTER_DTYPES) + def test_no_household_reads_no_reference_row(self, dtype, detached) -> None: + # Row -1 read the last household: P3 got household 20's reference. + context = _context(_probe_tables(dtype, detached)) + assert context.reference_rows().tolist() == [0, 1, -1, 1] + amounts = context.household_amount_for_persons(_RENT) + assert amounts[[0, 1, 3]].tolist() == [100.0, 900.0, 900.0] + assert np.isnan(amounts[2]) + + @_DETACHED + @pytest.mark.parametrize("dtype", _POINTER_DTYPES) + def test_partner_flag_ignores_household_row_order(self, dtype, detached) -> None: + # P3 partners P2, household 20's reference person. P3 was flagged + # while household 20 was the last row, and not once it moved up. + tables = _probe_tables(dtype, detached, partners=(None, 3, 2, None)) + role = RelationshipRole(role=Role.REFERENCE_PERSON_PARTNER) + for household in (tables["household"], tables["household"].iloc[::-1]): + context = _context({**tables, "household": household}) + flags = concept_mapping_module._role_flags(role, context) + assert flags.tolist() == [False] * 4 + + @_DETACHED + @pytest.mark.parametrize("dtype", _POINTER_DTYPES) + def test_allocation_skips_a_reference_person_with_no_household( + self, dtype, detached + ) -> None: + # Household 20 names P3, who belongs to no household, as its + # reference person; P3 took household 20's rent. + tables = _probe_tables(dtype, detached, references=(1, 3)) + out = _pointer_mapping().encode(tables).tables["person"] + assert out["rent_paid"].tolist() == [100.0, 0.0, 0.0, 0.0] + assert out["reference_person"].tolist() == [True, False, False, False] + + @pytest.mark.parametrize("dtype", _POINTER_DTYPES) + def test_null_household_id_finds_no_null_household_row(self, dtype) -> None: + # pd.NA finds pd.NA in get_indexer, so P3 found the third household. + tables = _probe_tables( + dtype, None, references=(1, 2, 3), household_ids=(10, 20, None) + ) + assert _context(tables).reference_rows().tolist() == [0, 1, -1, 1] + out = _pointer_mapping().encode(tables).tables["person"] + assert out["reference_person"].tolist() == [True, True, False, False] + assert out["rent_paid"].tolist() == [100.0, 900.0, 0.0, 0.0] + + def test_empty_household_table_leaves_everyone_without_one(self) -> None: + tables = _probe_tables("Int64", None) + tables["household"] = tables["household"].iloc[:0] + context = _context(tables) + assert context.reference_rows().tolist() == [-1] * 4 + assert np.isnan(context.household_amount_for_persons(_RENT)).all() + out = _pointer_mapping().encode(tables).tables["person"] + assert out["rent_paid"].tolist() == [0.0] * 4 + assert not out["reference_person"].any() + + @_STRAYS + @_with_probes + @given(strays=_strays()) + def test_i1_no_household_reads_no_households_row(self, strays) -> None: + tables = strays.tables + household = tables["household"] + context = _context(tables) + references = context.reference_rows() + amounts = context.household_amount_for_persons(_RENT) + out = _pointer_mapping().encode(tables).tables["person"] + row_of = { + int(value): row for row, value in enumerate(tables["person"]["person_id"]) + } + for row, home in enumerate(_homes(tables)): + if home is None: + assert references[row] == -1 + assert np.isnan(amounts[row]) + assert out["rent_paid"].iloc[row] == 0.0 + assert not any(out[role.value].iloc[row] for role in _REFERENCE_ROLES) + continue + reference = household["reference_person_id"].iloc[home] + assert references[row] == row_of.get(int(reference), -1) + assert amounts[row] == household["rent"].iloc[home] + + @_STRAYS + @_with_probes + @given(strays=_strays()) + def test_i2_row_order_changes_no_persons_values(self, strays) -> None: + tables = strays.tables + mapping = _pointer_mapping() + before = mapping.encode(tables).tables["person"] + household = tables["household"].iloc[list(strays.household_order)] + after = mapping.encode( + {**tables, "household": household.reset_index(drop=True)} + ) + pd.testing.assert_frame_equal(after.tables["person"], before) + order = list(strays.person_order) + person = tables["person"].iloc[order].reset_index(drop=True) + moved = mapping.encode({**tables, "person": person}).tables["person"] + pd.testing.assert_frame_equal(moved, before.iloc[order].reset_index(drop=True)) + + @_STRAYS + @_with_probes + @given(strays=_strays()) + def test_i3_each_household_keeps_its_amount(self, strays) -> None: + tables = strays.tables + household = tables["household"] + person_ids = tables["person"]["person_id"].tolist() + paid = _pointer_mapping().encode(tables).tables["person"]["rent_paid"] + homes = _homes(tables) + for row in range(len(household)): + members = [index for index, home in enumerate(homes) if home == row] + reference = household["reference_person_id"].iloc[row] + if pd.isna(reference) or int(reference) not in { + person_ids[member] for member in members + }: + continue + assert paid.iloc[members].sum() == household["rent"].iloc[row] diff --git a/test_support/microcosm_frame/concept_reference.py b/test_support/microcosm_frame/concept_reference.py index 5116821af6..3080fea506 100644 --- a/test_support/microcosm_frame/concept_reference.py +++ b/test_support/microcosm_frame/concept_reference.py @@ -54,22 +54,28 @@ def reference_values( persons = tables["person"].to_dict("records") households = { - row["household_id"]: row for row in tables["household"].to_dict("records") + row["household_id"]: row + for row in tables["household"].to_dict("records") + if _present(row["household_id"]) } by_id = {row["person_id"]: row for row in persons} transform = binding.transform + def household_of(row: dict) -> dict | None: + # A null or unknown household id names no household. + key = row["person_household_id"] + return households.get(key) if _present(key) else None + def value(row: dict, concept_id: str) -> object: item = CONCEPT_BY_ID[concept_id] if item.entity == "person": return row[item.name] - household = ( - row if "person_id" not in row else households[row["person_household_id"]] - ) - return household[item.name] + household = row if "person_id" not in row else household_of(row) + return np.nan if household is None else household[item.name] def reference_of(row: dict) -> object: - return households[row["person_household_id"]]["reference_person_id"] + household = household_of(row) + return None if household is None else household["reference_person_id"] def is_parent_of(row: dict, child: dict, max_age: int | None) -> bool: if max_age is not None and child["age"] > max_age: @@ -113,10 +119,9 @@ def is_parent_of(row: dict, child: dict, max_age: int | None) -> bool: first, second = (float(value(row, concept_id)) for concept_id in concepts) out.append(first * second) elif isinstance(transform, AllocateToReferencePerson): - household = households[row["person_household_id"]] name = CONCEPT_BY_ID[concepts[0]].name - mine = row["person_id"] == household["reference_person_id"] - out.append(float(household[name]) if mine else 0.0) + mine = row["person_id"] == reference_of(row) + out.append(float(household_of(row)[name]) if mine else 0.0) elif isinstance(transform, Predicate): out.append( all( From 85efd10727226fb5c78e9703a310c514d7e9e40c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 06:06:15 +0000 Subject: [PATCH 2/3] Make both holdout guards read malformed holdout rows alike The stage guard in _precalibration_stages read row["entity"] and row["measure"] from every holdout_references row, so a list or dict measure raised TypeError and a missing measure or entity raised KeyError. Activation's guard skipped a row that was not an object with a text measure, so such a row dropped silently out of the separation check. One predicate, _is_holdout, now defines a holdout row for both guards. The stage guard skips any other row, builds holdout_columns only from rows whose entity is text, and names a missing entity as "?". Activation's guard reports each row it cannot read as one gap that names the row, so a declared holdout never drops silently out of both guards. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_014eCgZkFeeyUZcCZLNXec1k --- .../src/microcosm/build/transport/compose.py | 38 +++- .../test_transport_holdout_separation.py | 185 +++++++++++++++++- 2 files changed, 208 insertions(+), 15 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/transport/compose.py b/packages/microcosm-build/src/microcosm/build/transport/compose.py index 620bbd4794..c30af16889 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/compose.py +++ b/packages/microcosm-build/src/microcosm/build/transport/compose.py @@ -1007,6 +1007,15 @@ def _reference_activation_gaps( _HOLDOUT_REFERENCES = "holdout_references" +def _is_holdout(row: object) -> bool: + """Whether a ``holdout_references`` row holds out its measure. + + Activation's guard and the stage guard both read holdouts through this. + """ + + return isinstance(row, Mapping) and _is_text(row.get("measure")) + + def _holdout_separation_gaps( documents: Iterable[tuple[str, Mapping]], ) -> Iterator[str]: @@ -1015,20 +1024,31 @@ def _holdout_separation_gaps( A holdout row tests the calibrated model against a fact it never saw, so no other reference document may carry a row with the same ``measure``, whatever its entity. Only rows of ``holdout_references`` count as - holdouts; a spec without that document has nothing to guard. + holdouts; a spec without that document has nothing to guard. A holdout + row that is not an object with a text measure is a gap itself, since + neither guard can hold it out. """ documents = tuple(documents) holdouts: dict[str, list[Mapping]] = {} + seen = set() for resource, document in documents: if resource != _HOLDOUT_REFERENCES: continue - for row in document.get("target_references", ()): - if isinstance(row, Mapping) and _is_text(row.get("measure")): + for index, row in enumerate(document.get("target_references", ())): + if _is_holdout(row): holdouts.setdefault(row["measure"], []).append(row) + continue + name = row.get("name", "?") if isinstance(row, Mapping) else "?" + gap = ( + f"holdout separation: {resource}.target_references[{index}] " + f"({name}) is not an object with a text measure" + ) + if gap not in seen: + seen.add(gap) + yield gap if not holdouts: return - seen = set() for resource, document in documents: if resource == _HOLDOUT_REFERENCES: continue @@ -2992,7 +3012,13 @@ def _precalibration_stages(spec: Mapping[str, object]) -> dict[str, Mapping]: if _has_resource(spec, "holdout_references") else () ) - holdout_columns = {(row["entity"], row["measure"]) for row in holdout} + # Activation's guard reports each row this skips. + holdout = [row for row in holdout if _is_holdout(row)] + holdout_columns = { + (row["entity"], row["measure"]) + for row in holdout + if _is_text(row.get("entity")) + } # A stage output measures a holdout's concept on any entity. holdout_measures = {row["measure"]: row for row in holdout} holdout_modules = { @@ -3105,7 +3131,7 @@ def _precalibration_stages(spec: Mapping[str, object]) -> dict[str, Mapping]: "precalibration_stages output is a holdout measure: stage " f"{identifier!r} output ({entity}, {column}) measures the " f"concept of holdout_references.{holdout_row.get('name', '?')} " - f"({holdout_row['entity']}, {holdout_row['measure']})." + f"({holdout_row.get('entity', '?')}, {holdout_row['measure']})." ) if stage["rulespec_path"] in holdout_modules: raise ValueError("precalibration_stages module produces a holdout measure.") diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_holdout_separation.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_holdout_separation.py index aadf3c50df..e78467f2e0 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_holdout_separation.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_holdout_separation.py @@ -19,8 +19,15 @@ pre-calibration stages. - Branches: the guard reads the mass document when no node selects it, counts a parked holdout as a holdout, and reads profile-selected views. -- A non-text ``measure`` on any row raises no ``TypeError``; activation's - outcome equals its outcome without the guard. +- A non-text ``measure`` on a non-holdout row raises no ``TypeError``; + activation's outcome equals its outcome without the guard. +- Malformed holdout rows (a list, dict or missing measure, a missing entity, + a row that is not an object) raise in neither guard. Activation's guard + reports one gap for each row that is not an object with a text measure, + and the stage guard skips that row; a row with no entity still holds out. +- I5, property: on random holdout rows, well formed or not, the stage guard + and activation's guard hold out the same measures, neither raises, and + activation reports exactly one gap per malformed row. - Row-level holdouts stay: a spec that declares no ``holdout_references`` rows composes whatever its rows measure. - Property: on random reference documents drawn from a small pool of @@ -298,16 +305,15 @@ def test_nz_stage_output_on_a_holdout_measure_refuses_across_entities(): @pytest.mark.parametrize("measure", [["receives_main"], {"a": 1}], ids=["list", "dict"]) -@pytest.mark.parametrize( - "resource", ["target_references", "precal_references", "holdout_references"] -) +@pytest.mark.parametrize("resource", ["target_references", "precal_references"]) def test_non_text_measure_keeps_activation_outcome(toy, monkeypatch, resource, measure): """A list or dict ``measure`` raises no ``TypeError`` from the guard. - On a non-holdout row the guard skips the row, as it skips a non-text - holdout measure; activation's outcome equals its outcome without the - guard. ``_activation_error`` catches only ``ValueError``, so a - ``TypeError`` fails the test. + On a non-holdout row the guard skips the row; activation's outcome + equals its outcome without the guard. ``_activation_error`` catches only + ``ValueError``, so a ``TypeError`` fails the test. A malformed holdout + row is a gap of its own; ``test_activation_reports_a_malformed_holdout_row`` + covers it. """ spec = copy.deepcopy(toy.spec) @@ -322,6 +328,167 @@ def test_non_text_measure_keeps_activation_outcome(toy, monkeypatch, resource, m assert _separation_gaps(guarded) == [] +_FAMILY_AMOUNT = "toy_family_weekly_amount" +_ODD_CASES = ("list measure", "dict measure", "no measure", "no entity", "not a row") + + +def _odd_holdout(case, measure=_FAMILY_AMOUNT): + """A malformed holdout row, as each case writes it.""" + + if case == "not a row": + return "toy_odd" + row = reference_row("toy_odd", entity="household", measure=measure) + if case == "list measure": + row["measure"] = [measure] + elif case == "dict measure": + row["measure"] = {"a": 1} + elif case == "no measure": + del row["measure"] + else: + del row["entity"] + return row + + +def _malformed_gap(name="toy_odd", index=1): + return ( + f"{SEPARATION}: holdout_references.target_references[{index}] ({name}) " + "is not an object with a text measure" + ) + + +@pytest.mark.parametrize("case", _ODD_CASES) +def test_stage_guard_reads_a_malformed_holdout_row_as_activation_does(toy, case): + """The stage guard raised ``TypeError`` or ``KeyError`` on each of these. + + It now skips a row activation's guard reports, and still holds out a + row with no entity. + """ + + spec = _family_stage(copy.deepcopy(toy.spec)) + _rows(spec, "holdout_references").append(_odd_holdout(case)) + if case != "no entity": + assert set(compose_module._precalibration_stages(spec)) == {"family_housing"} + return + with pytest.raises(ValueError) as refused: + compose_module._precalibration_stages(spec) + assert str(refused.value) == ( + "precalibration_stages output is a holdout measure: stage " + "'toy.precal.family_housing' output (family, toy_family_weekly_amount) " + "measures the concept of holdout_references.toy_odd " + "(?, toy_family_weekly_amount)." + ) + + +@pytest.mark.parametrize("case", _ODD_CASES) +def test_activation_reports_a_malformed_holdout_row(toy, case): + """Activation's guard skipped each of these rows without a word.""" + + spec = copy.deepcopy(toy.spec) + _rows(spec, "holdout_references").append(_odd_holdout(case)) + views = [("holdout_references", spec["resources"]["holdout_references"])] + gaps = list(compose_module._holdout_separation_gaps(views)) + if case == "no entity": + # A text measure holds out with or without an entity, and no other + # row measures it. + assert gaps == [] + assert _activation_error(spec) is None + return + if case == "not a row": + assert gaps == [_malformed_gap("?")] + # Activation's row check refuses it before the guard runs. + with pytest.raises(TypeError, match="holdout_references reference must be"): + validate_transport_activation(spec) + return + assert gaps == [_malformed_gap()] + assert _separation_gaps(_activation_error(spec)) == [_malformed_gap()] + + +_HELD = ("held_a", "held_b", "held_c") +_ODD_VALUES = (None, "", 7, ["held_a"], {"held_a": 1}) +_MISSING = object() +_MALFORMED_GAP = re.compile( + rf"{SEPARATION}: holdout_references\.target_references\[(\d+)\] \(.*\) " + "is not an object with a text measure" +) + + +def _holds(row): + """The oracle: an object whose measure is text holds that measure out.""" + + return isinstance(row, dict) and row.get("measure") in _HELD + + +@st.composite +def _holdout_rows(draw): + """Holdout rows on the measures ``_HELD``, well formed or not.""" + + rows = [] + for index in range(draw(st.integers(0, 4))): + if draw(st.integers(0, 3)) == 0: + rows.append(draw(st.sampled_from(("toy_odd", 7, None, ["held_a"])))) + continue + row = reference_row(f"toy_held_{index}", entity="person", measure="held_a") + for key, values in (("measure", _HELD), ("entity", _ENTITIES)): + value = draw(st.sampled_from((*values, *_ODD_VALUES, _MISSING))) + if value is _MISSING: + del row[key] + else: + row[key] = value + rows.append(row) + return rows + + +def _stage_holds_out(spec, measure): + """Whether the stage guard refuses a stage output named ``measure``.""" + + stage = spec["resources"]["precalibration_stages"]["stages"][0] + stage["derived_outputs"] = [ + {"entity": "family", "column": measure, "purpose": "calibration_measure"} + ] + try: + compose_module._precalibration_stages(spec) + except ValueError as error: + assert str(error).startswith( + "precalibration_stages output is a holdout measure: " + ) + return True + return False + + +@settings( + max_examples=60, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@example(rows=[_odd_holdout("list measure", "held_a")]) +@example(rows=[_odd_holdout("dict measure", "held_a")]) +@example(rows=[_odd_holdout("no measure", "held_a")]) +@example(rows=[_odd_holdout("no entity", "held_a")]) +@example(rows=[_odd_holdout("not a row", "held_a")]) +@given(rows=_holdout_rows()) +def test_i5_both_guards_hold_out_the_same_measures(toy, rows): + spec = _family_stage(copy.deepcopy(toy.spec)) + spec["resources"]["holdout_references"]["target_references"] = rows + malformed = [index for index, row in enumerate(rows) if not _holds(row)] + for measure in _HELD: + probe = reference_row("toy_probe", entity="person", measure=measure) + views = [ + ("holdout_references", {"target_references": rows}), + ("target_references", {"target_references": [probe]}), + ] + gaps = list(compose_module._holdout_separation_gaps(views)) + reported = [_MALFORMED_GAP.fullmatch(gap) for gap in gaps] + assert sorted(int(m.group(1)) for m in reported if m) == malformed + leaks = [gap for gap, m in zip(gaps, reported, strict=True) if not m] + assert all( + gap.startswith(f"{SEPARATION}: target_references.toy_probe ") + for gap in leaks + ) + held = [row for row in rows if _holds(row) and row["measure"] == measure] + assert len(leaks) == len(held) + assert _stage_holds_out(spec, measure) == bool(held) + + def _leak_gap(resource, name="toy_leak", entity="household"): return [ f"{SEPARATION}: {resource}.{name} ({entity}, receives_main) measures " From 9b49ef303bb4f0a3188f871236591ade3b363a07 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 3 Nov 2026 06:15:02 -0500 Subject: [PATCH 3/3] Pin the holdout guard's dedupe across views The NZ hub's mutant M4b drops the seen check in _holdout_separation_gaps, so the guard yields every gap it builds, and all 50 tests in test_transport_holdout_separation.py stayed green: no test passed a holdout document in more than one view. The new test passes one document with a malformed row in two views and expects one gap. Co-Authored-By: Claude Opus 5.5 --- .../shared/test_transport_holdout_separation.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_holdout_separation.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_holdout_separation.py index e78467f2e0..c4a5170531 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_holdout_separation.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_holdout_separation.py @@ -403,6 +403,15 @@ def test_activation_reports_a_malformed_holdout_row(toy, case): assert _separation_gaps(_activation_error(spec)) == [_malformed_gap()] +def test_guard_names_a_malformed_holdout_row_once_across_views(): + """The guard names a malformed row once, however many views carry it.""" + + document = {"target_references": [_odd_holdout("list measure")]} + views = [("holdout_references", document)] * 2 + gaps = list(compose_module._holdout_separation_gaps(views)) + assert gaps == [_malformed_gap(index=0)] + + _HELD = ("held_a", "held_b", "held_c") _ODD_VALUES = (None, "", 7, ["held_a"], {"held_a": 1}) _MISSING = object()