diff --git a/packages/microcosm-build/src/microcosm/build/transport/compose.py b/packages/microcosm-build/src/microcosm/build/transport/compose.py index 620bbd4794..c30af16889 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/compose.py +++ b/packages/microcosm-build/src/microcosm/build/transport/compose.py @@ -1007,6 +1007,15 @@ def _reference_activation_gaps( _HOLDOUT_REFERENCES = "holdout_references" +def _is_holdout(row: object) -> bool: + """Whether a ``holdout_references`` row holds out its measure. + + Activation's guard and the stage guard both read holdouts through this. + """ + + return isinstance(row, Mapping) and _is_text(row.get("measure")) + + def _holdout_separation_gaps( documents: Iterable[tuple[str, Mapping]], ) -> Iterator[str]: @@ -1015,20 +1024,31 @@ def _holdout_separation_gaps( A holdout row tests the calibrated model against a fact it never saw, so no other reference document may carry a row with the same ``measure``, whatever its entity. Only rows of ``holdout_references`` count as - holdouts; a spec without that document has nothing to guard. + holdouts; a spec without that document has nothing to guard. A holdout + row that is not an object with a text measure is a gap itself, since + neither guard can hold it out. """ documents = tuple(documents) holdouts: dict[str, list[Mapping]] = {} + seen = set() for resource, document in documents: if resource != _HOLDOUT_REFERENCES: continue - for row in document.get("target_references", ()): - if isinstance(row, Mapping) and _is_text(row.get("measure")): + for index, row in enumerate(document.get("target_references", ())): + if _is_holdout(row): holdouts.setdefault(row["measure"], []).append(row) + continue + name = row.get("name", "?") if isinstance(row, Mapping) else "?" + gap = ( + f"holdout separation: {resource}.target_references[{index}] " + f"({name}) is not an object with a text measure" + ) + if gap not in seen: + seen.add(gap) + yield gap if not holdouts: return - seen = set() for resource, document in documents: if resource == _HOLDOUT_REFERENCES: continue @@ -2992,7 +3012,13 @@ def _precalibration_stages(spec: Mapping[str, object]) -> dict[str, Mapping]: if _has_resource(spec, "holdout_references") else () ) - holdout_columns = {(row["entity"], row["measure"]) for row in holdout} + # Activation's guard reports each row this skips. + holdout = [row for row in holdout if _is_holdout(row)] + holdout_columns = { + (row["entity"], row["measure"]) + for row in holdout + if _is_text(row.get("entity")) + } # A stage output measures a holdout's concept on any entity. holdout_measures = {row["measure"]: row for row in holdout} holdout_modules = { @@ -3105,7 +3131,7 @@ def _precalibration_stages(spec: Mapping[str, object]) -> dict[str, Mapping]: "precalibration_stages output is a holdout measure: stage " f"{identifier!r} output ({entity}, {column}) measures the " f"concept of holdout_references.{holdout_row.get('name', '?')} " - f"({holdout_row['entity']}, {holdout_row['measure']})." + f"({holdout_row.get('entity', '?')}, {holdout_row['measure']})." ) if stage["rulespec_path"] in holdout_modules: raise ValueError("precalibration_stages module produces a holdout measure.") diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_holdout_separation.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_holdout_separation.py index aadf3c50df..c4a5170531 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_holdout_separation.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_holdout_separation.py @@ -19,8 +19,15 @@ pre-calibration stages. - Branches: the guard reads the mass document when no node selects it, counts a parked holdout as a holdout, and reads profile-selected views. -- A non-text ``measure`` on any row raises no ``TypeError``; activation's - outcome equals its outcome without the guard. +- A non-text ``measure`` on a non-holdout row raises no ``TypeError``; + activation's outcome equals its outcome without the guard. +- Malformed holdout rows (a list, dict or missing measure, a missing entity, + a row that is not an object) raise in neither guard. Activation's guard + reports one gap for each row that is not an object with a text measure, + and the stage guard skips that row; a row with no entity still holds out. +- I5, property: on random holdout rows, well formed or not, the stage guard + and activation's guard hold out the same measures, neither raises, and + activation reports exactly one gap per malformed row. - Row-level holdouts stay: a spec that declares no ``holdout_references`` rows composes whatever its rows measure. - Property: on random reference documents drawn from a small pool of @@ -298,16 +305,15 @@ def test_nz_stage_output_on_a_holdout_measure_refuses_across_entities(): @pytest.mark.parametrize("measure", [["receives_main"], {"a": 1}], ids=["list", "dict"]) -@pytest.mark.parametrize( - "resource", ["target_references", "precal_references", "holdout_references"] -) +@pytest.mark.parametrize("resource", ["target_references", "precal_references"]) def test_non_text_measure_keeps_activation_outcome(toy, monkeypatch, resource, measure): """A list or dict ``measure`` raises no ``TypeError`` from the guard. - On a non-holdout row the guard skips the row, as it skips a non-text - holdout measure; activation's outcome equals its outcome without the - guard. ``_activation_error`` catches only ``ValueError``, so a - ``TypeError`` fails the test. + On a non-holdout row the guard skips the row; activation's outcome + equals its outcome without the guard. ``_activation_error`` catches only + ``ValueError``, so a ``TypeError`` fails the test. A malformed holdout + row is a gap of its own; ``test_activation_reports_a_malformed_holdout_row`` + covers it. """ spec = copy.deepcopy(toy.spec) @@ -322,6 +328,176 @@ def test_non_text_measure_keeps_activation_outcome(toy, monkeypatch, resource, m assert _separation_gaps(guarded) == [] +_FAMILY_AMOUNT = "toy_family_weekly_amount" +_ODD_CASES = ("list measure", "dict measure", "no measure", "no entity", "not a row") + + +def _odd_holdout(case, measure=_FAMILY_AMOUNT): + """A malformed holdout row, as each case writes it.""" + + if case == "not a row": + return "toy_odd" + row = reference_row("toy_odd", entity="household", measure=measure) + if case == "list measure": + row["measure"] = [measure] + elif case == "dict measure": + row["measure"] = {"a": 1} + elif case == "no measure": + del row["measure"] + else: + del row["entity"] + return row + + +def _malformed_gap(name="toy_odd", index=1): + return ( + f"{SEPARATION}: holdout_references.target_references[{index}] ({name}) " + "is not an object with a text measure" + ) + + +@pytest.mark.parametrize("case", _ODD_CASES) +def test_stage_guard_reads_a_malformed_holdout_row_as_activation_does(toy, case): + """The stage guard raised ``TypeError`` or ``KeyError`` on each of these. + + It now skips a row activation's guard reports, and still holds out a + row with no entity. + """ + + spec = _family_stage(copy.deepcopy(toy.spec)) + _rows(spec, "holdout_references").append(_odd_holdout(case)) + if case != "no entity": + assert set(compose_module._precalibration_stages(spec)) == {"family_housing"} + return + with pytest.raises(ValueError) as refused: + compose_module._precalibration_stages(spec) + assert str(refused.value) == ( + "precalibration_stages output is a holdout measure: stage " + "'toy.precal.family_housing' output (family, toy_family_weekly_amount) " + "measures the concept of holdout_references.toy_odd " + "(?, toy_family_weekly_amount)." + ) + + +@pytest.mark.parametrize("case", _ODD_CASES) +def test_activation_reports_a_malformed_holdout_row(toy, case): + """Activation's guard skipped each of these rows without a word.""" + + spec = copy.deepcopy(toy.spec) + _rows(spec, "holdout_references").append(_odd_holdout(case)) + views = [("holdout_references", spec["resources"]["holdout_references"])] + gaps = list(compose_module._holdout_separation_gaps(views)) + if case == "no entity": + # A text measure holds out with or without an entity, and no other + # row measures it. + assert gaps == [] + assert _activation_error(spec) is None + return + if case == "not a row": + assert gaps == [_malformed_gap("?")] + # Activation's row check refuses it before the guard runs. + with pytest.raises(TypeError, match="holdout_references reference must be"): + validate_transport_activation(spec) + return + assert gaps == [_malformed_gap()] + assert _separation_gaps(_activation_error(spec)) == [_malformed_gap()] + + +def test_guard_names_a_malformed_holdout_row_once_across_views(): + """The guard names a malformed row once, however many views carry it.""" + + document = {"target_references": [_odd_holdout("list measure")]} + views = [("holdout_references", document)] * 2 + gaps = list(compose_module._holdout_separation_gaps(views)) + assert gaps == [_malformed_gap(index=0)] + + +_HELD = ("held_a", "held_b", "held_c") +_ODD_VALUES = (None, "", 7, ["held_a"], {"held_a": 1}) +_MISSING = object() +_MALFORMED_GAP = re.compile( + rf"{SEPARATION}: holdout_references\.target_references\[(\d+)\] \(.*\) " + "is not an object with a text measure" +) + + +def _holds(row): + """The oracle: an object whose measure is text holds that measure out.""" + + return isinstance(row, dict) and row.get("measure") in _HELD + + +@st.composite +def _holdout_rows(draw): + """Holdout rows on the measures ``_HELD``, well formed or not.""" + + rows = [] + for index in range(draw(st.integers(0, 4))): + if draw(st.integers(0, 3)) == 0: + rows.append(draw(st.sampled_from(("toy_odd", 7, None, ["held_a"])))) + continue + row = reference_row(f"toy_held_{index}", entity="person", measure="held_a") + for key, values in (("measure", _HELD), ("entity", _ENTITIES)): + value = draw(st.sampled_from((*values, *_ODD_VALUES, _MISSING))) + if value is _MISSING: + del row[key] + else: + row[key] = value + rows.append(row) + return rows + + +def _stage_holds_out(spec, measure): + """Whether the stage guard refuses a stage output named ``measure``.""" + + stage = spec["resources"]["precalibration_stages"]["stages"][0] + stage["derived_outputs"] = [ + {"entity": "family", "column": measure, "purpose": "calibration_measure"} + ] + try: + compose_module._precalibration_stages(spec) + except ValueError as error: + assert str(error).startswith( + "precalibration_stages output is a holdout measure: " + ) + return True + return False + + +@settings( + max_examples=60, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@example(rows=[_odd_holdout("list measure", "held_a")]) +@example(rows=[_odd_holdout("dict measure", "held_a")]) +@example(rows=[_odd_holdout("no measure", "held_a")]) +@example(rows=[_odd_holdout("no entity", "held_a")]) +@example(rows=[_odd_holdout("not a row", "held_a")]) +@given(rows=_holdout_rows()) +def test_i5_both_guards_hold_out_the_same_measures(toy, rows): + spec = _family_stage(copy.deepcopy(toy.spec)) + spec["resources"]["holdout_references"]["target_references"] = rows + malformed = [index for index, row in enumerate(rows) if not _holds(row)] + for measure in _HELD: + probe = reference_row("toy_probe", entity="person", measure=measure) + views = [ + ("holdout_references", {"target_references": rows}), + ("target_references", {"target_references": [probe]}), + ] + gaps = list(compose_module._holdout_separation_gaps(views)) + reported = [_MALFORMED_GAP.fullmatch(gap) for gap in gaps] + assert sorted(int(m.group(1)) for m in reported if m) == malformed + leaks = [gap for gap, m in zip(gaps, reported, strict=True) if not m] + assert all( + gap.startswith(f"{SEPARATION}: target_references.toy_probe ") + for gap in leaks + ) + held = [row for row in rows if _holds(row) and row["measure"] == measure] + assert len(leaks) == len(held) + assert _stage_holds_out(spec, measure) == bool(held) + + def _leak_gap(resource, name="toy_leak", entity="household"): return [ f"{SEPARATION}: {resource}.{name} ({entity}, receives_main) measures " diff --git a/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py b/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py index 1379070064..ae5a673f9c 100644 --- a/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py +++ b/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py @@ -2093,6 +2093,9 @@ def __init__(self, person: pd.DataFrame, household: pd.DataFrame) -> None: self._household_rows = pd.Index( _matched_ids(household[_HOUSEHOLD_ID]) ).get_indexer(_matched_ids(person[_PERSON_HOUSEHOLD_ID])) + # A null household id names no household, even where the household + # table holds a null id of its own, which get_indexer would match. + self._household_rows[person[_PERSON_HOUSEHOLD_ID].isna().to_numpy()] = -1 def has(self, concept_id: str) -> bool: item = CONCEPT_BY_ID[concept_id] @@ -2116,15 +2119,21 @@ def person_rows(self, pointer: pd.Series) -> np.ndarray: return rows def household_amount_for_persons(self, concept_id: str) -> np.ndarray: - """A household amount on each person's row.""" + """A household amount on each person's row; NaN with no household.""" amounts = self.values(concept_id).to_numpy(dtype=np.float64) - return amounts[self._household_rows] + # Row -1 reads the appended NaN, never the last household's amount. + return np.append(amounts, np.nan)[self._household_rows] def reference_rows(self) -> np.ndarray: - """For each person, the row of their household's reference person.""" + """For each person, the row of their household's reference person. + + -1 for a person with no household, as for a household whose + reference person is null or unknown. + """ _require(self, _REFERENCE_PERSON, "Placing values on the reference person") by_household = self.person_rows(self.household["reference_person_id"]) - return by_household[self._household_rows] + # Row -1 reads the appended -1, never the last household's reference. + return np.append(by_household, -1)[self._household_rows] def _matched_ids(values: pd.Series) -> np.ndarray: diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py b/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py index e8b155d5ad..e8b476946c 100644 --- a/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py +++ b/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py @@ -15,11 +15,12 @@ from contextlib import contextmanager from dataclasses import replace from types import MappingProxyType +from typing import NamedTuple import numpy as np import pandas as pd import pytest -from hypothesis import assume, given, settings +from hypothesis import assume, example, given, settings from hypothesis import strategies as st from microcosm.frame import concept_mapping as concept_mapping_module @@ -160,6 +161,27 @@ def _parameters(mapping: ConceptMapping, data) -> dict[str, dict[str, float]]: } +def _detached(tables: dict[str, pd.DataFrame], data) -> dict[str, pd.DataFrame]: + """``tables`` with some persons' household ids made null or unknown. + + ``encode`` runs no frame validator, so a person may name no household. + """ + + person = tables["person"].copy() + pointers = person["person_household_id"].tolist() + known = set(tables["household"]["household_id"].tolist()) + unknown = st.integers(0, 2**40).filter(lambda value: value not in known) + detached = st.lists(st.sampled_from(range(len(pointers))), unique=True) + for row in data.draw(detached, label="detached rows"): + pointers[row] = data.draw(st.none() | unknown, label=f"household of {row}") + person["person_household_id"] = ( + pd.array(pointers, dtype="Int64") + if None in pointers + else np.asarray(pointers, dtype=np.int64) + ) + return {**tables, "person": person} + + def _assert_same_values(actual: pd.Series, expected: pd.Series) -> None: if actual.dtype.kind in "fiub" and expected.dtype.kind in "fiub": np.testing.assert_allclose( @@ -291,6 +313,7 @@ def test_source_household_lineage_changes_no_engine_input( def test_encode_matches_a_row_by_row_reference(self, name, tables, data) -> None: mapping = MAPPINGS[name] parameters = _parameters(mapping, data) + tables = _detached(tables, data) compared = assert_encode_matches_reference( mapping, tables, parameters["shares"], parameters["take_up_rates"] ) @@ -2200,3 +2223,272 @@ def test_a_null_household_id_shares_a_household_with_nobody(self) -> None: } out = _adult_mapping(18).encode(tables).tables["person"] assert out[_ADULT_INPUT].tolist() == [False, False, False] + + +# --- A person with no household -------------------------------------------- + +_POINTER_DTYPES = ("Int64", "UInt64") +_RENT = "fact:household.rent" +#: P3's household id in the probe: null, or an id no household carries. +_DETACHED = pytest.mark.parametrize("detached", [None, 99], ids=["null", "unknown"]) +_REFERENCE_ROLES = ( + Role.REFERENCE_PERSON, + Role.REFERENCE_PERSON_PARTNER, + Role.UNMARRIED_PARTNER_OF_REFERENCE_PERSON, +) + + +def _probe_tables( + dtype: str, + detached: int | None, + *, + references=(1, 2), + partners=(None, None, None, None), + household_ids=(10, 20), +) -> dict[str, pd.DataFrame]: + """P1 in household 10, P2 and P4 in household 20, P3 in ``detached``.""" + + dtype = pd.api.types.pandas_dtype(dtype) + person = pd.DataFrame( + { + "person_id": np.array([1, 2, 3, 4], dtype=np.int64), + "person_household_id": typed([10, 20, detached, 20], dtype), + "partner_person_id": pd.array(list(partners), dtype="Int64"), + "parent_1_person_id": pd.array([None] * 4, dtype="Int64"), + "parent_2_person_id": pd.array([None] * 4, dtype="Int64"), + "legal_marital_status": ["never_married"] * 4, + } + ) + household = pd.DataFrame( + { + "household_id": typed(list(household_ids), dtype), + "reference_person_id": pd.array(list(references), dtype="Int64"), + "rent": [100.0, 900.0, 500.0][: len(household_ids)], + } + ) + return {"person": person, "household": household} + + +def _context(tables: dict[str, pd.DataFrame]): + return concept_mapping_module._Context(tables["person"], tables["household"]) + + +class _Strays(NamedTuple): + """A frame in which some persons may name no household, and a row order + for each of its tables.""" + + tables: dict[str, pd.DataFrame] + household_order: tuple[int, ...] + person_order: tuple[int, ...] + + +def _reversed(tables: dict[str, pd.DataFrame]) -> _Strays: + return _Strays( + tables, + tuple(reversed(range(len(tables["household"])))), + tuple(reversed(range(len(tables["person"])))), + ) + + +@st.composite +def _strays(draw) -> _Strays: + """A valid frame, then some persons' household ids made null or unknown. + + Ids are typed Int64 or UInt64. A household row whose own id is null may + join, led by a person whose household id is null: get_indexer matches + a null with a null. + """ + + tables = draw(concept_frames(max_households=3, max_members=4)) + dtype = pd.api.types.pandas_dtype(draw(st.sampled_from(_POINTER_DTYPES))) + person = tables["person"].copy() + household = tables["household"].copy() + known = household["household_id"].tolist() + pointers = person["person_household_id"].tolist() + unknown = st.integers(0, 2**40).filter(lambda value: value not in known) + for row in draw(st.lists(st.sampled_from(range(len(pointers))), unique=True)): + pointers[row] = draw(st.none() | unknown) + household_ids = list(known) + if draw(st.booleans()): + person_ids = person["person_id"].tolist() + nulls = [ + id_ for id_, at in zip(person_ids, pointers, strict=True) if at is None + ] + extra = household.iloc[[0]].copy() + extra["reference_person_id"] = pd.array( + [draw(st.sampled_from(nulls or person_ids))], dtype="Int64" + ) + household = pd.concat([household, extra], ignore_index=True) + household_ids.append(None) + person["person_household_id"] = typed(pointers, dtype) + household["household_id"] = typed(household_ids, dtype) + return _Strays( + {"person": person, "household": household}, + tuple(draw(st.permutations(range(len(household))))), + tuple(draw(st.permutations(range(len(person))))), + ) + + +def _homes(tables: dict[str, pd.DataFrame]) -> list[int | None]: + """Each person's household row, looked up by id; None for no household.""" + + rows = { + int(value): row + for row, value in enumerate(tables["household"]["household_id"]) + if not pd.isna(value) + } + return [ + None if pd.isna(value) else rows.get(int(value)) + for value in tables["person"]["person_household_id"] + ] + + +_STRAYS = settings(max_examples=50, deadline=None) +#: The probe cases, each with both tables' rows reversed. +_PROBES = ( + _reversed(_probe_tables("Int64", None)), + _reversed(_probe_tables("UInt64", 99, partners=(None, 3, 2, None))), + _reversed(_probe_tables("Int64", None, partners=(None, 3, 2, None))), + _reversed(_probe_tables("UInt64", None, references=(1, 3))), + _reversed(_probe_tables("Int64", 99, references=(1, 3))), + _reversed( + _probe_tables( + "UInt64", None, references=(1, 2, 3), household_ids=(10, 20, None) + ) + ), +) + + +def _with_probes(test): + for probe in _PROBES: + test = example(strays=probe)(test) + return test + + +class TestNoHousehold: + """A person whose household id is null or names no household has none. + + ``encode`` runs no frame validator, so such a person can reach it. + + - I1: they read no household's row. Their reference row is -1 and their + household amount NaN; no role flags them as a reference person or a + reference person's partner, and allocation gives them 0. + - I2: permuting the household rows changes no person's values, and + permuting the person rows permutes them alike. + - I3: each household whose reference person is a member keeps its + amount: its members' allocations sum to it. + """ + + @_DETACHED + @pytest.mark.parametrize("dtype", _POINTER_DTYPES) + def test_no_household_reads_no_reference_row(self, dtype, detached) -> None: + # Row -1 read the last household: P3 got household 20's reference. + context = _context(_probe_tables(dtype, detached)) + assert context.reference_rows().tolist() == [0, 1, -1, 1] + amounts = context.household_amount_for_persons(_RENT) + assert amounts[[0, 1, 3]].tolist() == [100.0, 900.0, 900.0] + assert np.isnan(amounts[2]) + + @_DETACHED + @pytest.mark.parametrize("dtype", _POINTER_DTYPES) + def test_partner_flag_ignores_household_row_order(self, dtype, detached) -> None: + # P3 partners P2, household 20's reference person. P3 was flagged + # while household 20 was the last row, and not once it moved up. + tables = _probe_tables(dtype, detached, partners=(None, 3, 2, None)) + role = RelationshipRole(role=Role.REFERENCE_PERSON_PARTNER) + for household in (tables["household"], tables["household"].iloc[::-1]): + context = _context({**tables, "household": household}) + flags = concept_mapping_module._role_flags(role, context) + assert flags.tolist() == [False] * 4 + + @_DETACHED + @pytest.mark.parametrize("dtype", _POINTER_DTYPES) + def test_allocation_skips_a_reference_person_with_no_household( + self, dtype, detached + ) -> None: + # Household 20 names P3, who belongs to no household, as its + # reference person; P3 took household 20's rent. + tables = _probe_tables(dtype, detached, references=(1, 3)) + out = _pointer_mapping().encode(tables).tables["person"] + assert out["rent_paid"].tolist() == [100.0, 0.0, 0.0, 0.0] + assert out["reference_person"].tolist() == [True, False, False, False] + + @pytest.mark.parametrize("dtype", _POINTER_DTYPES) + def test_null_household_id_finds_no_null_household_row(self, dtype) -> None: + # pd.NA finds pd.NA in get_indexer, so P3 found the third household. + tables = _probe_tables( + dtype, None, references=(1, 2, 3), household_ids=(10, 20, None) + ) + assert _context(tables).reference_rows().tolist() == [0, 1, -1, 1] + out = _pointer_mapping().encode(tables).tables["person"] + assert out["reference_person"].tolist() == [True, True, False, False] + assert out["rent_paid"].tolist() == [100.0, 900.0, 0.0, 0.0] + + def test_empty_household_table_leaves_everyone_without_one(self) -> None: + tables = _probe_tables("Int64", None) + tables["household"] = tables["household"].iloc[:0] + context = _context(tables) + assert context.reference_rows().tolist() == [-1] * 4 + assert np.isnan(context.household_amount_for_persons(_RENT)).all() + out = _pointer_mapping().encode(tables).tables["person"] + assert out["rent_paid"].tolist() == [0.0] * 4 + assert not out["reference_person"].any() + + @_STRAYS + @_with_probes + @given(strays=_strays()) + def test_i1_no_household_reads_no_households_row(self, strays) -> None: + tables = strays.tables + household = tables["household"] + context = _context(tables) + references = context.reference_rows() + amounts = context.household_amount_for_persons(_RENT) + out = _pointer_mapping().encode(tables).tables["person"] + row_of = { + int(value): row for row, value in enumerate(tables["person"]["person_id"]) + } + for row, home in enumerate(_homes(tables)): + if home is None: + assert references[row] == -1 + assert np.isnan(amounts[row]) + assert out["rent_paid"].iloc[row] == 0.0 + assert not any(out[role.value].iloc[row] for role in _REFERENCE_ROLES) + continue + reference = household["reference_person_id"].iloc[home] + assert references[row] == row_of.get(int(reference), -1) + assert amounts[row] == household["rent"].iloc[home] + + @_STRAYS + @_with_probes + @given(strays=_strays()) + def test_i2_row_order_changes_no_persons_values(self, strays) -> None: + tables = strays.tables + mapping = _pointer_mapping() + before = mapping.encode(tables).tables["person"] + household = tables["household"].iloc[list(strays.household_order)] + after = mapping.encode( + {**tables, "household": household.reset_index(drop=True)} + ) + pd.testing.assert_frame_equal(after.tables["person"], before) + order = list(strays.person_order) + person = tables["person"].iloc[order].reset_index(drop=True) + moved = mapping.encode({**tables, "person": person}).tables["person"] + pd.testing.assert_frame_equal(moved, before.iloc[order].reset_index(drop=True)) + + @_STRAYS + @_with_probes + @given(strays=_strays()) + def test_i3_each_household_keeps_its_amount(self, strays) -> None: + tables = strays.tables + household = tables["household"] + person_ids = tables["person"]["person_id"].tolist() + paid = _pointer_mapping().encode(tables).tables["person"]["rent_paid"] + homes = _homes(tables) + for row in range(len(household)): + members = [index for index, home in enumerate(homes) if home == row] + reference = household["reference_person_id"].iloc[row] + if pd.isna(reference) or int(reference) not in { + person_ids[member] for member in members + }: + continue + assert paid.iloc[members].sum() == household["rent"].iloc[row] diff --git a/test_support/microcosm_frame/concept_reference.py b/test_support/microcosm_frame/concept_reference.py index 5116821af6..3080fea506 100644 --- a/test_support/microcosm_frame/concept_reference.py +++ b/test_support/microcosm_frame/concept_reference.py @@ -54,22 +54,28 @@ def reference_values( persons = tables["person"].to_dict("records") households = { - row["household_id"]: row for row in tables["household"].to_dict("records") + row["household_id"]: row + for row in tables["household"].to_dict("records") + if _present(row["household_id"]) } by_id = {row["person_id"]: row for row in persons} transform = binding.transform + def household_of(row: dict) -> dict | None: + # A null or unknown household id names no household. + key = row["person_household_id"] + return households.get(key) if _present(key) else None + def value(row: dict, concept_id: str) -> object: item = CONCEPT_BY_ID[concept_id] if item.entity == "person": return row[item.name] - household = ( - row if "person_id" not in row else households[row["person_household_id"]] - ) - return household[item.name] + household = row if "person_id" not in row else household_of(row) + return np.nan if household is None else household[item.name] def reference_of(row: dict) -> object: - return households[row["person_household_id"]]["reference_person_id"] + household = household_of(row) + return None if household is None else household["reference_person_id"] def is_parent_of(row: dict, child: dict, max_age: int | None) -> bool: if max_age is not None and child["age"] > max_age: @@ -113,10 +119,9 @@ def is_parent_of(row: dict, child: dict, max_age: int | None) -> bool: first, second = (float(value(row, concept_id)) for concept_id in concepts) out.append(first * second) elif isinstance(transform, AllocateToReferencePerson): - household = households[row["person_household_id"]] name = CONCEPT_BY_ID[concepts[0]].name - mine = row["person_id"] == household["reference_person_id"] - out.append(float(household[name]) if mine else 0.0) + mine = row["person_id"] == reference_of(row) + out.append(float(household_of(row)[name]) if mine else 0.0) elif isinstance(transform, Predicate): out.append( all(