diff --git a/packages/microcosm-build/src/microcosm/build/transport/column_kernels.py b/packages/microcosm-build/src/microcosm/build/transport/column_kernels.py index a1a482479b..679ecfad7f 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/column_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/transport/column_kernels.py @@ -6,7 +6,12 @@ country package. ``unit_rule`` is the resolved :class:`BenefitUnitRule` representation, rather than an unresolved country proposal. ``mapping`` is a :class:`ConceptMapping` representation; ``rulespec_paths`` selects its -bindings and the closure's defaults. +bindings and the closure's defaults. With a closure, each selected path names +a module the closure lists, which may bind nothing, since a module may default +every input it reads; without one, each names a module the mapping binds. +``concepts.encode@1`` writes the defaults of the mapping's ``person_role`` +entities on the person table, beside their bindings, and the group kernels +refuse such an entity. The encode and scenario kernels derive no take-up draws: they refuse ``take_up_rates``, so the encoder refuses any selected take-up-threshold @@ -162,19 +167,30 @@ def _paths(context: KernelContext, ref: str) -> tuple[str, ...] | None: def _full_mapping(context: KernelContext, ref: str) -> ConceptMapping: - """The whole declared mapping, after checking its selected paths exist.""" + """The whole declared mapping, after checking its selected paths exist. + + Each selected path names a module the mapping binds or the closure lists. + With a closure, :func:`_closure` also refuses a path the closure lacks, + so there each path names a module the closure lists, which may bind + nothing: a module may default every input it reads. + """ mapping = ConceptMapping.from_dict(_document(context, ref, "mapping")) paths = _paths(context, ref) if paths is not None: - unknown = set(paths) - {binding.module for binding in mapping.bindings} + known = {binding.module for binding in mapping.bindings} + if "closure" in context.params: + closure = InputClosure.from_dict(_document(context, ref, "closure")) + known |= set(closure.modules) + unknown = set(paths) - known if unknown: raise ValueError(f"{ref} has unknown mapping paths {sorted(unknown)}.") return mapping -def _mapping(context: KernelContext, ref: str) -> ConceptMapping: - """The mapping restricted to the selected paths, for person encoding.""" - mapping = _full_mapping(context, ref) +def _mapping( + context: KernelContext, ref: str, mapping: ConceptMapping +) -> ConceptMapping: + """``mapping`` restricted to the selected paths, for person encoding.""" paths = _paths(context, ref) if paths is None: return mapping @@ -214,20 +230,30 @@ def _defaults( ref: str, tables: Mapping[str, pd.DataFrame], closure: InputClosure | None, - engine_entities: Mapping[str, str], + engine_entities: Mapping[str, tuple[str, ...]], ) -> dict[str, pd.DataFrame]: + """``tables`` with the closure's defaults for the selected paths. + + ``engine_entities`` maps each frame entity to the engine entities whose + inputs its table holds. Their defaults merge into one set per table, so + two defaults that name one input must agree. + """ out = dict(tables) if closure is None: return out - for frame_entity, engine_entity in engine_entities.items(): + for frame_entity, entities in engine_entities.items(): defaults: dict[str, bool | int | float] = {} for path in _paths(context, ref): - for name, value in closure.defaults(path, engine_entity).items(): - if name in defaults and ( - type(defaults[name]) is not type(value) or defaults[name] != value - ): - raise ValueError(f"{ref} has conflicting defaults for {name!r}.") - defaults[name] = value + for engine_entity in entities: + for name, value in closure.defaults(path, engine_entity).items(): + if name in defaults and ( + type(defaults[name]) is not type(value) + or defaults[name] != value + ): + raise ValueError( + f"{ref} has conflicting defaults for {name!r}." + ) + defaults[name] = value out[frame_entity] = apply_defaults(out[frame_entity], defaults) return out @@ -399,17 +425,25 @@ def run(self, context: KernelContext) -> KernelResult: context, self.ref, required=_ENCODE_REQUIRED, optional=_ENCODE_OPTIONAL ) tables = _whole_tables(context, self.ref, ("person", "household")) - mapping = _mapping(context, self.ref) + full = _full_mapping(context, self.ref) + mapping = _mapping(context, self.ref, full) encoded = mapping.encode( tables, shares=_optional_resource_document(context, self.ref, "shares"), ) + # A person_role entity (Axiom's Child) has one row per person in the + # role, so its defaults go on the person table, as its bindings do. + # The whole mapping names these entities: a selected module may + # default every input of an entity that only other modules bind. + # Only person_role bindings add an entity here: this kernel writes no + # defaults for an engine entity outside the correspondence that no + # person_role binding names, even when the closure defaults its inputs. + entities = { + frame: (engine,) for frame, engine in full.entity_correspondence.items() + } + entities["person"] += full.person_role_entities() outputs = _defaults( - context, - self.ref, - encoded.tables, - _closure(context, self.ref), - mapping.entity_correspondence, + context, self.ref, encoded.tables, _closure(context, self.ref), entities ) return KernelResult( columns=_columns(context, self.ref, outputs), @@ -430,6 +464,13 @@ def _group_inputs(context: KernelContext, ref: str): # state bindings against every concept binding's input name. mapping = _full_mapping(context, ref) entity = string_param(context, ref, "engine_entity") + # concepts.encode@1 writes a person_role entity's bindings and defaults + # on the person table. Here its defaults would land on a unit table too. + if entity in mapping.person_role_entities(): + raise ValueError( + f"{ref} engine_entity {entity!r} holds persons in a role, not built " + "units; concepts.encode@1 writes its inputs on the person table." + ) declaration = _unit_declaration(context, ref) physical = context.params.get("unit_entity") if physical is None: @@ -467,7 +508,7 @@ def _group_inputs(context: KernelContext, ref: str): state_bindings=() if closure is None else closure.state_bindings(), shares=_optional_resource_document(context, ref, "shares"), ) - outputs = _defaults(context, ref, encoded.tables, closure, {rule.entity: entity}) + outputs = _defaults(context, ref, encoded.tables, closure, {rule.entity: (entity,)}) return mapping, encoded, outputs diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_column_kernels.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_column_kernels.py index 8848d99782..0c0dde6bd5 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_column_kernels.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_column_kernels.py @@ -287,6 +287,57 @@ def _mapping_with(bindings): ), ), ) +CHILD_PATH = "xx/child.yaml" +CHILD_DEFAULTS_PATH = "xx/child_defaults.yaml" + + +def _default(module, entity, name, value): + return ClosureEntry( + module=module, + entity=entity, + input=name, + closure_class="defaulted", + value=value, + reason="Synthetic default.", + citation="Toy fixture.", + ) + + +# A person_role entity, as Axiom's Child: one module binds an input and +# defaults another, and a second module only defaults. +ROLE_MAPPING = _mapping_with( + ( + *BINDINGS, + _binding( + "child_age", + "Child", + "fact:person.age", + group_rule="person_role", + module=CHILD_PATH, + ), + ) +) +CHILD_ENCODED = ClosureEntry( + module=CHILD_PATH, + entity="Child", + input="child_age", + closure_class="encoded", + encoded_by="concept_binding", +) +ROLE_CLOSURE = replace( + CLOSURE, + modules={ + **CLOSURE.modules, + CHILD_PATH: RESOURCE_SHA, + CHILD_DEFAULTS_PATH: RESOURCE_SHA, + }, + entries=( + *CLOSURE.entries, + CHILD_ENCODED, + _default(CHILD_PATH, "Child", "child_in_care", False), + _default(CHILD_DEFAULTS_PATH, "Child", "child_rate", 0.5), + ), +) def _rule_params(rule=UNIT_RULE): @@ -296,13 +347,15 @@ def _rule_params(rule=UNIT_RULE): } -def _encode_params(*, groups=False, knobs=None, mapping=MAPPING): +def _encode_params( + *, groups=False, knobs=None, mapping=MAPPING, closure=CLOSURE, paths=None +): params = { "mapping": canonical_text(mapping.to_dict()), "mapping_sha256": digest(mapping.to_dict()), - "closure": canonical_text(CLOSURE.to_dict()), - "closure_sha256": digest(CLOSURE.to_dict()), - "rulespec_paths": (GROUP_PATH if groups else PERSON_PATH,), + "closure": canonical_text(closure.to_dict()), + "closure_sha256": digest(closure.to_dict()), + "rulespec_paths": paths or (GROUP_PATH if groups else PERSON_PATH,), } if groups: params.update(_rule_params()) @@ -565,6 +618,205 @@ def test_person_encoding_equals_direct_mapping_and_defaults(tables): _assert_column(result, frame, "person", output.column, expected[output.column]) +@PROPERTY +@given(tables=concept_frames(max_households=3, max_members=4)) +def test_person_role_bindings_and_defaults_go_on_the_person_table(tables): + frame = _generated_frame(tables) + outputs = ( + Owned("person", "input_age", "int64"), + Owned("person", "eligibility_code", "int64"), + Owned("person", "child_age", "int64"), + Owned("person", "child_in_care", "bool"), + Owned("person", "child_rate", "float64"), + ) + params = _encode_params( + mapping=ROLE_MAPPING, + closure=ROLE_CLOSURE, + paths=(PERSON_PATH, CHILD_PATH, CHILD_DEFAULTS_PATH), + ) + result = CONCEPTS_ENCODE.run(_context(CONCEPTS_ENCODE, frame, params, outputs)) + rows = len(frame.person) + expected = { + "input_age": frame.person["age"], + "eligibility_code": pd.Series([1] * rows), + "child_age": frame.person["age"], + "child_in_care": pd.Series([False] * rows), + "child_rate": pd.Series([0.5] * rows), + } + for output in outputs: + _assert_column(result, frame, "person", output.column, expected[output.column]) + + +@PROPERTY +@given( + tables=concept_frames(max_households=3, max_members=4), + role=st.booleans(), + data=st.data(), +) +def test_without_person_role_defaults_encoding_matches_the_old_gather( + tables, role, data +): + # Before person_role entities joined the person table's entities, the + # kernel gathered defaults for the correspondence's entities only. With + # no closure root on a person_role entity, it writes the same bytes. This + # checks the gather's non-Child part; the merge property below checks its + # Child part. + frame = _generated_frame(tables) + mapping = ROLE_MAPPING if role else MAPPING + closure = replace( + ROLE_CLOSURE, entries=(*CLOSURE.entries, *((CHILD_ENCODED,) if role else ())) + ) + others = sorted({binding.module for binding in mapping.bindings} - {PERSON_PATH}) + chosen = data.draw(st.lists(st.sampled_from(others), unique=True)) + paths = tuple(data.draw(st.permutations([PERSON_PATH, *chosen]))) + outputs = ( + Owned("person", "input_age", "int64"), + Owned("person", "input_income", "float64"), + Owned("person", "eligibility_code", "int64"), + *((Owned("person", "child_age", "int64"),) if CHILD_PATH in paths else ()), + ) + params = _encode_params(mapping=mapping, closure=closure, paths=paths) + result = CONCEPTS_ENCODE.run(_context(CONCEPTS_ENCODE, frame, params, outputs)) + with mock.patch.object(ConceptMapping, "person_role_entities", return_value=()): + old = CONCEPTS_ENCODE.run(_context(CONCEPTS_ENCODE, frame, params, outputs)) + assert result.columns.keys() == old.columns.keys() + for key, values in result.columns.items(): + pd.testing.assert_series_equal(values, old.columns[key], check_exact=True) + + +# Names apart from every encoded input and fixture default, and values whose +# types differ where they compare equal (False == 0 == 0.0). +MERGE_NAMES = ("role_flag", "role_count", "role_share") +MERGE_VALUES = st.sampled_from([False, True, 0, 1, 0.0, 0.5]) +DTYPE_TOKENS = {bool: "bool", int: "int64", float: "float64"} + + +@PROPERTY +@given( + tables=concept_frames(max_households=2, max_members=3), + person=st.dictionaries(st.sampled_from(MERGE_NAMES), MERGE_VALUES, max_size=3), + child=st.dictionaries( + st.sampled_from(MERGE_NAMES), MERGE_VALUES, min_size=1, max_size=3 + ), + child_first=st.booleans(), +) +def test_person_and_child_defaults_merge_on_the_person_table_or_refuse( + tables, person, child, child_first +): + # The person table's defaults merge into one set, in either gather + # order: a name that Person and Child both default must agree in type + # and value, or the kernel refuses. Otherwise each name gets one person + # column holding its default. + frame = _generated_frame(tables) + closure = replace( + ROLE_CLOSURE, + entries=( + *ROLE_CLOSURE.entries, + *(_default(PERSON_PATH, "Person", k, v) for k, v in person.items()), + *(_default(CHILD_DEFAULTS_PATH, "Child", k, v) for k, v in child.items()), + ), + ) + # The selected paths' order sets the gather order. + paths = (PERSON_PATH, CHILD_DEFAULTS_PATH)[:: -1 if child_first else 1] + merged = {**child, **person} + outputs = tuple( + Owned("person", name, DTYPE_TOKENS[type(value)]) + for name, value in sorted(merged.items()) + ) + params = _encode_params(mapping=ROLE_MAPPING, closure=closure, paths=paths) + context = _context(CONCEPTS_ENCODE, frame, params, outputs) + clash = any( + name in person + and (type(person[name]) is not type(value) or person[name] != value) + for name, value in child.items() + ) + if clash: + with pytest.raises(ValueError, match="conflicting defaults"): + CONCEPTS_ENCODE.run(context) + return + result = CONCEPTS_ENCODE.run(context) + for name, value in merged.items(): + expected = pd.Series([value] * len(frame.person)) + _assert_column(result, frame, "person", name, expected) + + +@PROPERTY +@given(tables=concept_frames(max_households=2, max_members=3)) +def test_group_kernels_refuse_a_person_role_entity(tables): + # concepts.encode@1 writes a person_role entity's bindings and defaults + # on the person table, so a group kernel refuses one as its engine + # entity instead of writing its defaults on a unit table as well. + frame = _generated_frame(tables) + knobs = { + "rent_allocation": "reference_unit", + "asset_test": True, + "rent_factor": 1.0, + } + params = _encode_params( + groups=True, + knobs=knobs, + mapping=ROLE_MAPPING, + closure=ROLE_CLOSURE, + paths=(GROUP_PATH, CHILD_PATH), + ) + family = (Owned("family", "input_assets", "float64", rewrite=True),) + child = {**params, "engine_entity": "Child", "rulespec_paths": (CHILD_PATH,)} + in_care = (Owned("family", "child_in_care", "bool", rewrite=True),) + for kernel in (CONCEPTS_ENCODE_GROUPS, TRANSPORT_SCENARIO_OVERRIDE): + # The same mapping still encodes its built units. + result = kernel.run(_context(kernel, frame, params, family)) + assert list(result.columns) == [("family", "input_assets")] + with pytest.raises(ValueError, match="'Child' holds persons in a role"): + kernel.run(_context(kernel, frame, child, in_care)) + + +def test_person_role_defaults_share_the_person_table_with_person_inputs(tmp_path): + sources = write_population_sources(tmp_path / "sources") + frame = _donor_frame(sources) + rows = len(frame.person) + other = _mapping_with( + ( + *ROLE_MAPPING.bindings, + _binding("input_other", "Person", "fact:person.age", module=OTHER_PATH), + ) + ) + + def run(entries=(), paths=(PERSON_PATH, CHILD_PATH), output=None, mapping=None): + closure = replace(ROLE_CLOSURE, entries=(*ROLE_CLOSURE.entries, *entries)) + params = _encode_params( + mapping=mapping or ROLE_MAPPING, closure=closure, paths=paths + ) + owned = output or Owned("person", "child_in_care", "bool") + return CONCEPTS_ENCODE.run(_context(CONCEPTS_ENCODE, frame, params, (owned,))) + + # The selected paths' order sets the gather order. In either order, a + # Person default and a Child default that agree make one column, and + # ones that disagree refuse. + for paths in ((PERSON_PATH, CHILD_PATH), (CHILD_PATH, PERSON_PATH)): + agreed = run( + (_default(PERSON_PATH, "Person", "child_in_care", False),), paths=paths + ) + _assert_column( + agreed, frame, "person", "child_in_care", pd.Series([False] * rows) + ) + with pytest.raises(ValueError, match="conflicting defaults"): + run((_default(PERSON_PATH, "Person", "child_in_care", True),), paths=paths) + # So does a Child default on a column a binding encodes. + with pytest.raises(ValueError, match="already encoded"): + run((_default(CHILD_PATH, "Child", "input_age", 3),)) + # A selected module that binds nothing still writes its defaults: the + # whole mapping names the person_role entities. + rate = Owned("person", "child_rate", "float64") + alone = run(paths=(CHILD_DEFAULTS_PATH,), output=rate) + _assert_column(alone, frame, "person", "child_rate", pd.Series([0.5] * rows)) + # A selected path is one the mapping binds or the closure lists, and + # with a closure, one the closure lists. + with pytest.raises(ValueError, match="unknown mapping paths"): + run(paths=("xx/missing.yaml",)) + with pytest.raises(ValueError, match="closure has no paths"): + run(paths=(OTHER_PATH,), mapping=other) + + @PROPERTY @given( tables=concept_frames(max_households=3, max_members=4), diff --git a/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py b/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py index ae5a673f9c..c2730ac8df 100644 --- a/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py +++ b/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py @@ -13,7 +13,10 @@ bindings are invertible. Bindings on engine group entities (tax units, benefit units, SPM units, families) need unit membership that a concept frame does not carry; they declare a :class:`GroupRule`, and :meth:`encode` -reports them as deferred. :meth:`ConceptMapping.encode_groups` executes them +reports them as deferred. The exception is ``person_role``: each row of such +an entity (Axiom's ``Child``) is one person, so :meth:`encode` writes the +binding on the person table, one value per person. +:meth:`ConceptMapping.encode_groups` executes the other rules once a unit-construction step has built the units from the relationship pointers (:mod:`microcosm.frame.unit_construction` builds benefit units) and hands their :class:`GroupMembership` over. It also executes declared state @@ -130,7 +133,9 @@ class GroupRule(StrEnum): """How a binding on an engine group entity collapses member values. :meth:`ConceptMapping.encode_groups` applies every rule except - :attr:`PERSON_ROLE`, given the units' :class:`GroupMembership`. + :attr:`PERSON_ROLE`, given the units' :class:`GroupMembership`; + :meth:`ConceptMapping.encode` applies :attr:`PERSON_ROLE`, which needs no + units. """ #: Sum the members' values. @@ -1115,7 +1120,8 @@ class EncodedInputs: deferred: Bindings on engine group entities, which need engine unit membership and were not executed here; group encoding (:meth:`ConceptMapping.encode_groups`) executes them on built - units. + units. ``person_role`` bindings need no units: encoding executes + them and never defers them. """ tables: Mapping[str, pd.DataFrame] @@ -1415,7 +1421,8 @@ class EncodedGroupInputs: aligned to that membership's unit table, with the unit id column plus the engine inputs. deferred: Group bindings not executed: their engine entity has no - membership here, or they take the per-person ``person_role`` rule. + membership here. Never a ``person_role`` binding, which + :meth:`ConceptMapping.encode` executes. """ tables: Mapping[str, pd.DataFrame] @@ -1499,6 +1506,7 @@ def __post_init__(self) -> None: ) self._check_shares() self._check_shared_names() + self._check_person_role_entities() object.__setattr__( self, "_by_concept", @@ -1550,27 +1558,67 @@ def _check_binding(self, binding: InputBinding) -> None: _check_group_rule(self.engine, binding) def _check_shared_names(self) -> None: - """One encoded column per input name: every module computes it alike. + """One encoded column per input name: every binding computes it alike. Module-scoped engines can read the same input name in several - modules. An encoded frame holds one column per name and entity, so - every binding of that name must compute it the same way. + modules. An encoded frame holds one column per name and table, so + every binding that writes the column must compute it the same way. + An executable binding writes its concept entity's table, one value + per row, so a ``person_role`` binding shares the person table with + the engine's person entity and with every other ``person_role`` + entity. A group binding writes its engine entity's units. """ - seen: dict[tuple[str, str], InputBinding] = {} + seen: dict[tuple[str, bool, str], InputBinding] = {} for binding in self.bindings: - key = (binding.engine_input, binding.engine_entity) + executable = self.is_executable(binding) + key = ( + binding.engine_input, + executable, + binding.concept_entity if executable else binding.engine_entity, + ) first = seen.setdefault(key, binding) - if (first.concepts, first.transform, first.group_rule) != ( + # Executable bindings all give each row its own value, so the + # rule is part of the computation only for group bindings. + if ( + first.concepts, + first.transform, + None if executable else first.group_rule, + ) != ( binding.concepts, binding.transform, - binding.group_rule, + None if executable else binding.group_rule, ): + entities = " and ".join( + repr(entity) + for entity in sorted({first.engine_entity, binding.engine_entity}) + ) raise ValueError( - f"{self.engine}: {binding.engine_input!r} on " - f"{binding.engine_entity!r} is computed differently in " - f"{first.module} and {binding.module}." + f"{self.engine}: {binding.engine_input!r} on {entities} is " + f"computed differently in {first.module} and {binding.module}." ) + def _check_person_role_entities(self) -> None: + """An engine entity's rows are persons in a role or built units. + + :meth:`encode` writes a ``person_role`` binding on the person table, + and :meth:`encode_groups` writes other group bindings on built units, + so one entity can't take both. + """ + rules: dict[str, set[GroupRule]] = defaultdict(set) + for binding in self.bindings: + if binding.group_rule is not None: + rules[binding.engine_entity].add(binding.group_rule) + mixed = sorted( + entity + for entity, found in rules.items() + if GroupRule.PERSON_ROLE in found and len(found) > 1 + ) + if mixed: + raise ValueError( + f"{self.engine}: {mixed} mix person_role with other group " + "rules; an entity's rows are persons in a role or built units." + ) + def _check_shares(self) -> None: groups: dict[tuple[str, str | None], list[InputBinding]] = defaultdict(list) fractions: set[str] = set() @@ -1738,8 +1786,28 @@ def legal_alignments( return tuple(out.values()) def is_executable(self, binding: InputBinding) -> bool: - """Whether :meth:`encode` computes the binding (it is not deferred).""" - return binding.group_rule is None + """Whether :meth:`encode` computes the binding (it is not deferred). + + A ``person_role`` binding takes each person's own value, so it + executes on the person table like a binding without a group rule. + """ + return binding.group_rule in (None, GroupRule.PERSON_ROLE) + + def person_role_entities(self) -> tuple[str, ...]: + """The engine entities whose bindings use ``person_role``, sorted. + + Such an entity (Axiom's ``Child``) has one row per person in the + role, so :meth:`encode` writes its inputs on the person table. + """ + return tuple( + sorted( + { + binding.engine_entity + for binding in self.bindings + if binding.group_rule is GroupRule.PERSON_ROLE + } + ) + ) def invertible_concepts(self) -> tuple[str, ...]: """Concepts :meth:`decode` recovers exactly from the engine inputs. @@ -1799,9 +1867,11 @@ def encode( ) -> EncodedInputs: """Compute the engine inputs a concept frame supplies. - Every executable binding whose concepts are all present in ``tables`` - is computed. Bindings on engine group entities are returned as - deferred (:meth:`encode_groups` executes them on built units). Share + It computes every executable binding whose concepts are all present + in ``tables``, ``person_role`` bindings included: each writes one + value per person on the person table. It returns other bindings on + engine group entities as deferred (:meth:`encode_groups` executes + them on built units). Share parameters and take-up rates are needed only for the bindings that use them. A take-up rate of ``None`` marks a program the caller leaves unseeded: its flag is not written, so the engine @@ -1870,8 +1940,9 @@ def encode_groups( other units, or shared per adult when ``knobs`` says so; either way each household's units sum back to its amount. - ``person_role`` bindings, and bindings on an engine entity with no - membership, are returned as deferred. Each state binding is executed + It skips ``person_role`` bindings, which :meth:`encode` executes, + and returns bindings on an engine entity with no membership as + deferred. Each state binding is executed the same way from its model-state columns. ``knobs`` then zero or scale named inputs. @@ -1943,10 +2014,10 @@ def write(entity: str, name: str, values: np.ndarray) -> None: produced[name].add(frame_entity) for binding in self.bindings: - if binding.group_rule is None or not keep(binding.module): + if self.is_executable(binding) or not keep(binding.module): continue group = units.get(binding.engine_entity) - if group is None or binding.group_rule is GroupRule.PERSON_ROLE: + if group is None: deferred.append(binding) continue if not all(context.has(concept_id) for concept_id in binding.concepts): diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py b/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py index e8b476946c..df234c1739 100644 --- a/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py +++ b/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py @@ -613,7 +613,10 @@ def test_allocation_needs_the_reference_person(self) -> None: with pytest.raises(ValueError, match="reference_person_id"): _mapping([binding]).encode(tables) - def test_group_bindings_are_deferred_not_executed(self) -> None: + def test_group_bindings_other_than_person_role_are_deferred(self) -> None: + # A person_role binding takes each person's own value, so encode runs + # it (Belgium's Child inputs); every other group binding waits for + # built units. tables = { "person": pd.DataFrame({"person_id": [1], "person_household_id": [1]}), "household": pd.DataFrame({"household_id": [1]}), @@ -625,7 +628,7 @@ def test_group_bindings_are_deferred_not_executed(self) -> None: assert deferred == { binding.engine_input for binding in mapping.bindings - if binding.group_rule is not None + if binding.group_rule not in (None, GroupRule.PERSON_ROLE) } @@ -946,6 +949,24 @@ def _module_binding(module: str, **change) -> InputBinding: ) +# Engine entities whose bindings write the person table, and two ways to +# compute one column there. +_PERSON_WRITERS = { + "Person": {}, + "Child": {"engine_entity": "Child", "group_rule": GroupRule.PERSON_ROLE}, + "Dependant": {"engine_entity": "Dependant", "group_rule": GroupRule.PERSON_ROLE}, +} +_PERSON_COMPUTATIONS = { + "age": {}, + "enrolled": { + "concepts": ("fact:person.education_enrollment",), + "transform": Predicate( + clauses=(("fact:person.education_enrollment", ("tertiary",)),) + ), + }, +} + + class TestConstructionRules: def test_a_minimal_mapping_constructs(self) -> None: mapping = _mapping([_binding()]) @@ -1117,6 +1138,78 @@ def test_a_shared_input_name_is_computed_alike_in_every_module(self) -> None: **_AXIOM, ) + def test_bindings_that_write_one_person_column_compute_it_alike(self) -> None: + # A person_role binding writes the person table, as a binding on the + # engine's person entity does, so a name they share is one column. + role = _PERSON_WRITERS["Child"] + same = _mapping([_module_binding("a"), _module_binding("b", **role)], **_AXIOM) + assert len(same.refs()) == 2 + enrolled = _PERSON_COMPUTATIONS["enrolled"] + for other in ("Person", "Dependant"): + pair = [ + _module_binding("a", **role), + _module_binding("b", **_PERSON_WRITERS[other], **enrolled), + ] + for bindings in (pair, pair[::-1]): + with pytest.raises(ValueError, match="computed differently"): + _mapping(bindings, **_AXIOM) + + @PROPERTY + @given( + writers=st.lists( + st.tuples( + st.sampled_from(sorted(_PERSON_WRITERS)), + st.sampled_from(sorted(_PERSON_COMPUTATIONS)), + ), + min_size=1, + max_size=5, + ), + data=st.data(), + ) + def test_one_person_column_passes_in_any_order_only_when_computed_alike( + self, writers, data + ) -> None: + # Every binding writes the person table's x, each from its own module. + bindings = [ + _module_binding( + f"m{index}", + **_PERSON_WRITERS[entity], + **_PERSON_COMPUTATIONS[computation], + ) + for index, (entity, computation) in enumerate(writers) + ] + ordered = data.draw(st.permutations(bindings)) + if len({computation for _, computation in writers}) == 1: + assert len(_mapping(ordered, **_AXIOM).refs()) == len(ordered) + else: + with pytest.raises(ValueError, match="computed differently"): + _mapping(ordered, **_AXIOM) + + def test_an_entity_holds_persons_in_a_role_or_built_units(self) -> None: + role = {"engine_entity": "Child", "group_rule": GroupRule.PERSON_ROLE} + mapping = _mapping( + [ + _module_binding("a", **role), + _module_binding("b", engine_input="y", **role), + _module_binding("c", engine_input="z"), + ], + **_AXIOM, + ) + assert mapping.person_role_entities() == ("Child",) + with pytest.raises(ValueError, match="mix person_role with other group"): + _mapping( + [ + _module_binding("a", **role), + _module_binding( + "b", + engine_input="y", + engine_entity="Child", + group_rule=GroupRule.REFERENCE_MEMBER, + ), + ], + **_AXIOM, + ) + def test_entity_correspondence_covers_both_concept_entities(self) -> None: with pytest.raises(ValueError, match="entity_correspondence"): _mapping([_binding()], entity_correspondence={"person": "person"}) diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_group_encode.py b/packages/microcosm-frame/tests/engine_free/shared/test_group_encode.py index 26258e7be6..fd1acf89e3 100644 --- a/packages/microcosm-frame/tests/engine_free/shared/test_group_encode.py +++ b/packages/microcosm-frame/tests/engine_free/shared/test_group_encode.py @@ -10,6 +10,9 @@ alternative conserves it too; - ``sum_over_members`` conserves member totals; ``any_member`` and ``reference_member`` read the members and the head; +- ``person_role`` writes each person's own value on the person table through + plain ``encode``: neither encoder defers it, and adding it changes no other + binding's output; - encoding is deterministic and row-aligned to the unit table; - state bindings given as a list, iterator or generator encode, or are refused, exactly as the same bindings given as a tuple; @@ -541,13 +544,79 @@ def test_a_one_shot_iterator_of_state_bindings_takes_a_column_knob(self) -> None ) assert got.deferred == expected.deferred - def test_unexecutable_group_bindings_are_deferred(self) -> None: + def test_only_bindings_without_built_units_are_deferred(self) -> None: + # child_age takes person_role, which needs no units: plain encode + # writes it on the person table, and neither encoder defers it. tables, _, _, _, group = _example() encoded = _encode(tables, group) assert {binding.engine_input for binding in encoded.deferred} == { - "child_age", "tax_unit_income", } + plain = MAPPING.encode(tables) + assert "child_age" not in {binding.engine_input for binding in plain.deferred} + assert ( + plain.tables["person"]["child_age"].tolist() + == tables["person"]["age"].tolist() + ) + + @PROPERTY + @given(case=unit_frames()) + def test_person_role_encodes_each_persons_own_value(self, case) -> None: + # A person_role entity (Axiom's Child) has one row per person in the + # role, so each person's row carries that person's own value. Both + # transforms Belgium uses run here: an identity and a predicate. + tables, rule, _, _, group = case + others = tuple( + binding + for binding in BINDINGS + if binding.group_rule is not GroupRule.PERSON_ROLE + ) + role = ( + *(b for b in BINDINGS if b.group_rule is GroupRule.PERSON_ROLE), + _binding( + "child_in_higher_education", + ("fact:person.education_enrollment",), + Predicate( + clauses=(("fact:person.education_enrollment", ("tertiary",)),) + ), + GroupRule.PERSON_ROLE, + engine_entity="Child", + ), + ) + with_role, without = _mapping((*others, *role)), _mapping(others) + person = tables["person"] + + encoded = with_role.encode(tables) + assert encoded.tables["person"]["child_age"].tolist() == person["age"].tolist() + assert encoded.tables["person"]["child_in_higher_education"].tolist() == [ + value == "tertiary" for value in person["education_enrollment"] + ] + assert not {b.engine_input for b in encoded.deferred} & { + b.engine_input for b in role + } + + # Adding the person_role bindings changes no other binding's output. + plain = without.encode(tables) + pd.testing.assert_frame_equal( + encoded.tables["person"].drop( + columns=[binding.engine_input for binding in role] + ), + plain.tables["person"], + ) + pd.testing.assert_frame_equal( + encoded.tables["household"], plain.tables["household"] + ) + assert encoded.deferred == plain.deferred + grouped = with_role.encode_groups( + tables, {"Family": group}, state_bindings=STATE + ) + reference = without.encode_groups( + tables, {"Family": group}, state_bindings=STATE + ) + pd.testing.assert_frame_equal( + grouped.tables[rule.entity], reference.tables[rule.entity] + ) + assert grouped.deferred == reference.deferred def test_modules_filter_bindings(self) -> None: bindings = [ @@ -1025,7 +1094,8 @@ def test_a_state_binding_needs_a_membership(self) -> None: MAPPING.encode_groups(tables, {"Family": group}, state_bindings=[other]) def test_unit_composition_needs_membership_in_plain_encode(self) -> None: - # plain encode defers every group binding, unit composition included + # plain encode defers every group binding but person_role, unit + # composition included tables, _, _, _, _ = _example() deferred = {binding.engine_input for binding in MAPPING.encode(tables).deferred} assert {f"unit_{feature.value}" for feature in UnitFeature} <= deferred