From 2981316ae4b418b3a04b3e8c304876b24316fb39 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 3 Nov 2026 00:32:02 -0500 Subject: [PATCH 1/4] Encode person_role bindings as each person's own value A person_role binding feeds an engine entity with one row per person in a role, such as Axiom's Child. `encode` deferred every binding with a group rule, and `encode_groups` deferred person_role, so no step ever wrote these inputs. Belgium's 35 Child bindings (27 identities on age and 8 predicates on education enrolment, across 11 modules) had no column. `is_executable` now admits person_role. `encode` writes each such binding on the person table through its existing line, one value per person, and `encode_groups` skips it. Validation already requires a person_role binding to read person concepts. Bindings on entities without built units stay deferred. Two tests pinned the old deferral and now pin the new one. A new property checks each person's own value for an identity and a predicate, that neither encoder defers them, and that adding them changes no other binding's output. The row-by-row reference differential now counts Belgium's 35 bindings. No decoder changes in any packaged mapping, and the NZ, US and UK mappings hold no person_role binding. Co-Authored-By: Claude Opus 5.5 --- .../src/microcosm/frame/concept_mapping.py | 40 ++++++---- .../shared/test_concept_mapping.py | 7 +- .../engine_free/shared/test_group_encode.py | 76 ++++++++++++++++++- 3 files changed, 105 insertions(+), 18 deletions(-) diff --git a/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py b/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py index 1379070064..9d0d392882 100644 --- a/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py +++ b/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py @@ -13,7 +13,10 @@ bindings are invertible. Bindings on engine group entities (tax units, benefit units, SPM units, families) need unit membership that a concept frame does not carry; they declare a :class:`GroupRule`, and :meth:`encode` -reports them as deferred. :meth:`ConceptMapping.encode_groups` executes them +reports them as deferred. The exception is ``person_role``: each row of such +an entity (Axiom's ``Child``) is one person, so :meth:`encode` writes the +binding on the person table, one value per person. +:meth:`ConceptMapping.encode_groups` executes the other rules once a unit-construction step has built the units from the relationship pointers (:mod:`microcosm.frame.unit_construction` builds benefit units) and hands their :class:`GroupMembership` over. It also executes declared state @@ -130,7 +133,9 @@ class GroupRule(StrEnum): """How a binding on an engine group entity collapses member values. :meth:`ConceptMapping.encode_groups` applies every rule except - :attr:`PERSON_ROLE`, given the units' :class:`GroupMembership`. + :attr:`PERSON_ROLE`, given the units' :class:`GroupMembership`; + :meth:`ConceptMapping.encode` applies :attr:`PERSON_ROLE`, which needs no + units. """ #: Sum the members' values. @@ -1115,7 +1120,8 @@ class EncodedInputs: deferred: Bindings on engine group entities, which need engine unit membership and were not executed here; group encoding (:meth:`ConceptMapping.encode_groups`) executes them on built - units. + units. ``person_role`` bindings need no units: encoding executes + them and never defers them. """ tables: Mapping[str, pd.DataFrame] @@ -1415,7 +1421,8 @@ class EncodedGroupInputs: aligned to that membership's unit table, with the unit id column plus the engine inputs. deferred: Group bindings not executed: their engine entity has no - membership here, or they take the per-person ``person_role`` rule. + membership here. Never a ``person_role`` binding, which + :meth:`ConceptMapping.encode` executes. """ tables: Mapping[str, pd.DataFrame] @@ -1738,8 +1745,12 @@ def legal_alignments( return tuple(out.values()) def is_executable(self, binding: InputBinding) -> bool: - """Whether :meth:`encode` computes the binding (it is not deferred).""" - return binding.group_rule is None + """Whether :meth:`encode` computes the binding (it is not deferred). + + A ``person_role`` binding takes each person's own value, so it + executes on the person table like a binding without a group rule. + """ + return binding.group_rule in (None, GroupRule.PERSON_ROLE) def invertible_concepts(self) -> tuple[str, ...]: """Concepts :meth:`decode` recovers exactly from the engine inputs. @@ -1799,9 +1810,11 @@ def encode( ) -> EncodedInputs: """Compute the engine inputs a concept frame supplies. - Every executable binding whose concepts are all present in ``tables`` - is computed. Bindings on engine group entities are returned as - deferred (:meth:`encode_groups` executes them on built units). Share + It computes every executable binding whose concepts are all present + in ``tables``, ``person_role`` bindings included: each writes one + value per person on the person table. It returns other bindings on + engine group entities as deferred (:meth:`encode_groups` executes + them on built units). Share parameters and take-up rates are needed only for the bindings that use them. A take-up rate of ``None`` marks a program the caller leaves unseeded: its flag is not written, so the engine @@ -1870,8 +1883,9 @@ def encode_groups( other units, or shared per adult when ``knobs`` says so; either way each household's units sum back to its amount. - ``person_role`` bindings, and bindings on an engine entity with no - membership, are returned as deferred. Each state binding is executed + It skips ``person_role`` bindings, which :meth:`encode` executes, + and returns bindings on an engine entity with no membership as + deferred. Each state binding is executed the same way from its model-state columns. ``knobs`` then zero or scale named inputs. @@ -1943,10 +1957,10 @@ def write(entity: str, name: str, values: np.ndarray) -> None: produced[name].add(frame_entity) for binding in self.bindings: - if binding.group_rule is None or not keep(binding.module): + if self.is_executable(binding) or not keep(binding.module): continue group = units.get(binding.engine_entity) - if group is None or binding.group_rule is GroupRule.PERSON_ROLE: + if group is None: deferred.append(binding) continue if not all(context.has(concept_id) for concept_id in binding.concepts): diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py b/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py index e8b155d5ad..3ffaa8d6a3 100644 --- a/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py +++ b/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py @@ -590,7 +590,10 @@ def test_allocation_needs_the_reference_person(self) -> None: with pytest.raises(ValueError, match="reference_person_id"): _mapping([binding]).encode(tables) - def test_group_bindings_are_deferred_not_executed(self) -> None: + def test_group_bindings_other_than_person_role_are_deferred(self) -> None: + # A person_role binding takes each person's own value, so encode runs + # it (Belgium's Child inputs); every other group binding waits for + # built units. tables = { "person": pd.DataFrame({"person_id": [1], "person_household_id": [1]}), "household": pd.DataFrame({"household_id": [1]}), @@ -602,7 +605,7 @@ def test_group_bindings_are_deferred_not_executed(self) -> None: assert deferred == { binding.engine_input for binding in mapping.bindings - if binding.group_rule is not None + if binding.group_rule not in (None, GroupRule.PERSON_ROLE) } diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_group_encode.py b/packages/microcosm-frame/tests/engine_free/shared/test_group_encode.py index 26258e7be6..fd1acf89e3 100644 --- a/packages/microcosm-frame/tests/engine_free/shared/test_group_encode.py +++ b/packages/microcosm-frame/tests/engine_free/shared/test_group_encode.py @@ -10,6 +10,9 @@ alternative conserves it too; - ``sum_over_members`` conserves member totals; ``any_member`` and ``reference_member`` read the members and the head; +- ``person_role`` writes each person's own value on the person table through + plain ``encode``: neither encoder defers it, and adding it changes no other + binding's output; - encoding is deterministic and row-aligned to the unit table; - state bindings given as a list, iterator or generator encode, or are refused, exactly as the same bindings given as a tuple; @@ -541,13 +544,79 @@ def test_a_one_shot_iterator_of_state_bindings_takes_a_column_knob(self) -> None ) assert got.deferred == expected.deferred - def test_unexecutable_group_bindings_are_deferred(self) -> None: + def test_only_bindings_without_built_units_are_deferred(self) -> None: + # child_age takes person_role, which needs no units: plain encode + # writes it on the person table, and neither encoder defers it. tables, _, _, _, group = _example() encoded = _encode(tables, group) assert {binding.engine_input for binding in encoded.deferred} == { - "child_age", "tax_unit_income", } + plain = MAPPING.encode(tables) + assert "child_age" not in {binding.engine_input for binding in plain.deferred} + assert ( + plain.tables["person"]["child_age"].tolist() + == tables["person"]["age"].tolist() + ) + + @PROPERTY + @given(case=unit_frames()) + def test_person_role_encodes_each_persons_own_value(self, case) -> None: + # A person_role entity (Axiom's Child) has one row per person in the + # role, so each person's row carries that person's own value. Both + # transforms Belgium uses run here: an identity and a predicate. + tables, rule, _, _, group = case + others = tuple( + binding + for binding in BINDINGS + if binding.group_rule is not GroupRule.PERSON_ROLE + ) + role = ( + *(b for b in BINDINGS if b.group_rule is GroupRule.PERSON_ROLE), + _binding( + "child_in_higher_education", + ("fact:person.education_enrollment",), + Predicate( + clauses=(("fact:person.education_enrollment", ("tertiary",)),) + ), + GroupRule.PERSON_ROLE, + engine_entity="Child", + ), + ) + with_role, without = _mapping((*others, *role)), _mapping(others) + person = tables["person"] + + encoded = with_role.encode(tables) + assert encoded.tables["person"]["child_age"].tolist() == person["age"].tolist() + assert encoded.tables["person"]["child_in_higher_education"].tolist() == [ + value == "tertiary" for value in person["education_enrollment"] + ] + assert not {b.engine_input for b in encoded.deferred} & { + b.engine_input for b in role + } + + # Adding the person_role bindings changes no other binding's output. + plain = without.encode(tables) + pd.testing.assert_frame_equal( + encoded.tables["person"].drop( + columns=[binding.engine_input for binding in role] + ), + plain.tables["person"], + ) + pd.testing.assert_frame_equal( + encoded.tables["household"], plain.tables["household"] + ) + assert encoded.deferred == plain.deferred + grouped = with_role.encode_groups( + tables, {"Family": group}, state_bindings=STATE + ) + reference = without.encode_groups( + tables, {"Family": group}, state_bindings=STATE + ) + pd.testing.assert_frame_equal( + grouped.tables[rule.entity], reference.tables[rule.entity] + ) + assert grouped.deferred == reference.deferred def test_modules_filter_bindings(self) -> None: bindings = [ @@ -1025,7 +1094,8 @@ def test_a_state_binding_needs_a_membership(self) -> None: MAPPING.encode_groups(tables, {"Family": group}, state_bindings=[other]) def test_unit_composition_needs_membership_in_plain_encode(self) -> None: - # plain encode defers every group binding, unit composition included + # plain encode defers every group binding but person_role, unit + # composition included tables, _, _, _, _ = _example() deferred = {binding.engine_input for binding in MAPPING.encode(tables).deferred} assert {f"unit_{feature.value}" for feature in UnitFeature} <= deferred From 9128040ad84e20d9d0678e4e11df5f2e8a6b3c50 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 3 Nov 2026 01:38:04 -0500 Subject: [PATCH 2/4] Refuse bindings that compute one person column differently encode writes each person_role binding to the person table, beside the engine's Person bindings. _check_shared_names compared shared names per engine entity, so it admitted a Child binding and a Person binding, or two person_role entities, that share a name but compute it differently, and encode kept whichever came last. It now compares per table written: executable bindings per concept entity, group bindings per engine entity as before. Bindings that compute a shared column alike still pass. A test refuses an age identity and an enrolment predicate that share a person column, in both orders. A property checks that bindings writing one person column pass, in any order, exactly when they compute it alike. Co-Authored-By: Claude Opus 5.5 --- .../src/microcosm/frame/concept_mapping.py | 38 ++++++++--- .../shared/test_concept_mapping.py | 65 +++++++++++++++++++ 2 files changed, 93 insertions(+), 10 deletions(-) diff --git a/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py b/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py index 9d0d392882..2f547644dd 100644 --- a/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py +++ b/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py @@ -1557,25 +1557,43 @@ def _check_binding(self, binding: InputBinding) -> None: _check_group_rule(self.engine, binding) def _check_shared_names(self) -> None: - """One encoded column per input name: every module computes it alike. + """One encoded column per input name: every binding computes it alike. Module-scoped engines can read the same input name in several - modules. An encoded frame holds one column per name and entity, so - every binding of that name must compute it the same way. + modules. An encoded frame holds one column per name and table, so + every binding that writes the column must compute it the same way. + An executable binding writes its concept entity's table, one value + per row, so a ``person_role`` binding shares the person table with + the engine's person entity and with every other ``person_role`` + entity. A group binding writes its engine entity's units. """ - seen: dict[tuple[str, str], InputBinding] = {} + seen: dict[tuple[str, bool, str], InputBinding] = {} for binding in self.bindings: - key = (binding.engine_input, binding.engine_entity) + executable = self.is_executable(binding) + key = ( + binding.engine_input, + executable, + binding.concept_entity if executable else binding.engine_entity, + ) first = seen.setdefault(key, binding) - if (first.concepts, first.transform, first.group_rule) != ( + # Executable bindings all give each row its own value, so the + # rule is part of the computation only for group bindings. + if ( + first.concepts, + first.transform, + None if executable else first.group_rule, + ) != ( binding.concepts, binding.transform, - binding.group_rule, + None if executable else binding.group_rule, ): + entities = " and ".join( + repr(entity) + for entity in sorted({first.engine_entity, binding.engine_entity}) + ) raise ValueError( - f"{self.engine}: {binding.engine_input!r} on " - f"{binding.engine_entity!r} is computed differently in " - f"{first.module} and {binding.module}." + f"{self.engine}: {binding.engine_input!r} on {entities} is " + f"computed differently in {first.module} and {binding.module}." ) def _check_shares(self) -> None: diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py b/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py index 3ffaa8d6a3..e31e120405 100644 --- a/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py +++ b/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py @@ -926,6 +926,24 @@ def _module_binding(module: str, **change) -> InputBinding: ) +# Engine entities whose bindings write the person table, and two ways to +# compute one column there. +_PERSON_WRITERS = { + "Person": {}, + "Child": {"engine_entity": "Child", "group_rule": GroupRule.PERSON_ROLE}, + "Dependant": {"engine_entity": "Dependant", "group_rule": GroupRule.PERSON_ROLE}, +} +_PERSON_COMPUTATIONS = { + "age": {}, + "enrolled": { + "concepts": ("fact:person.education_enrollment",), + "transform": Predicate( + clauses=(("fact:person.education_enrollment", ("tertiary",)),) + ), + }, +} + + class TestConstructionRules: def test_a_minimal_mapping_constructs(self) -> None: mapping = _mapping([_binding()]) @@ -1097,6 +1115,53 @@ def test_a_shared_input_name_is_computed_alike_in_every_module(self) -> None: **_AXIOM, ) + def test_bindings_that_write_one_person_column_compute_it_alike(self) -> None: + # A person_role binding writes the person table, as a binding on the + # engine's person entity does, so a name they share is one column. + role = _PERSON_WRITERS["Child"] + same = _mapping([_module_binding("a"), _module_binding("b", **role)], **_AXIOM) + assert len(same.refs()) == 2 + enrolled = _PERSON_COMPUTATIONS["enrolled"] + for other in ("Person", "Dependant"): + pair = [ + _module_binding("a", **role), + _module_binding("b", **_PERSON_WRITERS[other], **enrolled), + ] + for bindings in (pair, pair[::-1]): + with pytest.raises(ValueError, match="computed differently"): + _mapping(bindings, **_AXIOM) + + @PROPERTY + @given( + writers=st.lists( + st.tuples( + st.sampled_from(sorted(_PERSON_WRITERS)), + st.sampled_from(sorted(_PERSON_COMPUTATIONS)), + ), + min_size=1, + max_size=5, + ), + data=st.data(), + ) + def test_one_person_column_passes_in_any_order_only_when_computed_alike( + self, writers, data + ) -> None: + # Every binding writes the person table's x, each from its own module. + bindings = [ + _module_binding( + f"m{index}", + **_PERSON_WRITERS[entity], + **_PERSON_COMPUTATIONS[computation], + ) + for index, (entity, computation) in enumerate(writers) + ] + ordered = data.draw(st.permutations(bindings)) + if len({computation for _, computation in writers}) == 1: + assert len(_mapping(ordered, **_AXIOM).refs()) == len(ordered) + else: + with pytest.raises(ValueError, match="computed differently"): + _mapping(ordered, **_AXIOM) + def test_entity_correspondence_covers_both_concept_entities(self) -> None: with pytest.raises(ValueError, match="entity_correspondence"): _mapping([_binding()], entity_correspondence={"person": "person"}) From f905863c2c9526dcc8fd3fe5050dc9d40cc73eaf Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 3 Nov 2026 01:38:04 -0500 Subject: [PATCH 3/4] Write person_role entities' closure defaults on the person table concepts.encode@1 gathered closure defaults only for the engine entities in the mapping's entity_correspondence, so a defaulted root on a person_role entity such as Axiom's Child reached no table. It now adds the whole mapping's person_role entities to the person table's, beside their bindings. It merges every entity's defaults for one table before it applies them, so two that agree share one column and two that disagree refuse. A selected path may now name a module that the closure lists and the mapping doesn't bind, since a module may default every input it reads. Validation refuses an engine entity that mixes person_role with other group rules: its rows are persons in a role or built units, not both. Tests cover a Child default that reaches the person table, agreeing and disagreeing Person and Child defaults in both orders, a Child default on an encoded column, and a module that only defaults. A property checks that, without person_role defaults, the kernel writes what the old gather wrote. Co-Authored-By: Claude Opus 5.5 --- .../build/transport/column_kernels.py | 70 ++++--- .../shared/test_transport_column_kernels.py | 172 +++++++++++++++++- .../src/microcosm/frame/concept_mapping.py | 39 ++++ .../shared/test_concept_mapping.py | 25 +++ 4 files changed, 281 insertions(+), 25 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/transport/column_kernels.py b/packages/microcosm-build/src/microcosm/build/transport/column_kernels.py index a1a482479b..bc626d9be7 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/column_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/transport/column_kernels.py @@ -6,7 +6,11 @@ country package. ``unit_rule`` is the resolved :class:`BenefitUnitRule` representation, rather than an unresolved country proposal. ``mapping`` is a :class:`ConceptMapping` representation; ``rulespec_paths`` selects its -bindings and the closure's defaults. +bindings and the closure's defaults. Each selected path names a module the +mapping binds or, with a closure, a module the closure lists, since a module +may default every input it reads. ``concepts.encode@1`` writes the defaults +of the mapping's ``person_role`` entities on the person table, beside their +bindings. The encode and scenario kernels derive no take-up draws: they refuse ``take_up_rates``, so the encoder refuses any selected take-up-threshold @@ -162,19 +166,28 @@ def _paths(context: KernelContext, ref: str) -> tuple[str, ...] | None: def _full_mapping(context: KernelContext, ref: str) -> ConceptMapping: - """The whole declared mapping, after checking its selected paths exist.""" + """The whole declared mapping, after checking its selected paths exist. + + The mapping binds an input in each selected path, or the closure lists + the path: a module may default every input it reads and bind none. + """ mapping = ConceptMapping.from_dict(_document(context, ref, "mapping")) paths = _paths(context, ref) if paths is not None: - unknown = set(paths) - {binding.module for binding in mapping.bindings} + known = {binding.module for binding in mapping.bindings} + if "closure" in context.params: + closure = InputClosure.from_dict(_document(context, ref, "closure")) + known |= set(closure.modules) + unknown = set(paths) - known if unknown: raise ValueError(f"{ref} has unknown mapping paths {sorted(unknown)}.") return mapping -def _mapping(context: KernelContext, ref: str) -> ConceptMapping: - """The mapping restricted to the selected paths, for person encoding.""" - mapping = _full_mapping(context, ref) +def _mapping( + context: KernelContext, ref: str, mapping: ConceptMapping +) -> ConceptMapping: + """``mapping`` restricted to the selected paths, for person encoding.""" paths = _paths(context, ref) if paths is None: return mapping @@ -214,20 +227,30 @@ def _defaults( ref: str, tables: Mapping[str, pd.DataFrame], closure: InputClosure | None, - engine_entities: Mapping[str, str], + engine_entities: Mapping[str, tuple[str, ...]], ) -> dict[str, pd.DataFrame]: + """``tables`` with the closure's defaults for the selected paths. + + ``engine_entities`` maps each frame entity to the engine entities whose + inputs its table holds. Their defaults merge into one set per table, so + two defaults that name one input must agree. + """ out = dict(tables) if closure is None: return out - for frame_entity, engine_entity in engine_entities.items(): + for frame_entity, entities in engine_entities.items(): defaults: dict[str, bool | int | float] = {} for path in _paths(context, ref): - for name, value in closure.defaults(path, engine_entity).items(): - if name in defaults and ( - type(defaults[name]) is not type(value) or defaults[name] != value - ): - raise ValueError(f"{ref} has conflicting defaults for {name!r}.") - defaults[name] = value + for engine_entity in entities: + for name, value in closure.defaults(path, engine_entity).items(): + if name in defaults and ( + type(defaults[name]) is not type(value) + or defaults[name] != value + ): + raise ValueError( + f"{ref} has conflicting defaults for {name!r}." + ) + defaults[name] = value out[frame_entity] = apply_defaults(out[frame_entity], defaults) return out @@ -399,17 +422,22 @@ def run(self, context: KernelContext) -> KernelResult: context, self.ref, required=_ENCODE_REQUIRED, optional=_ENCODE_OPTIONAL ) tables = _whole_tables(context, self.ref, ("person", "household")) - mapping = _mapping(context, self.ref) + full = _full_mapping(context, self.ref) + mapping = _mapping(context, self.ref, full) encoded = mapping.encode( tables, shares=_optional_resource_document(context, self.ref, "shares"), ) + # A person_role entity (Axiom's Child) has one row per person in the + # role, so its defaults go on the person table, as its bindings do. + # The whole mapping names these entities: a selected module may + # default every input of an entity that only other modules bind. + entities = { + frame: (engine,) for frame, engine in full.entity_correspondence.items() + } + entities["person"] += full.person_role_entities() outputs = _defaults( - context, - self.ref, - encoded.tables, - _closure(context, self.ref), - mapping.entity_correspondence, + context, self.ref, encoded.tables, _closure(context, self.ref), entities ) return KernelResult( columns=_columns(context, self.ref, outputs), @@ -467,7 +495,7 @@ def _group_inputs(context: KernelContext, ref: str): state_bindings=() if closure is None else closure.state_bindings(), shares=_optional_resource_document(context, ref, "shares"), ) - outputs = _defaults(context, ref, encoded.tables, closure, {rule.entity: entity}) + outputs = _defaults(context, ref, encoded.tables, closure, {rule.entity: (entity,)}) return mapping, encoded, outputs diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_column_kernels.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_column_kernels.py index 8848d99782..1d16c05c49 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_column_kernels.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_column_kernels.py @@ -287,6 +287,57 @@ def _mapping_with(bindings): ), ), ) +CHILD_PATH = "xx/child.yaml" +CHILD_DEFAULTS_PATH = "xx/child_defaults.yaml" + + +def _default(module, entity, name, value): + return ClosureEntry( + module=module, + entity=entity, + input=name, + closure_class="defaulted", + value=value, + reason="Synthetic default.", + citation="Toy fixture.", + ) + + +# A person_role entity, as Axiom's Child: one module binds an input and +# defaults another, and a second module only defaults. +ROLE_MAPPING = _mapping_with( + ( + *BINDINGS, + _binding( + "child_age", + "Child", + "fact:person.age", + group_rule="person_role", + module=CHILD_PATH, + ), + ) +) +CHILD_ENCODED = ClosureEntry( + module=CHILD_PATH, + entity="Child", + input="child_age", + closure_class="encoded", + encoded_by="concept_binding", +) +ROLE_CLOSURE = replace( + CLOSURE, + modules={ + **CLOSURE.modules, + CHILD_PATH: RESOURCE_SHA, + CHILD_DEFAULTS_PATH: RESOURCE_SHA, + }, + entries=( + *CLOSURE.entries, + CHILD_ENCODED, + _default(CHILD_PATH, "Child", "child_in_care", False), + _default(CHILD_DEFAULTS_PATH, "Child", "child_rate", 0.5), + ), +) def _rule_params(rule=UNIT_RULE): @@ -296,13 +347,15 @@ def _rule_params(rule=UNIT_RULE): } -def _encode_params(*, groups=False, knobs=None, mapping=MAPPING): +def _encode_params( + *, groups=False, knobs=None, mapping=MAPPING, closure=CLOSURE, paths=None +): params = { "mapping": canonical_text(mapping.to_dict()), "mapping_sha256": digest(mapping.to_dict()), - "closure": canonical_text(CLOSURE.to_dict()), - "closure_sha256": digest(CLOSURE.to_dict()), - "rulespec_paths": (GROUP_PATH if groups else PERSON_PATH,), + "closure": canonical_text(closure.to_dict()), + "closure_sha256": digest(closure.to_dict()), + "rulespec_paths": paths or (GROUP_PATH if groups else PERSON_PATH,), } if groups: params.update(_rule_params()) @@ -565,6 +618,117 @@ def test_person_encoding_equals_direct_mapping_and_defaults(tables): _assert_column(result, frame, "person", output.column, expected[output.column]) +@PROPERTY +@given(tables=concept_frames(max_households=3, max_members=4)) +def test_person_role_bindings_and_defaults_go_on_the_person_table(tables): + frame = _generated_frame(tables) + outputs = ( + Owned("person", "input_age", "int64"), + Owned("person", "eligibility_code", "int64"), + Owned("person", "child_age", "int64"), + Owned("person", "child_in_care", "bool"), + Owned("person", "child_rate", "float64"), + ) + params = _encode_params( + mapping=ROLE_MAPPING, + closure=ROLE_CLOSURE, + paths=(PERSON_PATH, CHILD_PATH, CHILD_DEFAULTS_PATH), + ) + result = CONCEPTS_ENCODE.run(_context(CONCEPTS_ENCODE, frame, params, outputs)) + rows = len(frame.person) + expected = { + "input_age": frame.person["age"], + "eligibility_code": pd.Series([1] * rows), + "child_age": frame.person["age"], + "child_in_care": pd.Series([False] * rows), + "child_rate": pd.Series([0.5] * rows), + } + for output in outputs: + _assert_column(result, frame, "person", output.column, expected[output.column]) + + +@PROPERTY +@given( + tables=concept_frames(max_households=3, max_members=4), + role=st.booleans(), + data=st.data(), +) +def test_without_person_role_defaults_encoding_matches_the_old_gather( + tables, role, data +): + # Before person_role entities joined the person table's entities, the + # kernel gathered defaults for the correspondence's entities only. With + # no closure root on a person_role entity, it writes the same bytes. + frame = _generated_frame(tables) + mapping = ROLE_MAPPING if role else MAPPING + closure = replace( + ROLE_CLOSURE, entries=(*CLOSURE.entries, *((CHILD_ENCODED,) if role else ())) + ) + others = sorted({binding.module for binding in mapping.bindings} - {PERSON_PATH}) + chosen = data.draw(st.lists(st.sampled_from(others), unique=True)) + paths = tuple(data.draw(st.permutations([PERSON_PATH, *chosen]))) + outputs = ( + Owned("person", "input_age", "int64"), + Owned("person", "input_income", "float64"), + Owned("person", "eligibility_code", "int64"), + *((Owned("person", "child_age", "int64"),) if CHILD_PATH in paths else ()), + ) + params = _encode_params(mapping=mapping, closure=closure, paths=paths) + result = CONCEPTS_ENCODE.run(_context(CONCEPTS_ENCODE, frame, params, outputs)) + with mock.patch.object(ConceptMapping, "person_role_entities", return_value=()): + old = CONCEPTS_ENCODE.run(_context(CONCEPTS_ENCODE, frame, params, outputs)) + assert result.columns.keys() == old.columns.keys() + for key, values in result.columns.items(): + pd.testing.assert_series_equal(values, old.columns[key], check_exact=True) + + +def test_person_role_defaults_share_the_person_table_with_person_inputs(tmp_path): + sources = write_population_sources(tmp_path / "sources") + frame = _donor_frame(sources) + rows = len(frame.person) + other = _mapping_with( + ( + *ROLE_MAPPING.bindings, + _binding("input_other", "Person", "fact:person.age", module=OTHER_PATH), + ) + ) + + def run(entries=(), paths=(PERSON_PATH, CHILD_PATH), output=None, mapping=None): + closure = replace(ROLE_CLOSURE, entries=(*ROLE_CLOSURE.entries, *entries)) + params = _encode_params( + mapping=mapping or ROLE_MAPPING, closure=closure, paths=paths + ) + owned = output or Owned("person", "child_in_care", "bool") + return CONCEPTS_ENCODE.run(_context(CONCEPTS_ENCODE, frame, params, (owned,))) + + # The selected paths' order sets the gather order. In either order, a + # Person default and a Child default that agree make one column, and + # ones that disagree refuse. + for paths in ((PERSON_PATH, CHILD_PATH), (CHILD_PATH, PERSON_PATH)): + agreed = run( + (_default(PERSON_PATH, "Person", "child_in_care", False),), paths=paths + ) + _assert_column( + agreed, frame, "person", "child_in_care", pd.Series([False] * rows) + ) + with pytest.raises(ValueError, match="conflicting defaults"): + run((_default(PERSON_PATH, "Person", "child_in_care", True),), paths=paths) + # So does a Child default on a column a binding encodes. + with pytest.raises(ValueError, match="already encoded"): + run((_default(CHILD_PATH, "Child", "input_age", 3),)) + # A selected module that binds nothing still writes its defaults: the + # whole mapping names the person_role entities. + rate = Owned("person", "child_rate", "float64") + alone = run(paths=(CHILD_DEFAULTS_PATH,), output=rate) + _assert_column(alone, frame, "person", "child_rate", pd.Series([0.5] * rows)) + # A selected path is one the mapping binds or the closure lists, and + # with a closure, one the closure lists. + with pytest.raises(ValueError, match="unknown mapping paths"): + run(paths=("xx/missing.yaml",)) + with pytest.raises(ValueError, match="closure has no paths"): + run(paths=(OTHER_PATH,), mapping=other) + + @PROPERTY @given( tables=concept_frames(max_households=3, max_members=4), diff --git a/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py b/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py index 2f547644dd..0bdda62b41 100644 --- a/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py +++ b/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py @@ -1506,6 +1506,7 @@ def __post_init__(self) -> None: ) self._check_shares() self._check_shared_names() + self._check_person_role_entities() object.__setattr__( self, "_by_concept", @@ -1596,6 +1597,28 @@ def _check_shared_names(self) -> None: f"computed differently in {first.module} and {binding.module}." ) + def _check_person_role_entities(self) -> None: + """An engine entity's rows are persons in a role or built units. + + :meth:`encode` writes a ``person_role`` binding on the person table, + and :meth:`encode_groups` writes other group bindings on built units, + so one entity can't take both. + """ + rules: dict[str, set[GroupRule]] = defaultdict(set) + for binding in self.bindings: + if binding.group_rule is not None: + rules[binding.engine_entity].add(binding.group_rule) + mixed = sorted( + entity + for entity, found in rules.items() + if GroupRule.PERSON_ROLE in found and len(found) > 1 + ) + if mixed: + raise ValueError( + f"{self.engine}: {mixed} mix person_role with other group " + "rules; an entity's rows are persons in a role or built units." + ) + def _check_shares(self) -> None: groups: dict[tuple[str, str | None], list[InputBinding]] = defaultdict(list) fractions: set[str] = set() @@ -1770,6 +1793,22 @@ def is_executable(self, binding: InputBinding) -> bool: """ return binding.group_rule in (None, GroupRule.PERSON_ROLE) + def person_role_entities(self) -> tuple[str, ...]: + """The engine entities whose bindings use ``person_role``, sorted. + + Such an entity (Axiom's ``Child``) has one row per person in the + role, so :meth:`encode` writes its inputs on the person table. + """ + return tuple( + sorted( + { + binding.engine_entity + for binding in self.bindings + if binding.group_rule is GroupRule.PERSON_ROLE + } + ) + ) + def invertible_concepts(self) -> tuple[str, ...]: """Concepts :meth:`decode` recovers exactly from the engine inputs. diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py b/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py index e31e120405..5b949d6686 100644 --- a/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py +++ b/packages/microcosm-frame/tests/engine_free/shared/test_concept_mapping.py @@ -1162,6 +1162,31 @@ def test_one_person_column_passes_in_any_order_only_when_computed_alike( with pytest.raises(ValueError, match="computed differently"): _mapping(ordered, **_AXIOM) + def test_an_entity_holds_persons_in_a_role_or_built_units(self) -> None: + role = {"engine_entity": "Child", "group_rule": GroupRule.PERSON_ROLE} + mapping = _mapping( + [ + _module_binding("a", **role), + _module_binding("b", engine_input="y", **role), + _module_binding("c", engine_input="z"), + ], + **_AXIOM, + ) + assert mapping.person_role_entities() == ("Child",) + with pytest.raises(ValueError, match="mix person_role with other group"): + _mapping( + [ + _module_binding("a", **role), + _module_binding( + "b", + engine_input="y", + engine_entity="Child", + group_rule=GroupRule.REFERENCE_MEMBER, + ), + ], + **_AXIOM, + ) + def test_entity_correspondence_covers_both_concept_entities(self) -> None: with pytest.raises(ValueError, match="entity_correspondence"): _mapping([_binding()], entity_correspondence={"person": "person"}) From 8a4e7a4719dc7a20ae4e6afa3d02a7a85312e0f6 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 3 Nov 2026 04:00:08 -0500 Subject: [PATCH 4/4] Refuse a person_role entity in the group kernels and test the Child gather Applies the four optional P3s from the S2b PR 2 review (brief 6036): - P3-1: concepts.encode@1 writes a person_role entity's bindings and defaults on the person table, but concepts.encode_groups@1 and transport.scenario_override@1 still took one as their engine_entity and wrote its closure defaults on a unit table too. _group_inputs now refuses it. test_group_kernels_refuse_a_person_role_entity checks both kernels, with Family on the same mapping as the control. - P3-2: the differential property holds no defaulted Child root, so it checks only the gather's non-Child part. test_person_and_child_defaults_merge_on_the_person_table_or_refuse draws Person and Child defaults and the gather order. Agreeing defaults share one person column; ones that differ in type or value refuse. - P3-3: the module docstring and _full_mapping now state the path rule exactly. With a closure, each selected path names a module the closure lists; without one, a module the mapping binds. - P3-4: a comment in ConceptsEncodeKernel.run records that only person_role bindings add an entity to the person table's defaults. Only column_kernels.py and its tests change, so the kernels that move are the ones that already move from main, and the US worker identity stays put. Co-Authored-By: Claude Opus 5.5 --- .../build/transport/column_kernels.py | 27 ++++-- .../shared/test_transport_column_kernels.py | 90 ++++++++++++++++++- 2 files changed, 109 insertions(+), 8 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/transport/column_kernels.py b/packages/microcosm-build/src/microcosm/build/transport/column_kernels.py index bc626d9be7..679ecfad7f 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/column_kernels.py +++ b/packages/microcosm-build/src/microcosm/build/transport/column_kernels.py @@ -6,11 +6,12 @@ country package. ``unit_rule`` is the resolved :class:`BenefitUnitRule` representation, rather than an unresolved country proposal. ``mapping`` is a :class:`ConceptMapping` representation; ``rulespec_paths`` selects its -bindings and the closure's defaults. Each selected path names a module the -mapping binds or, with a closure, a module the closure lists, since a module -may default every input it reads. ``concepts.encode@1`` writes the defaults -of the mapping's ``person_role`` entities on the person table, beside their -bindings. +bindings and the closure's defaults. With a closure, each selected path names +a module the closure lists, which may bind nothing, since a module may default +every input it reads; without one, each names a module the mapping binds. +``concepts.encode@1`` writes the defaults of the mapping's ``person_role`` +entities on the person table, beside their bindings, and the group kernels +refuse such an entity. The encode and scenario kernels derive no take-up draws: they refuse ``take_up_rates``, so the encoder refuses any selected take-up-threshold @@ -168,8 +169,10 @@ def _paths(context: KernelContext, ref: str) -> tuple[str, ...] | None: def _full_mapping(context: KernelContext, ref: str) -> ConceptMapping: """The whole declared mapping, after checking its selected paths exist. - The mapping binds an input in each selected path, or the closure lists - the path: a module may default every input it reads and bind none. + Each selected path names a module the mapping binds or the closure lists. + With a closure, :func:`_closure` also refuses a path the closure lacks, + so there each path names a module the closure lists, which may bind + nothing: a module may default every input it reads. """ mapping = ConceptMapping.from_dict(_document(context, ref, "mapping")) paths = _paths(context, ref) @@ -432,6 +435,9 @@ def run(self, context: KernelContext) -> KernelResult: # role, so its defaults go on the person table, as its bindings do. # The whole mapping names these entities: a selected module may # default every input of an entity that only other modules bind. + # Only person_role bindings add an entity here: this kernel writes no + # defaults for an engine entity outside the correspondence that no + # person_role binding names, even when the closure defaults its inputs. entities = { frame: (engine,) for frame, engine in full.entity_correspondence.items() } @@ -458,6 +464,13 @@ def _group_inputs(context: KernelContext, ref: str): # state bindings against every concept binding's input name. mapping = _full_mapping(context, ref) entity = string_param(context, ref, "engine_entity") + # concepts.encode@1 writes a person_role entity's bindings and defaults + # on the person table. Here its defaults would land on a unit table too. + if entity in mapping.person_role_entities(): + raise ValueError( + f"{ref} engine_entity {entity!r} holds persons in a role, not built " + "units; concepts.encode@1 writes its inputs on the person table." + ) declaration = _unit_declaration(context, ref) physical = context.params.get("unit_entity") if physical is None: diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_column_kernels.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_column_kernels.py index 1d16c05c49..0c0dde6bd5 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_column_kernels.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_column_kernels.py @@ -658,7 +658,9 @@ def test_without_person_role_defaults_encoding_matches_the_old_gather( ): # Before person_role entities joined the person table's entities, the # kernel gathered defaults for the correspondence's entities only. With - # no closure root on a person_role entity, it writes the same bytes. + # no closure root on a person_role entity, it writes the same bytes. This + # checks the gather's non-Child part; the merge property below checks its + # Child part. frame = _generated_frame(tables) mapping = ROLE_MAPPING if role else MAPPING closure = replace( @@ -682,6 +684,92 @@ def test_without_person_role_defaults_encoding_matches_the_old_gather( pd.testing.assert_series_equal(values, old.columns[key], check_exact=True) +# Names apart from every encoded input and fixture default, and values whose +# types differ where they compare equal (False == 0 == 0.0). +MERGE_NAMES = ("role_flag", "role_count", "role_share") +MERGE_VALUES = st.sampled_from([False, True, 0, 1, 0.0, 0.5]) +DTYPE_TOKENS = {bool: "bool", int: "int64", float: "float64"} + + +@PROPERTY +@given( + tables=concept_frames(max_households=2, max_members=3), + person=st.dictionaries(st.sampled_from(MERGE_NAMES), MERGE_VALUES, max_size=3), + child=st.dictionaries( + st.sampled_from(MERGE_NAMES), MERGE_VALUES, min_size=1, max_size=3 + ), + child_first=st.booleans(), +) +def test_person_and_child_defaults_merge_on_the_person_table_or_refuse( + tables, person, child, child_first +): + # The person table's defaults merge into one set, in either gather + # order: a name that Person and Child both default must agree in type + # and value, or the kernel refuses. Otherwise each name gets one person + # column holding its default. + frame = _generated_frame(tables) + closure = replace( + ROLE_CLOSURE, + entries=( + *ROLE_CLOSURE.entries, + *(_default(PERSON_PATH, "Person", k, v) for k, v in person.items()), + *(_default(CHILD_DEFAULTS_PATH, "Child", k, v) for k, v in child.items()), + ), + ) + # The selected paths' order sets the gather order. + paths = (PERSON_PATH, CHILD_DEFAULTS_PATH)[:: -1 if child_first else 1] + merged = {**child, **person} + outputs = tuple( + Owned("person", name, DTYPE_TOKENS[type(value)]) + for name, value in sorted(merged.items()) + ) + params = _encode_params(mapping=ROLE_MAPPING, closure=closure, paths=paths) + context = _context(CONCEPTS_ENCODE, frame, params, outputs) + clash = any( + name in person + and (type(person[name]) is not type(value) or person[name] != value) + for name, value in child.items() + ) + if clash: + with pytest.raises(ValueError, match="conflicting defaults"): + CONCEPTS_ENCODE.run(context) + return + result = CONCEPTS_ENCODE.run(context) + for name, value in merged.items(): + expected = pd.Series([value] * len(frame.person)) + _assert_column(result, frame, "person", name, expected) + + +@PROPERTY +@given(tables=concept_frames(max_households=2, max_members=3)) +def test_group_kernels_refuse_a_person_role_entity(tables): + # concepts.encode@1 writes a person_role entity's bindings and defaults + # on the person table, so a group kernel refuses one as its engine + # entity instead of writing its defaults on a unit table as well. + frame = _generated_frame(tables) + knobs = { + "rent_allocation": "reference_unit", + "asset_test": True, + "rent_factor": 1.0, + } + params = _encode_params( + groups=True, + knobs=knobs, + mapping=ROLE_MAPPING, + closure=ROLE_CLOSURE, + paths=(GROUP_PATH, CHILD_PATH), + ) + family = (Owned("family", "input_assets", "float64", rewrite=True),) + child = {**params, "engine_entity": "Child", "rulespec_paths": (CHILD_PATH,)} + in_care = (Owned("family", "child_in_care", "bool", rewrite=True),) + for kernel in (CONCEPTS_ENCODE_GROUPS, TRANSPORT_SCENARIO_OVERRIDE): + # The same mapping still encodes its built units. + result = kernel.run(_context(kernel, frame, params, family)) + assert list(result.columns) == [("family", "input_assets")] + with pytest.raises(ValueError, match="'Child' holds persons in a role"): + kernel.run(_context(kernel, frame, child, in_care)) + + def test_person_role_defaults_share_the_person_table_with_person_inputs(tmp_path): sources = write_population_sources(tmp_path / "sources") frame = _donor_frame(sources)