diff --git a/.zed/rules b/.zed/rules index d6b8aba9d..de110c088 100644 --- a/.zed/rules +++ b/.zed/rules @@ -370,6 +370,16 @@ FunctionNode. Operators: inspect tags (never data content), can rename columns, cannot synthesize values. Function Pods: inspect data content (never tags), synthesize new values, cached by content. +No data element comes from a tag value (developer, November 11, 2026, ITL-955). An operator +may use tags (to join, filter, group or order), but tag content never becomes data: every +element of an output data column came from an input data column, and an operator never turns a +tag column into a data column. When adding or changing an operator, check keys() of the output: +no input tag name may appear among the data columns. + +GroupBy is the only operator that reduces row count many→one: it collapses N rows sharing a +tag tuple into one row with list-valued members. It synthesizes no new data values and keys +only on tags, never on data. Its non-key tags stay tags, list-valued, as Batch keeps them. + ### Two identity chains Every pipeline element has two parallel hashes: @@ -409,7 +419,9 @@ Prefixes are computed from SystemConstant in system_constants.py. 2. Name-extending — multi-input ops. System tag column name gets ::{pipeline_hash}:{canonical_position} appended. Commutative operators sort by pipeline_hash and sort system tag values per row. -3. Reducing — many→one ops (Batch, GroupBy). User tag and data columns become list[T], and +3. Reducing — many→one ops (Batch, GroupBy). User tag and data columns become list[T] and + stay in their own group (a tag stays a tag; only the scalar keys of a GroupBy are not + list-wrapped), and source-info columns become list[str] with one element per member. System tag columns must stay scalar, because _build_record_id_preimage (core/nodes/function_node.py) hashes them directly to derive record identity. They fold to a deterministic digest via @@ -512,6 +524,10 @@ data. The rules: Batch — group before batching, not after). Members sort by non-key tag values with system record_id as tiebreaker; groups emit in key order, so reordered input yields a byte-identical table. +- GroupBy's output tag columns are its input's tag columns, in input order: keys scalar, + every other tag list[T] (ITL-955). A grouped stream can be grouped again on a coarser key; + its list-valued tags then nest (list[list[T]]) and still order the members, so the member + sort key (_null_last_key) is recursive and sorts nulls last at every depth. - Data source-info values may be str, None, or list[...], nested to any depth; Arrow/Python types are derived from the value, with None mapping to large_string. That rule has one definition, arrow_utils.source_info_arrow_type, and applies only to tokens supplied as diff --git a/CLAUDE.md b/CLAUDE.md index 1bac378de..90a01c8ed 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -390,9 +390,16 @@ FunctionNode. | Synthesizes new values | No | Yes | | Stream arity | Configurable | Single in, single out | +**No data element comes from a tag value** (developer, November 11, 2026, ITL-955). An operator +may *use* tags (to join, filter, group or order), but tag content never becomes data: every +element of an output data column came from an input data column, and an operator never turns a +tag column into a data column. When adding or changing an operator, check `keys()` of the output: +no input tag name may appear among the data columns. + `GroupBy` is the only operator that reduces row count many→one: it collapses N rows sharing a tag tuple into one row with list-valued members. It still synthesizes no new data values — -every emitted element came from an input row — and it keys only on tags, never on data. +every emitted element came from an input row — and it keys only on tags, never on data. Its +non-key tags stay tags, list-valued, as `Batch` keeps them. ### Two identity chains @@ -441,7 +448,9 @@ Prefixes are computed from `SystemConstant` in `system_constants.py`. The `const name gets `::{pipeline_hash}:{canonical_position}` appended. Commutative operators canonically order inputs by `pipeline_hash` and sort system tag values per row. 3. **Reducing** — many→one ops (`Batch`, `GroupBy`). User tag and data columns become - `list[T]`, and source-info columns become `list[str]` with one element per member. System + `list[T]` and stay in their own group (a tag stays a tag; only the scalar keys of a + `GroupBy` are not list-wrapped), and source-info columns become `list[str]` with one element + per member. System tag columns must stay **scalar**, because `_build_record_id_preimage` (`core/nodes/function_node.py`) hashes them directly to derive record identity. They fold to a deterministic digest via `arrow_utils.fold_system_tag_values` and their column name @@ -560,6 +569,10 @@ data. The rules: tag values with the system `record_id` as tiebreaker, so the hashed lists are stable across runs. Groups themselves are emitted in key order, so a reordered input yields a byte-identical table. +- `GroupBy`'s output tag columns are its input's tag columns, in input order: keys scalar, + every other tag `list[T]` (ITL-955). A grouped stream can be grouped again on a coarser key; + its list-valued tags then nest (`list[list[T]]`) and still order the members, so the member + sort key (`_null_last_key`) is recursive and sorts nulls last at every depth. - `Data` source-info values may be `str`, `None`, or `list[...]`, nested to any depth; the Arrow and Python types are derived from the value, with `None` mapping to `large_string`. That rule has one definition, `arrow_utils.source_info_arrow_type`, and applies only to diff --git a/changelog.d/ITL-955.md b/changelog.d/ITL-955.md new file mode 100644 index 000000000..2e807e570 --- /dev/null +++ b/changelog.d/ITL-955.md @@ -0,0 +1,32 @@ +### Breaking changes and migration + +#### `GroupBy` keeps non-key tags as tags (ITL-955) + +`GroupBy` used to turn every tag that was not a grouping key into a list-valued **data** column, +with null source info, where a function pod could read it. That broke the rule that separates an +operator from a function pod: no data element may come from a tag value. Non-key tags now stay +in the tag group as list-valued tags, in the input's tag order, the way `Batch` keeps its tags. + +| `GroupBy(by=["subject"])` on tags `subject, session, probe`, data `path` | Before | Now | +|---|---|---| +| Tag columns | `subject` | `subject`, `session: list[int]`, `probe: list[str]` | +| Data columns | `session: list[int]`, `probe: list[str]`, `path: list[str]` | `path: list[str]` | + +- **What to change.** A function that took a moved tag as a parameter + (`def f(probe: list[int], path: list[str])`) now fails input-schema validation; drop the + parameter. A function cannot read a tag: if it needs the value, make it a data column of the + source. A `select_data_columns` that was only there to drop the moved tags can go. To remove + the list-valued tags from the output, use `drop_tag_columns`. +- **Unchanged.** `by` must still name scalar tag columns; members are still ordered by their + non-key tag values, then by record id; groups are emitted in key order. The Arrow types of the + list columns, the folded system tags and the data columns' list-valued source info are the + same as before. The two all-null `_source_` columns are gone. +- **New.** A grouped stream can be grouped again on a coarser key; its list-valued tags nest + (`list[list[T]]`). +- **Hashes and stored results.** No pipeline hash and no table path moves: the `GroupBy` + stream's `pipeline_hash` and `content_hash`, and the `pipeline_hash` of everything downstream, + are the same. Each grouped row's data hash changes, because the row's data no longer includes + the moved tags. A function whose input was already only the real data columns + (`GroupBy` then `select_data_columns`) receives the same data as before and reuses its stored + results without computing again; its output now also carries the list-valued tags. A function + whose signature has to change is a different function and computes once. diff --git a/design_issues/ITL-955-GB1.md b/design_issues/ITL-955-GB1.md new file mode 100644 index 000000000..38d721187 --- /dev/null +++ b/design_issues/ITL-955-GB1.md @@ -0,0 +1,28 @@ +## `src/orcapod/core/operators/group_by.py` — tags entering the data group (ITL-955) + +### GB1 — `GroupBy` moved non-key tag values into data columns +**Status:** resolved +**Severity:** high +**Issue:** ITL-955 + +`GroupBy(by=[...])` built the right table (keys scalar, every other column list-valued) but +declared only the keys as the output's tag columns, so each non-key tag became a list-valued +**data** column with an all-null `_source_` column. A function pod could then read tag +content, which breaks the operator / function pod boundary. `Batch` kept its tags as tags. + +The rule (developer, November 11, 2026): no data element may come from a tag value. An operator +may use tags, but tag content never becomes data. + +**Fix:** the output's tag columns are the input's tag columns, in input order: keys scalar, +every other tag `list[T]`; `unary_output_schema` predicts the same. A grouped stream can now be +grouped again on a coarser key with list-valued non-key tags, so the member sort key +(`_null_last_key`) is recursive and sorts nulls last at every depth; for scalar tags the order is +unchanged. No pipeline hash or table path moves; each grouped row's data hash does (six pinned +`pipeline:group_by.row[...]` golden entries replaced, approved). Results stored by a pod that +followed `GroupBy` through `select_data_columns` are reused without recomputation +(`tests/test_pipeline/test_aggregation_job.py::TestStoreWrittenBeforeTagsStayedTags`). + +Audit of the other operators against the rule (November 11, 2026): `Batch`, `Join`, `MergeJoin`, +`SemiJoin`, `SelectTagColumns`, `DropTagColumns`, `SelectDataColumns`, `DropDataColumns`, +`MapTags`, `MapData`, `PolarsFilter`, `Index` and `Pick` never put an input tag into the data +group. diff --git a/docs/concepts/operators.md b/docs/concepts/operators.md index 6dc1f9487..170cefc88 100644 --- a/docs/concepts/operators.md +++ b/docs/concepts/operators.md @@ -28,6 +28,10 @@ a tag tuple into a single row with list-valued members. It stays on the operator boundary because it synthesizes no new data values -- every element of every emitted list came from an input row -- and it keys only on tags, never on data content. +One rule holds for every operator: **no data element comes from a tag value**. An operator may +use tags to join, filter, group or order rows, but it never turns a tag column into a data +column, so a function pod never receives tag content. + ## Operator categories ### `UnaryOperator` -- single input @@ -209,20 +213,36 @@ source = DictSource( tag_columns=["subject_id", "probe"], ) -grouped = GroupBy(by=["subject_id"]).process(source) +grouped = GroupBy(by=["subject_id"])(source) +print(grouped.keys()) +# (('subject_id', 'probe'), ('spike_count',)) for tag, data in grouped.iter_data(): print("Tags:", tag.as_dict(), "Data:", data.as_dict()) - # Tags: {'subject_id': 'mouse_01'} Data: {'probe': ['imec0', 'imec1'], 'spike_count': [1200, 980]} - # Tags: {'subject_id': 'mouse_02'} Data: {'probe': ['imec0'], 'spike_count': [1450]} + # Tags: {'subject_id': 'mouse_01', 'probe': ['imec0', 'imec1']} Data: {'spike_count': [1200, 980]} + # Tags: {'subject_id': 'mouse_02', 'probe': ['imec0']} Data: {'spike_count': [1450]} ``` The output schema follows three rules: -- **Group keys stay scalar** and remain the output's tag columns (`subject_id: str`). -- **Non-key tag columns are promoted to list-valued data columns** (`probe: list[str]`), so a - consumer can tell which member each list element came from. +- **Group keys stay scalar tag columns** (`subject_id: str`). +- **Non-key tag columns stay tag columns and become list-valued** (`probe: list[str]`), one + element per member, in the same order as the data lists. The output's tag columns are the + input's tag columns, in the same order. This is what `Batch` does with its tags too. - **Data columns become `list[T]`** (`spike_count: list[int]`). +The data columns of the output are exactly the data columns of the input. A function pod that +follows `GroupBy` receives `spike_count` only; it cannot read `probe`, because a pod never sees +tags. If the function needs to know which member each element came from, make that value a data +column of the source. To remove the list-valued tags, use `drop_tag_columns`: + +```python +print(grouped.drop_tag_columns(["probe"]).keys()) +# (('subject_id',), ('spike_count',)) +``` + +A grouped stream can be grouped again on a coarser key. Its list-valued tags then nest one level +deeper (`list[list[T]]`), like the data lists. + Ordering is deterministic by construction, because orcapod hashes the emitted lists to build cache keys: members are sorted by their non-key tag values (with the internal record id as a final tiebreaker), and groups are emitted in group-key order. A reordered input therefore @@ -230,8 +250,8 @@ produces a byte-identical output table. Every name in `by` must be a **scalar tag column** of the input. `GroupBy` raises `InputValidationError` for an unknown column, a data column, or a list-valued tag column -- -the last usually means the stream came from `Batch`, so group *before* batching rather than -after. +the last means the stream came from `Batch` or from an earlier `GroupBy` that did not key on +that column, so group *before* batching rather than after. Streams also expose this as a fluent method: diff --git a/src/orcapod/core/operators/group_by.py b/src/orcapod/core/operators/group_by.py index 31dc066ba..cbe21cbc7 100644 --- a/src/orcapod/core/operators/group_by.py +++ b/src/orcapod/core/operators/group_by.py @@ -20,6 +20,27 @@ pa = LazyModule("pyarrow") +def _null_last_key(value: Any) -> tuple[Any, ...]: + """Build an order key that sorts nulls last, at any list depth. + + The leading integer keeps a null comparable against a real value of any + type. A list is keyed element by element, so a list-valued tag that holds + a null (the non-key tag of an already grouped stream) still compares. + + Args: + value: A tag or system-tag value: a scalar, ``None``, or a list of + these nested to any depth. + + Returns: + A tuple that orders values ascending, with nulls after every value. + """ + if value is None: + return (1,) + if isinstance(value, list): + return (0, tuple(_null_last_key(v) for v in value)) + return (0, value) + + class GroupBy(UnaryOperator): """Reduce rows sharing a tag tuple into one packet with list-valued members. @@ -31,17 +52,22 @@ class GroupBy(UnaryOperator): Given tags ``(subject, date, probe)`` and data ``(path)``, grouping by ``["subject", "date"]`` emits one row per distinct ``(subject, date)``: - * ``subject`` and ``date`` stay scalar and remain the output's tag columns - * ``probe`` becomes a list-valued **data** column, so a consumer can tell - which member each list element came from + * ``subject`` and ``date`` stay scalar tag columns + * ``probe`` stays a **tag** column and becomes list-valued, one element + per member, exactly as ``Batch`` keeps its tags. The tag columns of the + output are the tag columns of the input, in the same order. * ``path`` becomes list-valued - * the ``_source_*`` column of each original data column becomes - list-valued, one element per member. A promoted tag column such as - ``probe`` had no provenance token to begin with, so its ``_source_probe`` - is the scalar null the stream fills in, not a list. + * the ``_source_*`` column of each data column becomes list-valued, one + element per member * system-tag columns fold to a scalar digest and gain a ``::{pipeline_hash}`` name suffix + No data element ever comes from a tag value: ``GroupBy`` uses tags to form + groups, but the data columns of the output are exactly the data columns of + the input. A function pod downstream therefore receives only ``path``; if + it needs to know which member each element came from, that value has to + be a data column of the input. + Members are sorted by their non-group-key tag values, with ``record_id`` appended as a final tiebreaker, so the emitted lists are stable across runs. This matters because orcapod hashes those lists to build the cache @@ -56,8 +82,12 @@ class GroupBy(UnaryOperator): is materialized -- so their order is stable for a given source table, but undefined if the source table itself is permuted. + A grouped stream can be grouped again on a coarser key: its list-valued + tags nest one level deeper (``list[list[T]]``), as source info does. A + list-valued tag can order members but cannot be a grouping key. + Contrast with ``Batch``, which partitions by row count for throughput and - keeps its tag columns as list-valued tags. + makes every tag column list-valued, having no scalar keys. Args: by: Tag column names to group on. Must be non-empty, free of @@ -144,7 +174,9 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: stream: The upstream stream. Returns: - A stream with one row per distinct group-key tuple. + A stream with one row per distinct group-key tuple, whose tag + columns are the input's tag columns: group keys scalar, every + other tag list-valued. """ table = stream.as_table(columns={"source": True, "system_tags": True}) tag_columns, _ = stream.keys() @@ -188,13 +220,11 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: # Emit groups in key order rather than first-appearance order, so a # reordered input produces a byte-identical output table. for key, members in sorted( - groups.items(), key=lambda kv: tuple((v is None, v) for v in kv[0]) + groups.items(), key=lambda kv: tuple(_null_last_key(v) for v in kv[0]) ): if sort_columns: - # The leading bool keeps a null comparable against a real - # value of any type, and sorts nulls last. members.sort( - key=lambda r: tuple((r[c] is None, r[c]) for c in sort_columns) + key=lambda r: tuple(_null_last_key(r[c]) for c in sort_columns) ) grouped_rows.append( @@ -222,9 +252,12 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: grouped_table, stream.pipeline_hash().to_hex(n_char) ) + # Every input tag stays a tag (ITL-955): the non-key ones are the + # list-valued member columns built above. Declaring only `self.by` + # here would let them fall into the data group. return ArrowTableStream( grouped_table, - tag_columns=self.by, + tag_columns=tag_columns, data_context=stream.data_context, ) @@ -247,30 +280,25 @@ def unary_output_schema( all_info: Include all info columns. Returns: - A ``(tag_schema, data_schema)`` tuple. Group keys stay scalar in - the tag schema; promoted non-key tags and list-wrapped data columns - land in the data schema. + A ``(tag_schema, data_schema)`` tuple. The tag schema holds the + input's tags in input order: group keys scalar, every other user + tag ``list[T]``, system tags scalar under their suffixed names. + The data schema holds the input's data columns as ``list[T]``. """ tag_types, data_types = stream.output_schema(columns=columns, all_info=all_info) n_char = self.orcapod_config.hashing.system_tag_n_char suffix = stream.pipeline_hash().to_hex(n_char) out_tag_types: dict[str, Any] = {} - out_data_types: dict[str, Any] = {} - for name, col_type in tag_types.items(): if name.startswith(constants.SYSTEM_TAG_PREFIX): out_tag_types[f"{name}{constants.BLOCK_SEPARATOR}{suffix}"] = col_type elif name in self.by: out_tag_types[name] = col_type else: - # Promoted to a list-valued data column. No `_source_*` entry - # is emitted: `ArrowTableStream.output_schema` returns the data - # schema unconditionally, so `columns.source` is a no-op at the - # schema level even though `as_table` does add those columns. - out_data_types[name] = list[col_type] - - for name, col_type in data_types.items(): - out_data_types[name] = list[col_type] + # A non-key tag stays a tag, list-valued (ITL-955). + out_tag_types[name] = list[col_type] + + out_data_types = {name: list[t] for name, t in data_types.items()} return Schema(out_tag_types), Schema(out_data_types) diff --git a/src/orcapod/core/streams/base.py b/src/orcapod/core/streams/base.py index 2629401e6..37cc2347a 100644 --- a/src/orcapod/core/streams/base.py +++ b/src/orcapod/core/streams/base.py @@ -156,7 +156,8 @@ def group_by( """Reduce rows sharing a tag tuple into one packet per group. Group-key columns stay scalar tags; every other column becomes - list-valued. See ``orcapod.core.operators.GroupBy``. + list-valued and stays in its group: a non-key tag stays a tag, a data + column stays data. See ``orcapod.core.operators.GroupBy``. Args: by: Tag column names to group on. diff --git a/superpowers/plans/2026-11-11-itl-955-groupby-tags-stay-tags.md b/superpowers/plans/2026-11-11-itl-955-groupby-tags-stay-tags.md new file mode 100644 index 000000000..769021e84 --- /dev/null +++ b/superpowers/plans/2026-11-11-itl-955-groupby-tags-stay-tags.md @@ -0,0 +1,233 @@ +# ITL-955 GroupBy Keeps Non-Key Tags As Tags — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:executing-plans to implement this plan +> task-by-task (inline; the machine is memory-limited, so no helper agents run test suites). +> Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** After `GroupBy`, no data column holds tag values: non-key tags stay in the tag group as +list-valued tag columns, exactly as `Batch` keeps them. + +**Architecture:** `GroupBy.unary_static_process` already list-wraps every non-key column and builds +the right table; the bug is that it declares only `self.by` as the output's tag columns, so the +list-wrapped non-key tags fall into the data group (and gain scalar-null `_source_*` columns). +The fix declares all input tag columns as tags, mirrors that in `unary_output_schema`, and makes +the member sort key null-safe for list-valued tags (now reachable through `GroupBy` after +`GroupBy`). + +**Tech Stack:** Python, PyArrow, pytest, `uv run`. + +**Linear:** ITL-955. Branch `eywalker/itl-955-groupby-moves-non-key-tag-values-into-data`, PR +against `main`, "Fixes ITL-955". + +--- + +## Findings (measured November 11, 2026, scratch probes, public interface) + +Source: tags `(subject, session, probe)`, data `(path)`; `GroupBy(by=["subject"])`. + +| | Before | After (prototype) | +|---|---|---| +| `keys()` | `(subject)`, `(session, probe, path)` | `(subject, session, probe)`, `(path)` | +| tag schema | `subject: str` | `subject: str, session: list[int], probe: list[str]` | +| data schema | `session: list[int], probe: list[str], path: list[str]` | `path: list[str]` | +| table columns | same list columns, plus `_source_session`, `_source_probe` (scalar null) | same list columns; those two `_source_*` columns are gone | + +- Arrow types of the list columns are identical before and after (`large_list`, non-null). +- Predicted schema equals `static_process(...).output_schema(...)` for `{}`, `system_tags`, + `source + system_tags`. Zero-row output has the same types. Async rows equal sync rows + (`GroupBy` has no `async_execute` of its own; the barrier path calls the same code). +- System tags (scalar fold, `::{pipeline_hash}` suffix) and the data columns' `_source_*` lists + are unchanged. +- **Hashes.** The `GroupBy` stream's `pipeline_hash` and `content_hash` do NOT move (neither + commits to the output schema), and neither does a downstream pod's `pipeline_hash`, so no + database path moves. What moves: each grouped row's data content hash (fewer data columns) and + its tag dict. Golden `pipeline.json`: `group_by.content_hash`, `group_by.pipeline_hash`, + `group_by.system_tag_columns` unchanged; the six + `group_by.row[{"cohort":…}].{_tag_record_id…, _tag_source_id…, data_hash}` entries are replaced + by six keyed `row[{"cohort":"a","subject":["s1","s2"]}]` / `row[{"cohort":"b","subject":["s3"]}]` + (system-tag values identical, `data_hash` values new). +- **Stored rows.** `source → GroupBy → select_data_columns(["path"]) → pod` run against a Delta + store before the fix, then rerun after it on the same store: zero recomputation, cached results + reused, the pod's output now carries the list-valued tags. A pod that took a promoted tag as a + parameter (`def f(probe: list[int], path: list[str])`) now fails input-schema validation and + must drop the parameter: that is the breaking change. +- **Downstream operators on list-valued tags** (all work): `Join`/`SemiJoin`/`MergeJoin` with a + stream sharing only the scalar key; `Join` of two grouped streams sharing list-valued tags + (joins on list equality, as two `Batch` outputs already do); `Batch` after `GroupBy` and + `GroupBy` after `GroupBy` nest the list tags (`list[list[T]]`), as source info does; + `select_tag_columns` / `drop_tag_columns` remove them. `GroupBy(by=[])` still raises. +- **ITL-942** (stacked on ITL-910) changes `arrow_utils.fold_system_tag_values` and one assertion + in `tests/test_core/operators/test_group_by.py::test_falls_back_to_record_id_when_key_covers_all_tags`; + it does not touch `group_by.py`. No code overlap; its golden `group_by` row entries will need a + merge on whichever lands second. +- **Audit** (rule: no data element comes from a tag value). Clean: `Batch`, `Join`, `MergeJoin`, + `SemiJoin`, `SelectTagColumns`, `DropTagColumns`, `SelectDataColumns`, `DropDataColumns`, + `MapTags`, `MapData`, `PolarsFilter`, `Index`, `Pick`. Only `GroupBy` leaks. Out-of-scope + observations, reported not fixed: (1) `Join` / `MergeJoin` where a name is a tag on one side + and data on the other raise a bare `KeyError: Field "t" exists 2 times in schema`, not + `InputValidationError`; (2) `MapData(name_map={"": ...})` is a silent no-op. + +## Decisions (approved by the coordinator, November 11, 2026) + +1. Non-key tags become list-valued **tag** columns, in input tag order (as `Batch`). +2. `by` must still name scalar tag columns; members still sort by non-key tag values, then every + `record_id`. +3. The sort key becomes recursive so a list-valued non-key tag containing nulls compares. + +## File map + +- `src/orcapod/core/operators/group_by.py` — the fix, docstrings. +- `src/orcapod/core/streams/base.py` — `group_by` docstring. +- `tests/test_core/operators/test_group_by.py` — regression tests, updated shape tests. +- `tests/test_core/operators/test_async_sync_equivalence.py`, `test_chained_aggregation.py`, + `test_aggregated_list_types.py`, `test_materialize_source_info.py`, `test_empty_inputs.py`, + `tests/test_pipeline/test_aggregation_job.py`, `tests/test_arrays/test_array_operators.py`, + `tests/test_core/datagrams/test_data_source_info_types.py` — update where they assert promotion. +- `tests/test_hashing/golden/data/pipeline.json` — six row entries replaced. +- `docs/concepts/operators.md`, `CLAUDE.md`, `.zed/rules`, `CHANGELOG.md`, `DESIGN_ISSUES.md`. + +--- + +### Task 1: Failing regression tests + +**Files:** Modify `tests/test_core/operators/test_group_by.py` + +- [ ] **Step 1:** Replace `test_non_key_tags_promoted_to_list_data` and add the class below. + +```python +class TestGroupByTagsStayTags: + """ITL-955: no data element may come from a tag value.""" + + def test_non_key_tags_stay_list_valued_tags(self, session_source): + out = GroupBy(by=["subject", "date"])(session_source) + tag_cols, data_cols = out.keys() + assert tag_cols == ("subject", "date", "probe") + assert data_cols == ("path",) + assert out.as_table().column("probe").to_pylist() == [[0, 1], [0, 1]] + + def test_no_data_column_is_an_input_tag(self, session_source): + in_tags, in_data = session_source.keys() + out = GroupBy(by=["subject"])(session_source) + assert set(out.keys()[1]) == set(in_data) + assert not set(out.keys()[1]) & set(in_tags) + + def test_non_key_tag_schema_is_list_tag(self, session_source): + tag_schema, data_schema = GroupBy(by=["subject", "date"])( + session_source + ).output_schema() + assert "probe" in tag_schema and "probe" not in data_schema + assert get_origin(tag_schema["probe"]) is list + + def test_non_key_tags_get_no_source_info_column(self, session_source): + out = GroupBy(by=["subject", "date"])(session_source) + names = out.as_table(columns={"source": True}).column_names + assert f"{constants.SOURCE_PREFIX}probe" not in names + assert f"{constants.SOURCE_PREFIX}path" in names + + def test_datagrams_carry_non_key_tags_on_the_tag(self, session_source): + out = GroupBy(by=["subject", "date"])(session_source) + for tag, data in out.iter_data(): + assert tag.as_dict()["probe"] == [0, 1] + assert "probe" not in data.as_dict() + + def test_regroup_nests_list_tags(self, session_source): + out = GroupBy(by=["subject"])(GroupBy(by=["subject", "date"])(session_source)) + assert out.keys() == (("subject", "date", "probe"), ("path",)) + row = out.as_table().to_pylist()[0] + assert row["date"] == ["d1", "d2"] + assert row["probe"] == [[0, 1], [0, 1]] + + def test_regroup_orders_list_tags_containing_nulls(self): + table = pa.table( + { + "a": ["x", "x", "x", "x"], + "b": [1, 1, 2, 2], + "c": pa.array([None, 1, 2, 1], pa.int64()), + "v": [1, 2, 3, 4], + } + ) + src = ArrowTableSource(table, tag_columns=["a", "b", "c"], infer_nullable=True) + out = GroupBy(by=["a"])(GroupBy(by=["a", "b"])(src)) + row = out.as_table().to_pylist()[0] + assert row["c"] == [[1, None], [1, 2]] + assert row["v"] == [[2, 1], [4, 3]] +``` + +(Exact names of `constants` attributes and the fixture are checked against the file when +writing; `b` order `[1, 2]` puts `[1, None]` first.) + +- [ ] **Step 2:** `uv run pytest tests/test_core/operators/test_group_by.py -q` — expect the new + tests to FAIL (`probe` in data columns). Record the output in the PR. +- [ ] **Step 3:** Commit `test(group_by): non-key tags must stay tags (ITL-955)`. + +### Task 2: The fix + +**Files:** Modify `src/orcapod/core/operators/group_by.py` + +- [ ] **Step 1:** In `unary_static_process`, pass `tag_columns=tag_columns` to `ArrowTableStream` + (all input user tags, input order). Replace the two inline sort lambdas with a module-level + recursive key: + +```python +def _null_last_key(value: Any) -> tuple[Any, ...]: + """Order key that sorts nulls last, at any list depth.""" + if value is None: + return (1,) + if isinstance(value, list): + return (0, tuple(_null_last_key(v) for v in value)) + return (0, value) +``` + + Check first that this gives the same order as `(v is None, v)` for scalars (it does: the + leading element separates nulls, the second compares values) so no existing order moves. + +- [ ] **Step 2:** In `unary_output_schema`, a non-key user tag goes to + `out_tag_types[name] = list[col_type]`; iterate so the tag order equals the input's. +- [ ] **Step 3:** Rewrite the class and method docstrings (no "promoted"). +- [ ] **Step 4:** `uv run pytest tests/test_core/operators/test_group_by.py -q` — PASS. +- [ ] **Step 5:** Commit `fix(group_by): keep non-key tags as list-valued tags (ITL-955)`. + +### Task 3: Tests that asserted the old shape + +- [ ] `uv run pytest tests/test_core tests/test_pipeline tests/test_arrays -q -x` (one process), + fix each failure by asserting the corrected shape; pods in `test_aggregation_job.py` lose the + `probe` parameter. Extend the `GroupBy` case in `test_async_sync_equivalence.py` to a source + with a non-key tag if it has none. If a public-call test changes outcome for a reason other + than the tag/data move, stop and ask. +- [ ] Commit `test: update GroupBy expectations for tags staying tags (ITL-955)`. + +### Task 4: Golden entries + +- [ ] `uv run pytest tests/test_hashing/golden -q` — expect only `pipeline.group_by` row entries + to differ. +- [ ] `uv run python tests/test_hashing/golden/generate_golden_hashes.py`; `git diff --stat` + must show only `pipeline.json`, and the diff only the six `group_by.row[...]` lines removed and + six added. List them in the PR body. +- [ ] Commit `test(golden): regenerate group_by row entries (ITL-955)`. + +### Task 5: Docs + +- [ ] `docs/concepts/operators.md` GroupBy section: executed example showing `keys()` and the + list-valued tag; `uv run pytest tests/test_docs -q`. +- [ ] `CLAUDE.md` and `.zed/rules` together: boundary paragraph, reducing rule, GroupBy bullet, + and the rule "no data element comes from a tag value". +- [ ] `CHANGELOG.md` `[0.3.0] — unreleased`, "Breaking changes and migration". +- [ ] `DESIGN_ISSUES.md`: one entry appended at the end, status resolved. +- [ ] Commit `docs: GroupBy keeps non-key tags as tags (ITL-955)`. + +### Task 6: Verify and ship + +- [ ] `uv run ruff format . && uv run ruff check . && uv run --with basedpyright basedpyright` + (exit code 0), then `uv run pytest tests/` once, single process. +- [ ] Merge `origin/main` if it moved, rerun, push, open the PR (create-pr skill), link it + (`orca linear attach`, `orca worktree set --worktree current --pr `), CodeRabbit round, + `worker_done`. + +## Progress notes + +- 2026-11-11: investigation done, plan written, waiting for coordinator approval. +- 2026-11-11: coordinator approved (list-valued tags; keep the change easy to adjust if the + developer prefers dropping). Conditions: only the six group_by row golden entries move; scalar + sort order pinned by a test; stored-data rerun kept as a test; out-of-scope findings go to + worker_done only. +- 2026-11-11: Tasks 1-5 done and pushed (tests, fix, golden, docs). Remaining: Task 6. diff --git a/tests/test_arrays/test_array_operators.py b/tests/test_arrays/test_array_operators.py index 9cc8946b6..f53962ddc 100644 --- a/tests/test_arrays/test_array_operators.py +++ b/tests/test_arrays/test_array_operators.py @@ -145,8 +145,8 @@ def test_group_by() -> None: out = list(GroupBy(by=["g"])(arrays_source()).iter_data()) assert sorted(fields(t)["g"] for t, _ in out) == ["g0", "g1", "g2"] for tag, data in out: - # GroupBy moves the non-key tag ``k`` into the data as a list. - names = list(fields(data)["k"]) + # GroupBy keeps the non-key tag ``k`` as a list-valued tag. + names = list(fields(tag)["k"]) assert sorted(names) == sorted(n for n in NAMES if GROUP[n] == fields(tag)["g"]) for name, x in zip(names, fields(data)["x"], strict=True): assert_arrays_identical(x, expected_x(name)) diff --git a/tests/test_core/operators/test_async_sync_equivalence.py b/tests/test_core/operators/test_async_sync_equivalence.py index 809af1813..631c13f68 100644 --- a/tests/test_core/operators/test_async_sync_equivalence.py +++ b/tests/test_core/operators/test_async_sync_equivalence.py @@ -347,6 +347,26 @@ def test_async_rows_equal_sync_rows(case: Case) -> None: ] +def test_group_by_keeps_non_key_tags_as_tags_on_both_paths() -> None: + """Neither path moves a non-key tag into the data (ITL-955). + + ``source_a`` has tags ``k`` and ``ta``; grouping by ``ta`` leaves ``k`` a + list-valued tag, and the data of every row holds only the input's data + columns. + """ + _, data_keys = source_a().keys() + for rows in ( + sync_rows(GroupBy(["ta"]), source_a()), + async_rows(GroupBy(["ta"]), source_a()), + ): + members = sorted(tuple(cast(list[int], tag.as_dict()["k"])) for tag, _ in rows) + assert members == [(1, 2), (3,)] + for tag, data in rows: + assert set(tag.keys()) == {"k", "ta"} + assert set(data.keys()) == set(data_keys) + assert tag.as_table().schema.field("k").type == pa.large_list(pa.int16()) + + @pytest.mark.parametrize("case", PROVENANCE_CASES) def test_async_provenance_equals_sync(case: Case) -> None: """The async path carries the sync path's per-row source info.""" diff --git a/tests/test_core/operators/test_chained_aggregation.py b/tests/test_core/operators/test_chained_aggregation.py index 1136252e4..2b8d12c4c 100644 --- a/tests/test_core/operators/test_chained_aggregation.py +++ b/tests/test_core/operators/test_chained_aggregation.py @@ -289,8 +289,9 @@ def test_source_info_is_a_list_per_row_of_its_members_tokens( ) -> None: """Every row's provenance lists its own members' upstream tokens. - A tag that an upstream ``GroupBy`` promoted to data (``id``) has no - token (``None``); it is still aggregated into a list per row. + Only data columns carry provenance: a non-key tag that an upstream + ``GroupBy`` kept as a list-valued tag (``id``) has no source-info + column at all (ITL-955). """ upstream = list(chain.upstream.iter_data()) rows = list(chain.result.iter_data()) diff --git a/tests/test_core/operators/test_group_by.py b/tests/test_core/operators/test_group_by.py index 5519c94ef..2391abec0 100644 --- a/tests/test_core/operators/test_group_by.py +++ b/tests/test_core/operators/test_group_by.py @@ -3,6 +3,7 @@ from __future__ import annotations from pathlib import Path +from typing import get_origin import pyarrow as pa import pytest @@ -47,15 +48,12 @@ def test_one_row_per_distinct_key(self, session_source): def test_group_keys_are_scalar_tags(self, session_source): op = GroupBy(by=["subject", "date"]) out = op.process(session_source) - tag_cols, _ = out.keys() - assert set(tag_cols) == {"subject", "date"} + in_tag_schema, _ = session_source.output_schema() + tag_schema, _ = out.output_schema() + assert tag_schema["subject"] == in_tag_schema["subject"] + assert tag_schema["date"] == in_tag_schema["date"] assert out.as_table().column("subject").to_pylist() == ["G", "G"] - - def test_non_key_tags_promoted_to_list_data(self, session_source): - out = GroupBy(by=["subject", "date"]).process(session_source) - _, data_cols = out.keys() - assert "probe" in data_cols - assert out.as_table().column("probe").to_pylist() == [[0, 1], [0, 1]] + assert out.as_table().column("date").to_pylist() == ["d1", "d2"] def test_data_columns_are_lists(self, session_source): out = GroupBy(by=["subject", "date"]).process(session_source) @@ -81,6 +79,136 @@ def test_system_tags_are_scalar_and_renamed(self, session_source): assert col.count(constants.BLOCK_SEPARATOR) >= 2 +class TestGroupByTagsStayTags: + """No data element may come from a tag value (ITL-955). + + ``GroupBy`` uses tags to form groups, but tag content never becomes data: + a non-key tag stays in the tag group as a list-valued tag, as ``Batch`` + keeps it. + """ + + def test_non_key_tags_stay_list_valued_tags(self, session_source): + """A tag not named in ``by`` stays a tag and holds one value per member.""" + out = GroupBy(by=["subject", "date"])(session_source) + tag_cols, data_cols = out.keys() + assert tag_cols == ("subject", "date", "probe") + assert data_cols == ("path",) + assert out.as_table().column("probe").to_pylist() == [[0, 1], [0, 1]] + + def test_tag_order_is_the_input_tag_order(self, session_source): + """A key named last in ``by`` does not reorder the tag columns.""" + out = GroupBy(by=["date"])(session_source) + assert out.keys()[0] == ("subject", "date", "probe") + + def test_no_data_column_is_an_input_tag(self, session_source): + """The tag and data column names are the input's: no tag moves to data.""" + in_tags, in_data = session_source.keys() + out = GroupBy(by=["subject"])(session_source) + assert out.keys() == (in_tags, in_data) + + def test_non_key_tag_schema_is_a_list_tag(self, session_source): + """``output_schema()`` reports a non-key tag as a list in the tag schema.""" + out = GroupBy(by=["subject", "date"])(session_source) + tag_schema, data_schema = out.output_schema() + assert "probe" not in data_schema + assert get_origin(tag_schema["probe"]) is list + + def test_non_key_tag_keeps_its_arrow_type(self): + """The list element is the input tag's type: ``int8`` is not widened.""" + table = pa.table( + { + "subject": pa.array(["G", "G"], pa.large_string()), + "probe": pa.array([1, 0], pa.int8()), + "path": pa.array(["b", "a"], pa.large_string()), + } + ) + source = ArrowTableSource(table, tag_columns=["subject", "probe"]) + out = GroupBy(by=["subject"])(source) + assert out.as_table().schema.field("probe").type == pa.large_list(pa.int8()) + + def test_non_key_tags_get_no_source_info_column(self, session_source): + """Source info belongs to data columns; a tag has none.""" + out = GroupBy(by=["subject", "date"])(session_source) + names = out.as_table(columns={"source": True}).column_names + assert f"{constants.SOURCE_PREFIX}probe" not in names + assert f"{constants.SOURCE_PREFIX}path" in names + + def test_datagrams_carry_non_key_tags_on_the_tag(self, session_source): + """Each row's ``Tag`` holds the non-key tags; its ``Data`` holds only data.""" + out = GroupBy(by=["subject", "date"])(session_source) + rows = list(out.iter_data()) + assert len(rows) == 2 + for tag, data in rows: + assert tag.as_dict()["probe"] == [0, 1] + assert set(data.as_dict()) == {"path"} + + def test_a_pod_receives_only_data_columns(self, session_source): + """A function pod never sees tag content, so its signature is the data.""" + pod = FunctionPod( + PythonDataFunction(_count_paths, output_keys="n") # pyright: ignore[reportArgumentType] # protocol mismatch baselined elsewhere (ITL-844) + ) + out = pod(GroupBy(by=["subject", "date"])(session_source)) + assert [d.as_dict()["n"] for _, d in out.iter_data()] == [2, 2] + + def test_list_valued_non_key_tag_still_cannot_be_a_key(self, session_source): + """A list-valued tag of a grouped stream is refused as a grouping key.""" + grouped = GroupBy(by=["subject", "date"])(session_source) + with pytest.raises(InputValidationError, match="list-valued"): + GroupBy(by=["probe"])(grouped) + + def test_regroup_nests_list_tags(self, session_source): + """Grouping a grouped stream on a coarser key nests the list tags.""" + out = GroupBy(by=["subject"])(GroupBy(by=["subject", "date"])(session_source)) + assert out.keys() == (("subject", "date", "probe"), ("path",)) + row = out.as_table().to_pylist()[0] + assert row["date"] == ["d1", "d2"] + assert row["probe"] == [[0, 1], [0, 1]] + assert row["path"] == [["a", "b"], ["c", "d"]] + + def test_regroup_orders_list_tags_containing_nulls(self): + """A list-valued non-key tag holding a null still orders the members.""" + table = pa.table( + { + "a": ["x", "x", "x", "x"], + "b": [1, 1, 2, 2], + "c": pa.array([None, 1, 2, 1], pa.int64()), + "v": [1, 2, 3, 4], + } + ) + source = ArrowTableSource( + table, tag_columns=["a", "b", "c"], infer_nullable=True + ) + first = GroupBy(by=["a", "b"])(source) + # Regrouping on ``a`` alone leaves ``b`` scalar and ``c`` list-valued: + # the two members tie on nothing, and are ordered by ``b`` first. + out = GroupBy(by=["a"])(first) + row = out.as_table().to_pylist()[0] + assert row["b"] == [1, 2] + assert row["c"] == [[1, None], [1, 2]] + assert row["v"] == [[2, 1], [4, 3]] + + def test_members_tied_on_scalar_tags_order_by_list_tag_with_null(self): + """``[1, None]`` and ``[1, 2]`` compare without a ``TypeError``.""" + table = pa.table( + { + "a": ["x", "x", "x", "x"], + "b": ["p", "p", "q", "q"], + "c": pa.array([1, None, 1, 2], pa.int64()), + "v": [1, 2, 3, 4], + } + ) + source = ArrowTableSource( + table, tag_columns=["a", "b", "c"], infer_nullable=True + ) + # Drop ``b`` after the first grouping so the two members differ only in + # the list-valued tag ``c``: [1, None] against [1, 2]. + first = GroupBy(by=["a", "b"])(source).drop_tag_columns(["b"]) + out = GroupBy(by=["a"])(first) + row = out.as_table().to_pylist()[0] + assert row["c"] == [[1, 2], [1, None]] + assert row["v"] == [[3, 4], [1, 2]] + + class TestGroupByOrdering: """Member and group ordering. @@ -104,6 +232,28 @@ def test_members_sorted_by_non_key_tags(self, session_source): out = GroupBy(by=["subject", "date"]).process(session_source) assert out.as_table().column("probe").to_pylist()[0] == [0, 1] + def test_scalar_non_key_tags_sort_ascending_with_nulls_last(self): + """Pins the member order for scalar tags: ascending, nulls last. + + The sort key is recursive so that list-valued tags compare (ITL-955); + for scalar tags it must give exactly the order it always gave. + """ + table = pa.table( + { + "g": ["x"] * 5, + "n": pa.array([3, None, 1, 2, 1], pa.int64()), + "s": pa.array(["b", "a", None, "c", "a"], pa.large_string()), + "v": [0, 1, 2, 3, 4], + } + ) + source = ArrowTableSource( + table, tag_columns=["g", "n", "s"], infer_nullable=True + ) + row = GroupBy(by=["g"])(source).as_table().to_pylist()[0] + assert row["n"] == [1, 1, 2, 3, None] + assert row["s"] == ["a", None, "c", "b", "a"] + assert row["v"] == [4, 2, 3, 0, 1] + def test_row_order_does_not_affect_output(self, session_table): """Same rows, shuffled, must produce a byte-identical table.""" shuffled = session_table.take([3, 1, 2, 0]) @@ -331,6 +481,18 @@ def test_does_not_override_async_execute(self): # annotations stringified with no resolvable scope, and `Path` would fail to # resolve. Module scope keeps `-> Path` resolvable, matching how real pipelines # declare pods. +def _count_paths(path: list[str | None]) -> int: + """Return the number of members in a group's ``path`` list. + + Args: + path: The grouped ``path`` values, one per member. + + Returns: + The member count. + """ + return len(path) + + def _make_path(seed: str) -> Path: return Path(f"/data/sync_{seed}.parquet") diff --git a/tests/test_hashing/golden/data/pipeline.json b/tests/test_hashing/golden/data/pipeline.json index d5b5257ce..396884e78 100644 --- a/tests/test_hashing/golden/data/pipeline.json +++ b/tests/test_hashing/golden/data/pipeline.json @@ -60,12 +60,12 @@ ], "group_by.content_hash": "semantic_v0.1:ee6478a83847e5e9d6bea916138170139aba8c5b43c6992a22d048a07b9b4526", "group_by.pipeline_hash": "semantic_v0.1:5ab1cb5d5ce9b641d19045bae0d94bbb9e15703bebfe37ec57fd13b141d2a0f0", - "group_by.row[{\"cohort\":\"a\"}]._tag_record_id::13d8efe7433022d8f328bbd9ea2cf66397ef44167932c3b08fb43153bc1be1d4::13d8efe7433022d8f328bbd9ea2cf66397ef44167932c3b08fb43153bc1be1d4": "7d212ee5c358519c9e4729f4a0898aad", - "group_by.row[{\"cohort\":\"a\"}]._tag_source_id::13d8efe7433022d8f328bbd9ea2cf66397ef44167932c3b08fb43153bc1be1d4::13d8efe7433022d8f328bbd9ea2cf66397ef44167932c3b08fb43153bc1be1d4": "bab61585da6894512c5947710f0c0d4bb46e3f0fcd511a088a4c76780f1ca5ae", - "group_by.row[{\"cohort\":\"a\"}].data_hash": "arrow_v0.1:0000018910e185c56ec7890dcba74a0553f06e355191c27a97a423dbbc32bea2441149", - "group_by.row[{\"cohort\":\"b\"}]._tag_record_id::13d8efe7433022d8f328bbd9ea2cf66397ef44167932c3b08fb43153bc1be1d4::13d8efe7433022d8f328bbd9ea2cf66397ef44167932c3b08fb43153bc1be1d4": "b7f2f8f7412a54fdb03cd96b6b0b1643", - "group_by.row[{\"cohort\":\"b\"}]._tag_source_id::13d8efe7433022d8f328bbd9ea2cf66397ef44167932c3b08fb43153bc1be1d4::13d8efe7433022d8f328bbd9ea2cf66397ef44167932c3b08fb43153bc1be1d4": "90ffe0efe9214e2770ce632ae3146e4bb33e9d28568b55b7dd67f07862836010", - "group_by.row[{\"cohort\":\"b\"}].data_hash": "arrow_v0.1:000001ba6adaea3c56bf0be427961819e0491e678c6dc6486999ab5025d9074ec9bbd4", + "group_by.row[{\"cohort\":\"a\",\"subject\":[\"s1\",\"s2\"]}]._tag_record_id::13d8efe7433022d8f328bbd9ea2cf66397ef44167932c3b08fb43153bc1be1d4::13d8efe7433022d8f328bbd9ea2cf66397ef44167932c3b08fb43153bc1be1d4": "7d212ee5c358519c9e4729f4a0898aad", + "group_by.row[{\"cohort\":\"a\",\"subject\":[\"s1\",\"s2\"]}]._tag_source_id::13d8efe7433022d8f328bbd9ea2cf66397ef44167932c3b08fb43153bc1be1d4::13d8efe7433022d8f328bbd9ea2cf66397ef44167932c3b08fb43153bc1be1d4": "bab61585da6894512c5947710f0c0d4bb46e3f0fcd511a088a4c76780f1ca5ae", + "group_by.row[{\"cohort\":\"a\",\"subject\":[\"s1\",\"s2\"]}].data_hash": "arrow_v0.1:000001bb0b316000a81629aa9fac996122e39257d9ca5d2dc12608c972e0f8d55f28a3", + "group_by.row[{\"cohort\":\"b\",\"subject\":[\"s3\"]}]._tag_record_id::13d8efe7433022d8f328bbd9ea2cf66397ef44167932c3b08fb43153bc1be1d4::13d8efe7433022d8f328bbd9ea2cf66397ef44167932c3b08fb43153bc1be1d4": "b7f2f8f7412a54fdb03cd96b6b0b1643", + "group_by.row[{\"cohort\":\"b\",\"subject\":[\"s3\"]}]._tag_source_id::13d8efe7433022d8f328bbd9ea2cf66397ef44167932c3b08fb43153bc1be1d4::13d8efe7433022d8f328bbd9ea2cf66397ef44167932c3b08fb43153bc1be1d4": "90ffe0efe9214e2770ce632ae3146e4bb33e9d28568b55b7dd67f07862836010", + "group_by.row[{\"cohort\":\"b\",\"subject\":[\"s3\"]}].data_hash": "arrow_v0.1:000001d898f5f76e2c9c6c40b91a4accb47bbf833dbdf5100a9aff17e21fd988f36898", "group_by.system_tag_columns": [ "_tag_source_id::13d8efe7433022d8f328bbd9ea2cf66397ef44167932c3b08fb43153bc1be1d4::13d8efe7433022d8f328bbd9ea2cf66397ef44167932c3b08fb43153bc1be1d4", "_tag_record_id::13d8efe7433022d8f328bbd9ea2cf66397ef44167932c3b08fb43153bc1be1d4::13d8efe7433022d8f328bbd9ea2cf66397ef44167932c3b08fb43153bc1be1d4" diff --git a/tests/test_pipeline/test_aggregation_job.py b/tests/test_pipeline/test_aggregation_job.py index 598713fcf..8db345840 100644 --- a/tests/test_pipeline/test_aggregation_job.py +++ b/tests/test_pipeline/test_aggregation_job.py @@ -16,11 +16,14 @@ from orcapod.core.data_function import PythonDataFunction from orcapod.core.function_pod import FunctionPod -from orcapod.core.operators import Batch, GroupBy, MergeJoin +from orcapod.core.operators import Batch, GroupBy, MergeJoin, SelectDataColumns from orcapod import File from orcapod.core.sources import ArrowTableSource, DictSource +from orcapod.core.streams import ArrowTableStream from orcapod.databases import DeltaTableDatabase from orcapod.pipeline import PipelineJob +from orcapod.system_constants import constants +from orcapod.types import Schema # --------------------------------------------------------------------------- # Invocation recording @@ -37,27 +40,15 @@ def count_paths(path: list[str]) -> int: - """Record the batch's `path` members and return the batch size. + """Record the members' `path` values and return how many there are. - Used with `Batch`, which keeps `probe` as a list-valued *tag*, so `path` + `Batch` and `GroupBy` both keep `probe` as a list-valued *tag*, so `path` is the only data column reaching the pod. """ _CALLS["path"].append(list(path)) return len(path) -def count_group(probe: list[int], path: list[str]) -> int: - """Record the group's `path` members and return the group size. - - Used with `GroupBy`, which promotes the non-key tag `probe` to a - list-valued *data* column. `FunctionPod` requires the pod signature to - cover every incoming data column, so `probe` must be a parameter even - though only `path` is asserted on. - """ - _CALLS["path"].append(list(path)) - return len(path) - - def count_v(v: list[str]) -> int: """Record the merged `v` members and return the merged-list size.""" _CALLS["v"].append(list(v)) @@ -99,7 +90,7 @@ def _make(paths: list[str] | None = None) -> ArrowTableSource: return _make -def _run(store, source, operator, name, function=count_group): +def _run(store, source, operator, name, function=count_paths): """Record and run `source -> operator -> function` as a PipelineJob.""" pod = FunctionPod(PythonDataFunction(function, output_keys="n")) job = PipelineJob(name=name, store=store) @@ -186,6 +177,125 @@ def test_changed_member_invalidates_only_its_group( assert sorted(table.column("n").to_pylist()) == [2, 2] +class _LegacyGroupBy(GroupBy): + """``GroupBy`` with the output shape it had before ITL-955. + + Non-key tags are declared as data columns, which is what stores written by + an earlier version hold. It keeps the identity of ``GroupBy`` so that it + occupies the same place in the pipeline. + """ + + def identity_structure(self): + """Return the identity of ``GroupBy``, so stored tables are at the same paths.""" + return ("GroupBy", self.by) + + def unary_static_process(self, stream): + """Group as ``GroupBy`` does, then declare only the ``by`` columns as tags. + + The non-key tags become data columns, the shape ``GroupBy`` produced before + ITL-955. + + Args: + stream: The stream to group. + + Returns: + The grouped stream in the legacy shape. + """ + grouped = super().unary_static_process(stream) + return ArrowTableStream( + grouped.as_table(columns={"source": True, "system_tags": True}), + tag_columns=self.by, + data_context=grouped.data_context, # pyright: ignore[reportAttributeAccessIssue] # not on StreamProtocol, as in GroupBy itself (ITL-844) + ) + + def unary_output_schema(self, stream, *, columns=None, all_info=False): + """Predict the legacy schema: non-key tags are moved to the data schema. + + Args: + stream: The stream to group. + columns: Column groups to include, as for ``GroupBy``. + all_info: Whether to include every column group. + + Returns: + The ``(tag_schema, data_schema)`` pair of the legacy shape. + """ + tag_schema, data_schema = super().unary_output_schema( + stream, columns=columns, all_info=all_info + ) + tags = { + k: v + for k, v in tag_schema.items() + if k in self.by or k.startswith(constants.SYSTEM_TAG_PREFIX) + } + moved = {k: v for k, v in tag_schema.items() if k not in tags} + return Schema(tags), Schema({**moved, **data_schema}) + + +class TestStoreWrittenBeforeTagsStayedTags: + """Results stored before ITL-955 are still found after it. + + Before the fix a pod could only follow ``GroupBy`` by selecting the real + data columns (or by taking the moved tags as parameters). The fix changes + neither a pipeline hash nor the pod's input data, so a store written by the + old shape serves the new one without recomputing. + """ + + @staticmethod + def _run(store, source, group_by): + """Run ``source -> group_by -> select path -> counting pod`` on ``store``. + + Args: + store: The database the job writes to and reads from. + source: The source stream. + group_by: The grouping operator to use. + + Returns: + The job after its run. + """ + pod = FunctionPod( + PythonDataFunction(count_paths, output_keys="n") # pyright: ignore[reportArgumentType] # protocol mismatch baselined elsewhere (ITL-844) + ) + job = PipelineJob(name="legacy", store=store) + with job: + grouped = group_by(source, label="agg") + selected = SelectDataColumns(columns=["path"])(grouped, label="sel") + pod(selected, label="counter") + return job.run() + + def test_legacy_fixture_has_the_old_shape(self, session_source_factory): + """Guard the fixture -- if this stops holding, the next test is vacuous.""" + out = _LegacyGroupBy(by=["subject", "date"])(session_source_factory()) + assert out.keys() == (("subject", "date"), ("probe", "path")) + + def test_rerun_on_old_store_does_not_recompute(self, store, session_source_factory): + """A store written with the legacy shape serves the new one without recomputing.""" + before = self._run( + store, session_source_factory(), _LegacyGroupBy(by=["subject", "date"]) + ) + assert len(_CALLS["path"]) == 2 + old_counter = before.nodes["counter"] + assert old_counter.keys()[0] == ("subject", "date") + + _CALLS["path"].clear() + after = self._run( + store, session_source_factory(), GroupBy(by=["subject", "date"]) + ) + counter = after.nodes["counter"] + + assert counter.pipeline_hash() == old_counter.pipeline_hash() + assert _CALLS["path"] == [], "stored results must be reused, not recomputed" + assert after.nodes["agg"].keys() == (("subject", "date", "probe"), ("path",)) + assert counter.keys() == (("subject", "date", "probe"), ("n",)) + rows = sorted( + counter.as_table().select(["date", "probe", "n"]).to_pylist(), + key=lambda r: r["date"], + ) + assert rows == [ + {"date": "d1", "probe": [0, 1], "n": 2}, + {"date": "d2", "probe": [0, 1], "n": 2}, + ] + + class TestMergeJoinRegression: def test_merge_join_completes_in_job(self, store): """MergeJoin carries source columns as parallel lists. @@ -219,7 +329,7 @@ def test_merge_join_completes_in_job(self, store): # --------------------------------------------------------------------------- -def reduce_configs(probe: list[int], cfg: list[File]) -> int: +def reduce_configs(cfg: list[File]) -> int: """Record each member's file *contents* so a stale cache is visible.""" _CALLS["cfg"].append([Path(c).read_text().strip() for c in cfg]) return len(cfg)