diff --git a/.zed/rules b/.zed/rules index 8723b7d0e..7e43a38a8 100644 --- a/.zed/rules +++ b/.zed/rules @@ -1179,9 +1179,9 @@ data. The rules: unchanged path; any other type difference, and every data column, behaves as before. - GroupBy requires every column in `by` to be a scalar tag column; raises InputValidationError for unknown, data, or list-valued columns (list-valued usually means the stream came from - Batch — group before batching, not after). Members sort by non-key tag values with system - record_id, then source id, as tiebreakers; groups emit in key order, so reordered input yields a - byte-identical table. + Batch — group before batching, not after). Members sort by the non-key tags in name order + (or as member_order gives) with system record_id, then source id, as tiebreakers; groups + emit sorted by their keys in name order, so reordered input yields a byte-identical table. - GroupBy's output tag columns are its `by` columns, scalar, in the input's tag order; every other tag is dropped (ITL-1070; ITL-955 had kept them as list[T] tags, never released). The dropped tags still order the members, so the data lists are as before. Their values are not @@ -1190,6 +1190,18 @@ data. The rules: can still order members, so the member sort key (_null_last_key) is recursive and sorts nulls last at every depth. A grouped stream grouped again on a coarser key orders its members by the first grouping's keys outside the coarser key, then by record id. An opt-in mode that keeps non-key tags is ITL-1071. +- GroupBy never depends on how its input's columns are arranged (developer, November 17, + 2026, ITL-1226). Members are ordered by the non-key tags in name order (each ascending, + nulls last), then the record-id and source-id columns; `member_order` (non-key TAG names + only: a data column, a key, a repeat, a set or an empty list is refused; `"name"` is the + default; an ordering function is reserved, refused for now) names tags to order by first, + the rest follow in name order. A member is never ordered by a data value. The keys in `by` + are kept in name order: their order is not identity, and groups are emitted sorted by them. + `member_order` enters the identity and config only when given. Every arrangement of the + input (tag order, data order, table layout, the order of `by`) gives identical values, + record ids, folds, source-info tokens and hashes; only the output's column order follows + the input, as presentation. `TagBy`'s identity is its sorted names, and its new tags are + placed in name order. - Data source-info values may be str, None, or list[...], nested to any depth; Arrow/Python types are derived from the value, with None mapping to large_string. That rule has one definition, arrow_utils.source_info_arrow_type, and applies only to tokens supplied as diff --git a/CLAUDE.md b/CLAUDE.md index 01e8a0f69..401f7c86d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1223,10 +1223,11 @@ data. The rules: unchanged path; any other type difference, and every data column, behaves as before. - `GroupBy` requires every column in `by` to be a scalar tag column; it raises `InputValidationError` for unknown, data, or list-valued columns (the last usually means the - stream came from `Batch` — group before batching, not after). Members are sorted by non-key - tag values with the system `record_id` and then the source id as tiebreakers (a record id - names a row within its source only), so the hashed lists are stable across runs. Groups themselves are emitted in key order, so a reordered input yields a - byte-identical table. + stream came from `Batch` — group before batching, not after). Members are sorted by the + non-key tags in name order (or as `member_order` gives), with the system `record_id` and + then the source id as tiebreakers (a record id names a row within its source only), so the + hashed lists are stable across runs. Groups themselves are emitted sorted by their keys in + name order, so a reordered input yields a byte-identical table. - `GroupBy`'s output tag columns are its `by` columns, scalar, in the input's tag order; every other tag is dropped (ITL-1070; ITL-955 had kept them as `list[T]` tags, never released). The dropped tags still order the members, so the data lists are as before. Their values are not @@ -1235,6 +1236,18 @@ data. The rules: can still order members, so the member sort key (`_null_last_key`) is recursive and sorts nulls last at every depth. A grouped stream grouped again on a coarser key orders its members by the first grouping's keys outside the coarser key, then by record id. An opt-in mode that keeps non-key tags is ITL-1071. +- GroupBy never depends on how its input's columns are arranged (developer, November 17, + 2026, ITL-1226). Members are ordered by the non-key tags in name order (each ascending, + nulls last), then the record-id and source-id columns; `member_order` (non-key TAG names + only: a data column, a key, a repeat, a set or an empty list is refused; `"name"` is the + default; an ordering function is reserved, refused for now) names tags to order by first, + the rest follow in name order. A member is never ordered by a data value. The keys in `by` + are kept in name order: their order is not identity, and groups are emitted sorted by them. + `member_order` enters the identity and config only when given. Every arrangement of the + input (tag order, data order, table layout, the order of `by`) gives identical values, + record ids, folds, source-info tokens and hashes; only the output's column order follows + the input, as presentation. `TagBy`'s identity is its sorted names, and its new tags are + placed in name order. - `Data` source-info values may be `str`, `None`, or `list[...]`, nested to any depth; the Arrow and Python types are derived from the value, with `None` mapping to `large_string`. That rule has one definition, `arrow_utils.source_info_arrow_type`, and applies only to diff --git a/changelog.d/ITL-1023.md b/changelog.d/ITL-1023.md index 3b58c7f7e..a866eb1ab 100644 --- a/changelog.d/ITL-1023.md +++ b/changelog.d/ITL-1023.md @@ -27,8 +27,8 @@ plain with data-tag gives a data-tag. `GroupBy` keeps a data-tag key scalar in both roles. A data-tag non-key loses its tag role, as every non-key tag does (ITL-1070), but keeps its data column as a list with one element and one source-info - token per member; the members are ordered by the tuple of the input's non-key tags, - the same order for every data list of the group. + token per member; the members are ordered by the input's non-key tags in name order + (ITL-1226), the same order for every data list of the group. - A source-info column has the shape of its data column. A value merged from several equal elements (a data-tag `GroupBy` key; a `Join` key that is a data-tag on several inputs) has one merged token instead of a list of every member's token (ITL-1017): the merging @@ -100,8 +100,8 @@ - `TagBy` and the stream method `tag_by` give existing data columns the tag role, as data-tags (ITL-1175): `stream.tag_by("x", "y")` (or `tag_by(["x", "y"])`). Each named column becomes one column in both roles, with its values and source info - unchanged; the new tags follow the stream's tags in the order given, and that order is - part of the operator's identity. System tags, record ids and every data column are + unchanged; the new tags follow the stream's tags in name order, and the order the + names are given in is not part of the operator (ITL-1226). System tags, record ids and every data column are unchanged. A column a function computed may become a tag and then key a join or a group. Refused, naming the column: a name that is not a data column of the input, a name that is already a tag, a reserved name, and a type that cannot be a tag (a tag is diff --git a/changelog.d/ITL-1226.md b/changelog.d/ITL-1226.md new file mode 100644 index 000000000..c5183161c --- /dev/null +++ b/changelog.d/ITL-1226.md @@ -0,0 +1,65 @@ +### Breaking changes and migration + +- `GroupBy` no longer depends on how its input's columns are arranged (ITL-1226). Two + separate effects for pipelines written with 0.3.0: + - **Member order.** The members of a group are now ordered by the non-key tags in name + order, then by record id and source id as before; they were ordered by the non-key + tags in the input's tag order. Affected: a `GroupBy` whose input has two or more + non-key tags that are not in name order (for example `tag_columns=["subject", "unit", + "probe"]` grouped by `subject`: `unit` came before `probe`). For such a `GroupBy` the + order of every list in a group's row changes, and so do the group's folded record id + and source id (`group_v1:...`), the `group_v1` source-info token of a data-tag key, and + the data hash of every grouped row. Everything downstream that reads the lists + recomputes: a function over a grouped row gets a new input and is called again. The + hashes of the `GroupBy` itself and its stored paths do not change. A `GroupBy` with + one non-key tag, or whose non-key tags are already in name order, is not affected, + unless a non-key tag holds NaN (at any list depth): its members are ordered by the NaN + rule under Fixed, which can move them too. To keep the earlier order, name the tags: + `member_order=["unit", "probe"]`. Stored rows of every affected `GroupBy` are not + replaced. An operator node in `LOG` or `REPLAY` (the `GroupBy` and every + operator node after it) keeps its table: `REPLAY` returns the rows stored before the + upgrade, and a `LOG` run adds the new rows beside them, so `REPLAY` then returns both. + A function node after it keeps its pipeline table, so `node.as_source()` lists the + earlier records beside the new ones. Before running such a pipeline with this version, + delete those stored tables, or attach a new database; `clear_cache()` clears memory + only. + - **The order of `by`.** The keys are kept in name order: `GroupBy(by=["b", "a"])` is + now the same operator as `GroupBy(by=["a", "b"])`, and groups are emitted sorted by + the keys in name order (they were emitted in the order of `by`). Affected: a `GroupBy` + with two or more keys not in name order. Its content and pipeline hash change, and so + do its stored operator-cache path, the order of its rows, and the hashes and stored + pipeline-table paths of every node downstream. Values do not change: the lists, folded + ids and data hashes are as before, and a function downstream finds its stored results + (results are keyed by the function and the input data, neither of which changes), so + nothing is computed again; its records are written to a new pipeline table. + For `GroupBy`'s `by` and for `TagBy`, the order of a list or tuple no longer enters + the identity; it did before this change. + +### Added + +- `GroupBy(..., member_order=...)` and `stream.group_by(..., member_order=...)` (ITL-1226): + the tags that order the members of a group, in order; the other non-key tags follow in + name order. Only tags of the input may be named (plain tags or data-tags, not a key); a + member is never ordered by a data value, and a data column is refused with `tag_by` as + the way to make it a tag. `"name"` is the default order. An ordering function is + planned and refused for now. The option is part of the operator's identity only when + given, so a `GroupBy` without it keeps its hash. + +### Changed + +- `TagBy` and `tag_by` (ITL-1226): the order of the names is no longer part of the + operator. The identity is the sorted names, the new tags follow the stream's tags in name + order, and the names may be given in any iterable, a set included. + +### Fixed + +- `GroupBy` orders a member whose ordering tag holds NaN (ITL-1226): NaN sorts after + every number and before null, at any list depth, and two NaN tie (the record id breaks + the tie). Before, a NaN compared with nothing and the members of its group came out in + the input's row order, so their lists and hashes depended on it. A NaN group key is + unchanged: each row with a NaN key is a group of its own. + +### Known limitations + +- A raw `ArrowTableStream`'s content hash depends on the order of its `tag_columns` + (ITL-1227); its values and pipeline hash do not, and a source's hashes do not. diff --git a/docs/concepts/data-tags.md b/docs/concepts/data-tags.md index 4a67d2946..4966f5959 100644 --- a/docs/concepts/data-tags.md +++ b/docs/concepts/data-tags.md @@ -147,9 +147,9 @@ print(by_phase.keys()) # (('phase',), ('phase', 'unit', 'time')) ``` -- `s.tag_by("x", "y")` equals `s.tag_by(["x", "y"])`. The new tags follow the stream's - tags in the order given, and that order is part of the operator (`TagBy`): the names - are a list, never a set. +- `s.tag_by("x", "y")` equals `s.tag_by(["x", "y"])`, `s.tag_by("y", "x")` and + `s.tag_by({"x", "y"})`. The new tags follow the stream's tags in name order, whatever + order they are named in. The order of the names is not part of the operator (`TagBy`). - Nothing else changes: no row is added, removed or reordered, the data columns keep their names, types and values, every data column keeps its own source-info token, and the system tags keep their names and values. @@ -173,7 +173,7 @@ your own classes and logical types (`Path`, `UUID`, arrays) are refused. |---|---| | `TagBy` / `tag_by` | Makes the data column `x` a data-tag (above); everything else is unchanged. | | `Join` | Plain with plain: as before. Plain with data-tag, or data-tag with data-tag: a data-tag and **one** data column `x`. When `x` is a data-tag on several inputs, its source info is one merged token (below). A key that is `T` in one input and `T \| None` in another is `T` in both roles, as for a plain key. Every other data-name collision is still an error. | -| `GroupBy` | A group key stays scalar in both roles, and its source info is one merged token (below). A non-key loses its tag role, as every non-key tag does, but keeps its data column: a list like every other data column, one source-info token per member. The members are ordered by the tuple of the input's non-key tags (then record ids), the same order for every data list of the group, so all of them stay aligned. | +| `GroupBy` | A group key stays scalar in both roles, and its source info is one merged token (below). A non-key loses its tag role, as every non-key tag does, but keeps its data column: a list like every other data column, one source-info token per member. The members are ordered by the tags `member_order` names, in its order, then by the input's other non-key tags in name order, then by record ids and last by source ids. That order is the same for every data list of the group, so all of them stay aligned. | | `SelectDataColumns`, `DropDataColumns` | Removing the data column removes the data role only: `x` stays a plain tag with the same values, without source info, with a `DataTagMaterializationWarning`. | | `SelectTagColumns`, `DropTagColumns` | Removing the tag removes the tag role only: `x` stays a data column, with its source info. | | `SelectColumns`, `DropColumns` | `x` is one column, so naming it acts on both roles: selecting it keeps `x` a data-tag, dropping it (or not selecting it) removes the column from both roles. The roles are never split, so nothing is materialized. | diff --git a/docs/concepts/notebooks.md b/docs/concepts/notebooks.md index 1848ba135..f44532a04 100644 --- a/docs/concepts/notebooks.md +++ b/docs/concepts/notebooks.md @@ -168,8 +168,8 @@ means = job.nodes["session_means"] view = means.run() # the node holds a current run: this computes nothing print(view.header()) # FunctionJobNode[session_means] — all 8 rows -print(view.tail(3)) -# FunctionJobNode[session_means] — last 3 of 8 rows +print(view.sort_by_tags().tail(3)) +# FunctionJobNode[session_means] — last 3 of 8 rows, sorted by tags # * = tag # shape: (3, 3) # ┌─────────────┬──────────┬──────────────┐ diff --git a/docs/concepts/operators.md b/docs/concepts/operators.md index 176c62dfb..4b73ea29c 100644 --- a/docs/concepts/operators.md +++ b/docs/concepts/operators.md @@ -305,14 +305,49 @@ plain tag. A data-tag group key stays a data column too (scalar, in both roles), following function can read its value (see [Data-tags](data-tags.md)). Ordering is deterministic by construction, because orcapod hashes the emitted lists to build -cache keys: members are sorted by the input's non-key tag values (with the internal record id -as a final tiebreaker), even though those tags are dropped, and groups are emitted in group-key -order. A reordered input therefore produces a byte-identical output table. +cache keys: + +- **Members** are sorted by the input's non-key tags, even though those tags are dropped. + The tags are taken in name order, each ascending with nulls last, and the internal record + id breaks ties. `member_order` names tags to sort by first, in its order; the other + non-key tags follow in name order. A member is never ordered by a data value: a name in + `member_order` must be a tag of the input (a data-tag counts), and a data column is + refused (make it a tag first with `tag_by`). +- **Groups** are emitted sorted by their keys, taken in name order. The order of the names + in `by` is not part of the operator: `GroupBy(by=["a", "b"])` and `GroupBy(by=["b", "a"])` + are one operator. +- **No arrangement of the input changes the result.** The order of its tags, the order of + its data columns and the layout of its table leave every value, record id and hash of the + output unchanged. A reordered input produces a byte-identical output table. The output + lists its columns as the input does (the keys in the input's tag order); that order is + presentation, and no hash depends on it. + +```python +units = DictSource( + data=[ + {"subject_id": "mouse_01", "unit": 0, "probe": "imec1", "spike_count": 980}, + {"subject_id": "mouse_01", "unit": 1, "probe": "imec0", "spike_count": 1200}, + {"subject_id": "mouse_01", "unit": 0, "probe": "imec0", "spike_count": 1100}, + ], + tag_columns=["subject_id", "unit", "probe"], tag_only_columns=True, +) +for _, data in units.group_by(["subject_id"]).iter_data(): + print(data.as_dict()) + # {'spike_count': [1100, 1200, 980]} +for _, data in units.group_by(["subject_id"], member_order=["unit"]).iter_data(): + print(data.as_dict()) + # {'spike_count': [1100, 980, 1200]} +``` + +By default the members are ordered by `probe`, then `unit` (name order), whatever the order +of `tag_columns`. With `member_order=["unit"]`, they are ordered by `unit`, then `probe`. The +option is part of the operator's identity only when it is given, so a `GroupBy` without it +keeps its hash. A grouped stream can be grouped again on a coarser key; its data lists then nest one level deeper (`list[list[T]]`). The members of the second grouping are ordered by the first -grouping's keys that are not in the coarser key, then by record id; a tag the first grouping -dropped no longer takes part. +grouping's keys that are not in the coarser key (in name order, or as `member_order` gives), +then by record id; a tag the first grouping dropped no longer takes part. Every name in `by` must be a **scalar tag column** of the input. `GroupBy` raises `InputValidationError` for an unknown column, a data column, or a list-valued tag column -- diff --git a/docs/tutorial/04-many-rows-into-one.md b/docs/tutorial/04-many-rows-into-one.md index 6468e7738..4d909ff37 100644 --- a/docs/tutorial/04-many-rows-into-one.md +++ b/docs/tutorial/04-many-rows-into-one.md @@ -81,11 +81,15 @@ print([round(r, 2) for r in data.as_dict()["rate_hz"]]) # [2.48, 5.43, 11.03, 9.98, 3.58, 4.98, 2.25, 8.52] ``` -These are the eight rates of `mouse_01`'s first session, in the order of the tags that were -dropped: probe `imec0` units 0 to 3, then probe `imec1` units 0 to 3 (compare the first rows -of `rates` in Part 2). The members of a group are always in this order, whatever order the -input rows arrived in, but the list itself no longer says which probe or unit each rate -came from. If a later step needs that, it has to be a data column before you group; see +These are the eight rates of `mouse_01`'s first session, ordered by the tags that were +dropped, taken in name order: by `probe`, then by `unit`. So probe `imec0` units 0 to 3, +then probe `imec1` units 0 to 3 (compare the first rows of `rates` in Part 2). The members +of a group are always in this order, whatever order the input rows arrived in and however +the input's columns are arranged. They are never ordered by a data value. To order them +by other tags first, name those tags: `group_by([...], member_order=["unit"])`. + +The list itself no longer says which probe or unit each rate came from. If a later step +needs that, it has to be a data column before you group; see [GroupBy](../concepts/operators.md#groupby). `group_by` is an operator, so it computed nothing. Every number in those lists was already @@ -106,16 +110,20 @@ session_means = mean_rate.pod(per_session) print(session_means.as_pandas_df()) # subject_id session mean_rate_hz # 0 mouse_01 s1 6.033333 -# 1 mouse_01 s2 6.793750 -# 2 mouse_02 s1 5.845833 -# 3 mouse_02 s2 7.506250 -# 4 mouse_03 s1 4.906944 -# 5 mouse_03 s2 5.059375 -# 6 mouse_04 s1 5.136458 +# 1 mouse_02 s1 5.845833 +# 2 mouse_03 s1 4.906944 +# 3 mouse_04 s1 5.136458 +# 4 mouse_01 s2 6.793750 +# 5 mouse_02 s2 7.506250 +# 6 mouse_03 s2 5.059375 # 7 mouse_04 s2 5.129167 ``` That is the result this tutorial set out to compute: the mean firing rate of every session. +The rows come out sorted by the group keys, taken in name order: `session`, then +`subject_id`. That is why every animal's `s1` comes before any `s2`. The order you write +the keys in does not change it: `group_by(["session", "subject_id"])` is the same +operation. The knockout animals, `mouse_03` and `mouse_04`, fire less, as the generator intended. The grouped stream has one data column, `rate_hz`, and that is exactly the parameter of @@ -195,6 +203,9 @@ and `group_by` cannot use those. - You can only group by tag columns. - `group_by` keeps only the tags you group by; the other tags are dropped. `batch` keeps every tag, as a list. +- The members of a group are ordered by the dropped tags in name order, or by the tags you + name in `member_order`. They are never ordered by a data value. Groups come out sorted by + their keys in name order. ## Exercises @@ -211,9 +222,9 @@ and `group_by` cannot use those. print(per_probe.as_pandas_df().head(4)) # subject_id session probe mean_rate_hz # 0 mouse_01 s1 imec0 7.233333 - # 1 mouse_01 s1 imec1 4.833333 - # 2 mouse_01 s2 imec0 6.564583 - # 3 mouse_01 s2 imec1 7.022917 + # 1 mouse_02 s1 imec0 6.572222 + # 2 mouse_03 s1 imec0 5.102778 + # 3 mouse_04 s1 imec0 5.510417 print(per_probe.as_table().num_rows) # 16 ``` @@ -232,8 +243,8 @@ values: the number of units, the mean rate, and the highest rate. Apply it per s print(summary.as_pandas_df().head(3)) # subject_id session n_units mean_rate_hz max_rate_hz # 0 mouse_01 s1 8 6.033333 11.033333 - # 1 mouse_01 s2 8 6.793750 10.175000 - # 2 mouse_02 s1 8 5.845833 11.400000 + # 1 mouse_02 s1 8 5.845833 11.400000 + # 2 mouse_03 s1 8 4.906944 7.877778 ``` **3.** Compute one number for each genotype: the mean of the session means of its animals. diff --git a/src/orcapod/contexts/data/v1.json b/src/orcapod/contexts/data/v1.json index 60b06c478..b6053916a 100644 --- a/src/orcapod/contexts/data/v1.json +++ b/src/orcapod/contexts/data/v1.json @@ -244,13 +244,13 @@ "digest": "0a789cf9e8d1df9fb762d6dc2fabffcd3383cd6a21ef147e8c6583f963280c2e" }, { - "date": "2026-11-17", + "date": "2026-11-18", "release": null, "added": { - "pipeline-02.json": "6d01f7186e09e8383c62f97e4063d1030a19e465248d85c401b966e62b587f0a" + "pipeline-02.json": "1ec3cbbe09bbb353366d1c8f2573881467d1c8e1ae53a20f8df4e7f43bbfa74e" }, "previous": "0a789cf9e8d1df9fb762d6dc2fabffcd3383cd6a21ef147e8c6583f963280c2e", - "digest": "8a9857166c5b6e54ef80a7d21f345f3b15ada99037e9565dbfdb49d23947347f" + "digest": "71b1edf7a9091477d5a39f5882f2da72e1324b3c353bbfaf755d444469c84b28" } ] }, diff --git a/src/orcapod/core/operators/group_by.py b/src/orcapod/core/operators/group_by.py index a88d70c6a..6d1e31967 100644 --- a/src/orcapod/core/operators/group_by.py +++ b/src/orcapod/core/operators/group_by.py @@ -2,7 +2,8 @@ from __future__ import annotations -from collections.abc import Collection +import math +from collections.abc import Collection, Sequence, Set from typing import Any, ClassVar, TYPE_CHECKING, get_origin from orcapod.core.operators.base import UnaryOperator @@ -23,20 +24,26 @@ def _null_last_key(value: Any) -> tuple[Any, ...]: - """Build an order key that sorts nulls last, at any list depth. + """Build an order key that sorts NaN after every number and nulls last, at any depth. - The leading integer keeps a null comparable against a real value of any - type. A list is keyed element by element, so a list-valued non-key tag - that holds a null (a list-typed tag column of a source) still compares. + The leading integer keeps a NaN and a null comparable against a real value of + any type. A NaN never compares with ``<``, so without its own place a sort that + meets one keeps the input's row order; here every NaN ties every other NaN and + the next sort column (the record id) breaks the tie (ITL-1226), as Arrow and + Polars order a NaN. A list is keyed element by element, so a list-valued tag + that holds a NaN or a null (a list-typed tag column of a source) still compares. Args: value: A tag or system-tag value: a scalar, ``None``, or a list of these nested to any depth. Returns: - A tuple that orders values ascending, with nulls after every value. + A tuple that orders values ascending, NaN after every number and nulls + after everything. """ if value is None: + return (2,) + if isinstance(value, float) and math.isnan(value): return (1,) if isinstance(value, list): return (0, tuple(_null_last_key(v) for v in value)) @@ -52,6 +59,71 @@ def _null_last_key(value: Any) -> tuple[Any, ...]: """ +MEMBER_ORDER_NAME = "name" +"""The keyword for the default member order: the non-key tags in name order.""" + + +def _member_order_names( + member_order: str | Sequence[str] | None, by: tuple[str, ...] +) -> tuple[str, ...] | None: + """Return the tags ``member_order`` names, or ``None`` for the default order. + + Args: + member_order: ``None`` or ``"name"`` (the default order), or a sequence of tag + names. + by: The group keys. + + Returns: + The names in the order given, or ``None`` for the default order. + + Raises: + NotImplementedError: For a function: an ordering function is planned, not + available yet. + TypeError: For a set (unordered), or a name that is not a string. + ValueError: For an unknown keyword, an empty sequence, a repeated name, or + a group key. + """ + if member_order is None: + return None + if isinstance(member_order, str): + if member_order == MEMBER_ORDER_NAME: + return None + raise ValueError( + f"GroupBy: member_order {member_order!r} is not a known order; the " + + f"keyword is {MEMBER_ORDER_NAME!r} (the default: the non-key tags in " + + "name order); to order by a tag, give a list of tag names: " + + f"member_order=[{member_order!r}]" + ) + if callable(member_order): + raise NotImplementedError( + "GroupBy: an ordering function for member_order is planned but not " + + "available yet (ITL-1226); give a list of tag names" + ) + if isinstance(member_order, Set): + raise TypeError( + "GroupBy: give member_order as a list or a tuple, not a set: its order " + + "is the order the members are sorted by" + ) + names = tuple(member_order) + if not names: + raise ValueError( + "GroupBy: member_order needs at least one tag name; leave it out for " + + "the default order" + ) + if not_strings := [name for name in names if not isinstance(name, str)]: + raise TypeError( + f"GroupBy: member_order names must be strings, got {not_strings!r}" + ) + if repeated := sorted({name for name in names if names.count(name) > 1}): + raise ValueError(f"GroupBy: member_order names {repeated} more than once") + if keys := [name for name in names if name in by]: + raise ValueError( + f"GroupBy: member_order names {keys}, a group key: a key has one value " + + "in a group, so it cannot order the members" + ) + return names + + class GroupBy(UnaryOperator): """Reduce rows sharing a key into one row with list-valued data members. @@ -85,14 +157,25 @@ class GroupBy(UnaryOperator): a data-tag at the source (the default), declare it as data, or use ``DerivedSource(origin, tags_as_data={...})``. - Members are sorted by the input's non-group-key tag values, with ``record_id`` - appended as a final tiebreaker, so the emitted lists are stable across - runs. The list of a data-tag non-key is therefore in its own value - order, and every other data list is aligned with it. This matters because orcapod hashes those lists to build the cache - key -- an unsorted list would make an identical member set hash differently - and trigger a spurious recompute. Groups themselves are emitted in - group-key order for the same reason: a reordered input must produce an - identical output. + Members are sorted by the input's non-group-key tags in name order, each + ascending with nulls last, then by the ``record_id`` columns and then the + source-id columns as tiebreakers, so the emitted lists are stable across + runs (ITL-1226). ``member_order`` names the tags to sort by first, in its + order; the other non-key tags follow in name order, then the same + tiebreakers. A member is never ordered by a data value. The order does + not depend on how the input's columns are arranged: the order of its tags, + the order of its data columns and the layout of its table change no value, + id or hash of the output (developer, November 17, 2026). The output lists + its columns as the input does: the keys in the input's tag order, the data + in the input's order; that is presentation, and no hash depends on it. + + The list of a data-tag non-key is in member order, and every other data + list is aligned with it. This matters because orcapod hashes those lists + to build the cache key -- an unsorted list would make an identical member + set hash differently and trigger a spurious recompute. Groups themselves + are emitted sorted by their keys, in name order, for the same reason: a + reordered input or a reordered ``by`` must + produce an identical output. Tag tuples are expected to be unique within a stream, but nothing enforces that. If the input does contain duplicate tag tuples, the members sharing @@ -103,8 +186,9 @@ class GroupBy(UnaryOperator): A grouped stream can be grouped again on a coarser key: its data lists nest one level deeper (``list[list[T]]``), as source info does. The members of the second grouping are ordered by the first grouping's keys that are not - in the coarser key (scalar tags of the grouped stream), then by - ``record_id``; a tag the first grouping dropped no longer takes part. A + in the coarser key (scalar tags of the grouped stream, in name order or as + ``member_order`` gives), then by ``record_id``; a tag the first grouping + dropped no longer takes part. A list-valued tag of the input can order members but cannot be a grouping key. @@ -113,16 +197,37 @@ class GroupBy(UnaryOperator): what tells a batch's members apart. Args: - by: Tag column names to group on, or one name as a string (a bare - string is one name, never its characters). Must be non-empty, free - of duplicates, and must all be scalar tag columns of the input - stream. + by: Tag column names to group on, in any order (a set too), or one + name as a string (a bare string is one name, never its characters). + Must be non-empty, free of duplicates, and must all be scalar tag + columns of the input stream. Their order is not kept: it is not part + of the identity, and the groups are emitted sorted by the keys in name + order (developer, November 17, 2026, ITL-1226). + member_order: The order of the members of a group. ``None`` (or + ``"name"``): by the non-key tags in name order. A list or tuple of + non-key tag names (plain tags or data-tags): by those tags in the + order given, then by the other non-key tags in name order. Part of + the operator's identity only when given. An ordering function is + planned (ITL-1226) and refused for now. + + Raises: + ValueError: If ``by`` is empty or repeats a name, or ``member_order`` is + an unknown keyword, empty, repeats a name or names a group key. + TypeError: If ``by`` is not a string or an iterable of strings, or + ``member_order`` is a set or holds a name that is not a string. + NotImplementedError: If ``member_order`` is a function. """ # Data-tags (ITL-1023): handled. DATA_TAGS_ISSUE: ClassVar[str | None] = None - def __init__(self, by: str | Collection[str], **kwargs: Any) -> None: + def __init__( + self, + by: str | Collection[str], + *, + member_order: str | Sequence[str] | None = None, + **kwargs: Any, + ) -> None: by_tuple = column_names(by) if not by_tuple: raise ValueError("GroupBy requires at least one column in `by`.") @@ -132,11 +237,19 @@ def __init__(self, by: str | Collection[str], **kwargs: Any) -> None: f"GroupBy `by` contains duplicate column names: {duplicates}. " f"Got {list(by_tuple)}." ) - self.by = by_tuple + # The order of the keys only orders the groups (rows), never a value + # within a row: it is not identity (developer, November 17, 2026). + self.by = tuple(sorted(by_tuple)) + self.member_order = _member_order_names(member_order, by_tuple) super().__init__(**kwargs) def identity_structure(self) -> Any: - return (self.__class__.__name__, self.by) + # ``member_order`` enters only when given, so a GroupBy without it keeps the + # identity it had before the option existed (ITL-1226). + identity: tuple[Any, ...] = (self.__class__.__name__, self.by) + if self.member_order is not None: + identity += (("member_order", self.member_order),) + return identity def to_config(self) -> dict[str, Any]: """Serialize this GroupBy operator to a config dict. @@ -146,10 +259,12 @@ def to_config(self) -> dict[str, Any]: Returns: A dict with ``class_name``, ``module_path``, and ``config`` keys, - where ``config`` contains ``by``. + where ``config`` contains ``by``, and ``member_order`` when given. """ config = super().to_config() config["config"] = {"by": list(self.by)} + if self.member_order is not None: + config["config"]["member_order"] = list(self.member_order) return config # ------------------------------------------------------------------ @@ -164,7 +279,8 @@ def validate_unary_input(self, stream: StreamProtocol) -> None: Raises: InputValidationError: If any name in ``by`` is not a tag column, or - names a list-valued tag column. + names a list-valued tag column; or a name in ``member_order`` is + not a tag column. """ tag_columns, data_columns = stream.keys() missing = [c for c in self.by if c not in tag_columns] @@ -191,6 +307,21 @@ def validate_unary_input(self, stream: StreamProtocol) -> None: "group before batching rather than after." ) + # The members are ordered by tags only, never by a data value. + for name in self.member_order or (): + if name in tag_columns: + continue + if name in data_columns: + raise InputValidationError( + f"GroupBy: member_order names {name!r}, which is a data column " + + "of the input, not a tag: members are ordered by tags only. " + + f"Make it a tag first: stream.tag_by({name!r})" + ) + raise InputValidationError( + f"GroupBy: member_order names {name!r}, which is not a tag column " + + f"of the input; its tags are {list(tag_columns)}" + ) + # ------------------------------------------------------------------ # Processing # ------------------------------------------------------------------ @@ -234,7 +365,9 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: and c not in key_source_columns and c not in dropped_columns ) - # Non-key user tags order the members of a group. The `record_id` + # Non-key user tags order the members of a group: those `member_order` + # names, in its order, then the others in name order, so the order never + # depends on the input's tag order (ITL-1226). The `record_id` # columns are appended as a final tiebreaker: tag tuples are supposed # to be unique within a stream, but nothing enforces that, and without # the tiebreaker duplicate tuples would fall back to emission order -- @@ -251,7 +384,8 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: # names a row within its source only (ITL-1154), so the source-id columns # come last: two members with equal record ids from different sources still # have one order. - sort_columns = non_key_tags + named = self.member_order or () + sort_columns = named + tuple(sorted(c for c in non_key_tags if c not in named)) for prefix in ( constants.SYSTEM_TAG_RECORD_ID_PREFIX, constants.SYSTEM_TAG_SOURCE_ID_PREFIX, diff --git a/src/orcapod/core/operators/tag_by.py b/src/orcapod/core/operators/tag_by.py index 0a827b981..9d50365b3 100644 --- a/src/orcapod/core/operators/tag_by.py +++ b/src/orcapod/core/operators/tag_by.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections.abc import Sequence, Set +from collections.abc import Iterable from typing import Any, ClassVar from typing_extensions import override @@ -18,25 +18,22 @@ from orcapod.utils.schema_utils import column_names -def _tag_by_names(columns: str | Sequence[str]) -> tuple[str, ...]: - """Return the names ``TagBy`` marks, in the order given. +def _tag_by_names(columns: str | Iterable[str]) -> tuple[str, ...]: + """Return the names ``TagBy`` marks, in name order. + + The order the names are given in is not kept (ITL-1226): it is part of neither + the identity nor the output. Args: - columns: One name, or a sequence of names. + columns: One name, or any iterable of names (a list, a tuple, a set). Returns: - The names, in the order given. + The names, sorted. Raises: - TypeError: If ``columns`` is a set (unordered: the order of the names is the - order of the new tags) or holds a name that is not a string. + TypeError: If a name is not a string. ValueError: If no name is given, or a name is repeated. """ - if isinstance(columns, Set): - raise TypeError( - "TagBy: give the column names as a list or a tuple, not a set: their " - + "order is the order of the new tags" - ) try: names = column_names(columns) except TypeError as exc: @@ -45,7 +42,7 @@ def _tag_by_names(columns: str | Sequence[str]) -> tuple[str, ...]: raise ValueError("TagBy: expected at least one column name") if repeated := sorted({name for name in names if names.count(name) > 1}): raise ValueError(f"TagBy: columns {repeated} are named more than once") - return names + return tuple(sorted(names)) class TagBy(UnaryOperator): @@ -56,7 +53,8 @@ class TagBy(UnaryOperator): the data columns keep their names, order, types, values and source info, the system tags keep their names and values (rule 1, name-preserving), and no row is added, removed or reordered. The new tags follow the input's user - tags, in the order given, and the column is stored once, where it was. + tags, in name order whatever the order they are given in (ITL-1226), and the + column is stored once, where it was. The operator creates no value and moves none: every output data column is the input data column of that name, and no tag becomes data. A data column that a @@ -64,11 +62,12 @@ class TagBy(UnaryOperator): key a join or a group. Args: - columns: One data column name, or a sequence of them. Their order is the - order of the new tags, and part of the operator's identity. + columns: One data column name, or any iterable of them (a list, a tuple, + a set). Their order is not kept: the new tags are in name order, and + the identity is the sorted names. Raises: - TypeError: If ``columns`` is a set or holds a name that is not a string. + TypeError: If ``columns`` holds a name that is not a string. ValueError: If no name is given, or a name is repeated. InputValidationError: If a name is reserved: the meta prefix, a system column name, or one holding ``::``. @@ -77,7 +76,7 @@ class TagBy(UnaryOperator): # Data-tags (ITL-1023): handled; this operator makes them. DATA_TAGS_ISSUE: ClassVar[str | None] = None - def __init__(self, columns: str | Sequence[str], **kwargs: Any) -> None: + def __init__(self, columns: str | Iterable[str], **kwargs: Any) -> None: names = _tag_by_names(columns) schema_utils.require_names_not_meta(names) schema_utils.require_names_not_system(names) @@ -87,10 +86,10 @@ def __init__(self, columns: str | Sequence[str], **kwargs: Any) -> None: @override def identity_structure(self) -> Any: - """Return the operator's identity: its class name and the names, in order. + """Return the operator's identity: its class name and the sorted names. Returns: - ``("TagBy", [names in the order given])``. + ``("TagBy", [names in name order])``. """ return (self.__class__.__name__, list(self.columns)) @@ -100,7 +99,7 @@ def to_config(self) -> dict[str, Any]: Returns: A dict with ``class_name``, ``module_path`` and ``config`` keys, where - ``config`` holds ``columns``, the names in order. + ``config`` holds ``columns``, the names in name order. """ config = super().to_config() config["config"] = {"columns": list(self.columns)} diff --git a/src/orcapod/core/streams/base.py b/src/orcapod/core/streams/base.py index 23aee6fab..f59904d5a 100644 --- a/src/orcapod/core/streams/base.py +++ b/src/orcapod/core/streams/base.py @@ -6,6 +6,7 @@ AsyncIterator, Callable, Collection, + Iterable, Iterator, Mapping, Sequence, @@ -509,25 +510,31 @@ def group_by( self, by: str | Collection[str], label: str | None = None, + *, + member_order: str | Sequence[str] | None = None, ) -> StreamBase: """Reduce rows sharing a key into one row per group. Group-key columns stay scalar tags and are the only tags of the output; every other tag is dropped, and every data column becomes list-valued. A dropped tag's values are not carried: make the column data before - grouping if a later step needs it. See + grouping if a later step needs it. The members of a group are ordered + by the non-key tags in name order, or as ``member_order`` gives. See ``orcapod.core.operators.GroupBy``. Args: by: Tag column names to group on, or one name as a string. label: Optional node label for the pipeline graph. + member_order: ``None`` or ``"name"`` (the non-key tags in name + order), or a list of non-key tag names to order the members by + first. Returns: A stream with one row per distinct group-key tuple. """ from orcapod.core.operators import GroupBy - return GroupBy(by=by)(self, label=label) + return GroupBy(by=by, member_order=member_order)(self, label=label) def polars_filter( self, @@ -574,27 +581,27 @@ def drop_nulls( def tag_by( self, - *columns: str | Sequence[str], + *columns: str | Iterable[str], label: str | None = None, ) -> StreamBase: """Give the named data columns the tag role, as data-tags. - ``s.tag_by("x", "y")`` equals ``s.tag_by(["x", "y"])``. Each named data + ``s.tag_by("x", "y")`` equals ``s.tag_by(["x", "y"])``, + ``s.tag_by({"y", "x"})`` and ``s.tag_by("y", "x")``. Each named data column becomes a data-tag: one column, both a tag and the data column it was, with its values and source info unchanged. The new tags follow this - stream's tags, in the order given. See ``orcapod.core.operators.TagBy``. + stream's tags, in name order. See ``orcapod.core.operators.TagBy``. Args: - *columns: One or more data column names, or one list or tuple of names - as the only positional argument (not a set: the order matters). + *columns: One or more data column names, or one iterable of names + (a list, a tuple, a set) as the only positional argument. label: Optional node label for the pipeline graph. Returns: The stream with the named columns as data-tags. Raises: - TypeError: When no name is given, the names are a set, or a name is not - a string. + TypeError: When a name is not a string. ValueError: When a name is repeated. InputValidationError: When a name is reserved, is not a data column of this stream, is already a tag, or has a type that cannot be a tag. @@ -602,7 +609,7 @@ def tag_by( from orcapod.core.operators import TagBy if len(columns) == 1 and not isinstance(columns[0], str): - names: str | Sequence[str] = columns[0] + names: str | Iterable[str] = columns[0] else: names = cast("tuple[str, ...]", columns) return TagBy(names)(self, label=label) diff --git a/src/orcapod/protocols/core_protocols/streams.py b/src/orcapod/protocols/core_protocols/streams.py index 3a940b726..1616bbf53 100644 --- a/src/orcapod/protocols/core_protocols/streams.py +++ b/src/orcapod/protocols/core_protocols/streams.py @@ -1,6 +1,13 @@ from __future__ import annotations -from collections.abc import AsyncIterator, Collection, Iterator, Mapping, Sequence +from collections.abc import ( + AsyncIterator, + Collection, + Iterable, + Iterator, + Mapping, + Sequence, +) from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable from orcapod.protocols.core_protocols.datagrams import DataProtocol, TagProtocol @@ -425,13 +432,13 @@ def drop_nulls( def tag_by( self, - *columns: str | Sequence[str], + *columns: str | Iterable[str], label: str | None = None, ) -> "StreamProtocol": """ Give the named data columns the tag role, as data-tags: each stays one column, a tag and the data column it was, with its values and source - info unchanged. The new tags follow the stream's tags, in the order given. + info unchanged. The new tags follow the stream's tags, in name order. """ ... diff --git a/tests/test_core/data_tags/test_tag_by.py b/tests/test_core/data_tags/test_tag_by.py index 00b4a5460..b549b8d11 100644 --- a/tests/test_core/data_tags/test_tag_by.py +++ b/tests/test_core/data_tags/test_tag_by.py @@ -2,7 +2,7 @@ A named data column becomes a data-tag: one column, a tag and the data column it was, with its values and source info unchanged. The new tags follow the input's tags in -the order given. Every operator is driven through its public call and read through the +name order, whatever order they are named in (ITL-1226). Every operator is driven through its public call and read through the stream's public methods; a schema prediction is checked against ``static_process``. """ @@ -10,6 +10,7 @@ import datetime import decimal +import json import warnings from pathlib import Path from typing import Any @@ -65,6 +66,15 @@ def session_of(trial: int) -> str: return f"s{trial // 20}" +def json_rows(table: pa.Table) -> str: + """The rows of ``table`` as one text, independent of row and column order.""" + return json.dumps( + sorted( + json.dumps(row, sort_keys=True, default=str) for row in table.to_pylist() + ) + ) + + def _sorted_rows(table: pa.Table) -> list[dict[str, Any]]: return sorted(table.to_pylist(), key=lambda row: repr(sorted(row.items()))) @@ -72,8 +82,8 @@ def _sorted_rows(table: pa.Table) -> list[dict[str, Any]]: # ── what the output is ────────────────────────────────────────────────────── -def test_named_data_columns_become_data_tags_in_the_order_given() -> None: - tagged = _plain().tag_by("label", "trial") +def test_named_data_columns_become_data_tags_in_name_order() -> None: + tagged = _plain().tag_by("trial", "label") assert tagged.keys() == ( ("unit", "label", "trial"), ("label", "trial", "amp"), @@ -114,29 +124,34 @@ def test_an_existing_data_tag_is_kept() -> None: assert tagged.keys() == (("unit", "trial"), ("unit", "trial", "amp", "label")) -def test_names_as_strings_or_one_sequence_are_one_operator() -> None: - source = _plain() - by_strings = source.tag_by("trial", "label") - by_list = source.tag_by(["trial", "label"]) - by_tuple = source.tag_by(("trial", "label")) - assert ( - by_strings.content_hash() == by_list.content_hash() == by_tuple.content_hash() - ) - assert by_strings.pipeline_hash() == by_list.pipeline_hash() - - -def test_the_order_of_the_names_is_part_of_the_identity() -> None: +def test_names_as_strings_a_sequence_or_a_set_in_any_order_are_one_operator() -> None: + """The order of the names is neither identity nor output (ITL-1226).""" source = _plain() - forward = source.tag_by("trial", "label") - backward = source.tag_by("label", "trial") - assert forward.keys()[0] == ("unit", "trial", "label") - assert backward.keys()[0] == ("unit", "label", "trial") - assert forward.content_hash() != backward.content_hash() - assert forward.pipeline_hash() != backward.pipeline_hash() + forms = [ + source.tag_by("trial", "label"), + source.tag_by("label", "trial"), + source.tag_by(["trial", "label"]), + source.tag_by(("label", "trial")), + source.tag_by({"trial", "label"}), + source.tag_by(frozenset({"label", "trial"})), + ] + assert len({s.content_hash() for s in forms}) == 1 + assert len({s.pipeline_hash() for s in forms}) == 1 + for stream in forms: + assert stream.keys() == ( + ("unit", "label", "trial"), + ("label", "trial", "amp"), + ) + assert stream_data_tags(stream) == ("label", "trial") + rows = {json_rows(s.as_table(all_info=True)) for s in forms} + assert len(rows) == 1 assert TagBy(["trial", "label"]).identity_structure() == ( "TagBy", - ["trial", "label"], + ["label", "trial"], ) + assert TagBy({"trial", "label"}).to_config()["config"] == { + "columns": ["label", "trial"] + } @pytest.mark.parametrize("all_info", [False, True]) @@ -269,8 +284,6 @@ def test_a_nullable_column_is_tagged_after_drop_nulls() -> None: @pytest.mark.parametrize( ("columns", "error"), [ - ({"trial", "label"}, TypeError), - (frozenset({"trial"}), TypeError), ([], ValueError), (["trial", "trial"], ValueError), ([1], TypeError), @@ -280,8 +293,6 @@ def test_a_nullable_column_is_tagged_after_drop_nulls() -> None: (["a::b"], InputValidationError), ], ids=[ - "set", - "frozenset", "empty", "repeated", "not_a_string", @@ -296,9 +307,7 @@ def test_bad_names_are_refused_at_construction(columns: Any, error: type) -> Non _ = TagBy(columns) -def test_the_stream_method_refuses_no_name_and_a_set() -> None: - with pytest.raises(TypeError): - _ = _plain().tag_by({"trial"}) # pyright: ignore[reportArgumentType] # the refusal is the test +def test_the_stream_method_refuses_no_name() -> None: with pytest.raises(ValueError, match="at least one"): _ = _plain().tag_by([]) diff --git a/tests/test_core/operators/test_column_name_arguments.py b/tests/test_core/operators/test_column_name_arguments.py index 8889cb12c..a7f6cc31b 100644 --- a/tests/test_core/operators/test_column_name_arguments.py +++ b/tests/test_core/operators/test_column_name_arguments.py @@ -107,9 +107,9 @@ def test_a_one_character_name_still_works(self): @pytest.mark.parametrize( "by", [["session", "subject"], ("session", "subject"), ["subject", "session"]] ) - def test_collections_keep_their_identity(self, by): - """A list or tuple keeps its order in the identity, as before.""" - assert GroupBy(by).identity_structure() == ("GroupBy", tuple(by)) + def test_collections_enter_the_identity_in_name_order(self, by): + """A list or tuple gives its keys; their order is not identity (ITL-1226).""" + assert GroupBy(by).identity_structure() == ("GroupBy", tuple(sorted(by))) def test_bytes_are_not_a_name(self): """``bytes`` is neither a name nor a collection of names, empty or not.""" diff --git a/tests/test_core/operators/test_group_by.py b/tests/test_core/operators/test_group_by.py index b43c29b53..72aeb9d1a 100644 --- a/tests/test_core/operators/test_group_by.py +++ b/tests/test_core/operators/test_group_by.py @@ -477,8 +477,9 @@ def test_identity_structure_includes_by(self): def test_to_config_round_trip(self): op = GroupBy(by=["subject", "date"]) config = op.to_config() - # A list, not a tuple, so the config stays JSON-serializable. - assert config["config"]["by"] == ["subject", "date"] + # A list, not a tuple, so the config stays JSON-serializable; the keys in + # name order, since their order is not identity (ITL-1226). + assert config["config"]["by"] == ["date", "subject"] rebuilt = GroupBy.from_config(config) assert rebuilt.identity_structure() == op.identity_structure() diff --git a/tests/test_core/operators/test_group_by_member_order.py b/tests/test_core/operators/test_group_by_member_order.py new file mode 100644 index 000000000..3bc81e901 --- /dev/null +++ b/tests/test_core/operators/test_group_by_member_order.py @@ -0,0 +1,475 @@ +"""The order of a group's members (ITL-1226). + +Developer decision, November 17, 2026: ``GroupBy`` never depends on how its input's +columns are arranged. Without ``member_order``, the members of a group are ordered by the +non-key tags in name order, then by the record id and source id columns. With +``member_order``, by the tags it names, in that order, then by the other non-key tags in +name order, then by the same tiebreaks. The option is part of the operator's identity only +when given. No member is ever ordered by a data value. + +For one set of rows, every arrangement of the input's tag and data columns (the order of +``tag_columns``, the physical order of the table, plain tags, data-tags and both, a source +and a raw stream) gives the same grouped rows: groups, member order in every list, the +folded record id and source id, the source-info tokens, the row hashes, and the stream's +content and pipeline hash. Only the output's column order follows the input: it is +presentation, and no hash depends on it. +""" + +from __future__ import annotations + +import itertools +import json +from collections.abc import Callable +from typing import Any, cast + +import pyarrow as pa +import pytest + +import orcapod as op +from orcapod.core.operators import GroupBy +from orcapod.core.streams import ArrowTableStream +from orcapod.databases import InMemoryArrowDatabase +from orcapod.errors import InputValidationError +from orcapod.pipeline import ( + AsyncPipelineOrchestrator, + PipelineJob, + SyncPipelineOrchestrator, +) +from orcapod.pipeline.serialization import resolve_operator_from_config +from orcapod.core.streams.base import StreamBase +from orcapod.protocols.core_protocols import StreamProtocol + +# ``x`` and ``y`` order the members differently; ``y`` and ``b`` hold nulls; rows 1 and +# 2 share every tag, so only the record id tells them apart. +COLUMNS: dict[str, pa.Array[Any]] = { + "g": pa.array([1, 1, 1, 1, 1, 1], pa.int64()), + "k": pa.array([1, 1, 1, 2, 2, 2], pa.int64()), + "x": pa.array([2, 1, 1, 1, 2, 1], pa.int64()), + "y": pa.array([1, 2, 2, None, 1, 3], pa.int64()), + "a": pa.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6], pa.float64()), + "b": pa.array(["p", "q", "r", "s", None, "u"], pa.large_string()), +} +TAGS = ("g", "k", "x", "y") +NOT_NULL = ("g", "k", "x") + +# A few physical layouts: tags first, data first, interleaved, reversed. +LAYOUTS: tuple[tuple[str, ...], ...] = ( + ("g", "k", "x", "y", "a", "b"), + ("a", "b", "y", "x", "k", "g"), + ("x", "a", "g", "b", "y", "k"), + ("b", "y", "a", "k", "x", "g"), +) +TAG_ORDERS = tuple(itertools.permutations(TAGS)) + +MODES = ("plain", "data_tags", "mixed") + + +def _table(layout: tuple[str, ...]) -> pa.Table: + table = pa.table({name: COLUMNS[name] for name in layout}) + return table.cast( + pa.schema(f.with_nullable(f.name not in NOT_NULL) for f in table.schema) + ) + + +def _input( + mode: str, kind: str, tag_order: tuple[str, ...], layout: tuple[str, ...] +) -> StreamBase: + table = _table(layout) + if kind == "raw": + data_tags = { + "plain": [], + "data_tags": list(tag_order), + "mixed": [t for t in tag_order if t != "y"], + }[mode] + return ArrowTableStream(table, tag_columns=list(tag_order), data_tags=data_tags) + tag_only: bool | list[str] = {"plain": True, "data_tags": False, "mixed": ["y"]}[ + mode + ] + return op.sources.ArrowTableSource( + table, tag_columns=list(tag_order), tag_only_columns=tag_only, source_id="s" + ) + + +def _fingerprint(stream: StreamProtocol, *, content_hash: bool) -> dict[str, Any]: + """Everything about a stream's rows and identity except its column order.""" + rows = sorted( + json.dumps(row, sort_keys=True, default=str) + for row in stream.as_table(all_info=True).to_pylist() + ) + row_hashes = sorted( + (tag.content_hash().to_string(), data.content_hash().to_string()) + for tag, data in stream.iter_data() + ) + out: dict[str, Any] = { + "rows": rows, + "row_hashes": row_hashes, + "pipeline_hash": stream.pipeline_hash().to_string(), + } + if content_hash: + out["content_hash"] = stream.content_hash().to_string() + return out + + +def _group_twice( + stream: StreamBase, member_order: list[str] | None +) -> tuple[StreamBase, StreamBase]: + once = stream.group_by(["g", "k"], member_order=member_order) + # The grouped stream's only non-key tag is ``k``. + twice = once.group_by(["g"], member_order=["k"] if member_order else None) + return once, twice + + +# ── no dependence on the arrangement ──────────────────────────────────────── + + +@pytest.mark.parametrize("member_order", [None, ["y"], ["y", "x"]]) +@pytest.mark.parametrize("kind", ["source", "raw"]) +@pytest.mark.parametrize("mode", MODES) +def test_no_arrangement_of_the_input_changes_the_result( + mode: str, kind: str, member_order: list[str] | None +) -> None: + # A raw stream's own content hash depends on its tag order (ITL-1227): only its + # values and pipeline hash are compared. + content_hash = kind == "source" + seen: dict[str, Any] = {} + for tag_order, layout in itertools.product(TAG_ORDERS, LAYOUTS): + once, twice = _group_twice(_input(mode, kind, tag_order, layout), member_order) + found = { + "once": _fingerprint(once, content_hash=content_hash), + "twice": _fingerprint(twice, content_hash=content_hash), + } + if not seen: + seen = found + assert found == seen, (tag_order, layout) + + +@pytest.mark.parametrize("member_order", [None, ["y"]]) +@pytest.mark.parametrize("mode", MODES) +def test_a_job_gives_the_same_rows_sync_and_async_for_any_arrangement( + mode: str, member_order: list[str] | None +) -> None: + seen: list[str] = [] + for tag_order, layout in [ + (TAGS, LAYOUTS[0]), + (("y", "x", "k", "g"), LAYOUTS[1]), + (("x", "g", "y", "k"), LAYOUTS[2]), + ]: + for orchestrator in (SyncPipelineOrchestrator, AsyncPipelineOrchestrator): + job = PipelineJob(name="order", database=InMemoryArrowDatabase()) + with job: + _ = _input(mode, "source", tag_order, layout).group_by( + ["g", "k"], member_order=member_order, label="grouped" + ) + job.run(orchestrator=orchestrator()) + rows = sorted( + json.dumps(row, sort_keys=True, default=str) + for row in job.nodes["grouped"].as_table(all_info=True).to_pylist() + ) + if not seen: + seen = rows + assert rows == seen, (tag_order, layout, orchestrator.__name__) + + +def test_the_output_column_order_follows_the_input() -> None: + """Presentation only: the keys keep the input's tag order.""" + forward = _input("plain", "source", ("g", "k", "x", "y"), LAYOUTS[0]) + backward = _input("plain", "source", ("k", "g", "y", "x"), LAYOUTS[3]) + assert forward.group_by(["g", "k"]).keys() == (("g", "k"), ("a", "b")) + assert backward.group_by(["g", "k"]).keys() == (("k", "g"), ("b", "a")) + + +@pytest.mark.parametrize("member_order", [None, ["y"]]) +@pytest.mark.parametrize("mode", MODES) +def test_the_order_of_by_is_not_identity( + mode: str, member_order: list[str] | None +) -> None: + """The keys' order only ordered the groups (rows): it is not identity. + + Developer, November 17, 2026: ``GroupBy(["g", "k"])`` and ``GroupBy(["k", "g"])`` + are one operator: equal identity, config, hashes, rows and row order, whatever + the arrangement of the input. + """ + seen: dict[str, Any] = {} + for by, tag_order, layout in itertools.product( + (["g", "k"], ["k", "g"], {"k", "g"}), (TAGS, ("y", "k", "x", "g")), LAYOUTS + ): + group_by = GroupBy(by, member_order=member_order) + grouped = group_by(_input(mode, "source", tag_order, layout)) + found = { + "identity": group_by.identity_structure(), + "config": group_by.to_config(), + "label": grouped.label, + "row_order": [(tag["g"], tag["k"]) for tag, _ in grouped.iter_data()], + **_fingerprint(grouped, content_hash=True), + } + if not seen: + seen = found + assert found == seen, (by, tag_order, layout) + assert seen["config"]["config"]["by"] == ["g", "k"] + + +def test_groups_are_emitted_by_their_keys_in_name_order() -> None: + rows = [{"g": g, "k": k, "a": float(10 * g + k)} for g in (2, 1) for k in (1, 2)] + source = op.sources.DictSource( + rows, tag_columns=["k", "g"], tag_only_columns=True, source_id="emit" + ) + for by in (["k", "g"], ["g", "k"]): + grouped = source.group_by(by) + assert [(t["g"], t["k"]) for t, _ in grouped.iter_data()] == [ + (1, 1), + (1, 2), + (2, 1), + (2, 2), + ] + + +# ── the order itself ──────────────────────────────────────────────────────── + + +def _a_lists(stream: StreamProtocol) -> dict[object, object]: + return {tag["k"]: data["a"] for tag, data in stream.iter_data()} + + +@pytest.mark.parametrize("tag_order", [TAGS, ("g", "k", "y", "x")]) +def test_by_default_members_are_ordered_by_the_non_key_tags_in_name_order( + tag_order: tuple[str, ...], +) -> None: + grouped = _input("plain", "source", tag_order, LAYOUTS[0]).group_by(["g", "k"]) + # By ``x``, then ``y`` (nulls last), then the record id (rows 1 and 2 tie). + assert _a_lists(grouped) == {1: [0.2, 0.3, 0.1], 2: [0.6, 0.4, 0.5]} + + +@pytest.mark.parametrize("tag_order", [TAGS, ("g", "k", "y", "x")]) +def test_member_order_names_the_tags_that_order_the_members( + tag_order: tuple[str, ...], +) -> None: + source = _input("plain", "source", tag_order, LAYOUTS[0]) + # By ``y`` (nulls last), then ``x``, then the record id. + expected = {1: [0.1, 0.2, 0.3], 2: [0.5, 0.6, 0.4]} + assert _a_lists(source.group_by(["g", "k"], member_order=["y"])) == expected + assert _a_lists(source.group_by(["g", "k"], member_order=["y", "x"])) == expected + # Naming them in name order is the default order, under its own identity. + assert _a_lists(source.group_by(["g", "k"], member_order=["x", "y"])) == { + 1: [0.2, 0.3, 0.1], + 2: [0.6, 0.4, 0.5], + } + + +def test_a_data_tag_orders_like_a_tag_and_its_list_follows_the_members() -> None: + grouped = _input("data_tags", "source", TAGS, LAYOUTS[0]).group_by( + ["g", "k"], member_order=["y"] + ) + rows = {tag["k"]: data for tag, data in grouped.iter_data()} + assert rows[1]["y"] == [1, 2, 2] and rows[1]["x"] == [2, 1, 1] + assert rows[2]["y"] == [1, 3, None] and rows[2]["x"] == [2, 1, 1] + + +def test_a_grouped_stream_grouped_again_orders_by_the_remaining_keys() -> None: + rows = [ + {"g": 1, "k": k, "x": x, "a": float(10 * k + x)} + for k in (2, 1, 3) + for x in (1, 2) + ] + source = op.sources.DictSource( + rows, tag_columns=["g", "k", "x"], tag_only_columns=True, source_id="nested" + ) + twice = source.group_by(["g", "k"]).group_by(["g"]) + ((_, data),) = list(twice.iter_data()) + assert data["a"] == [[11.0, 12.0], [21.0, 22.0], [31.0, 32.0]] + + +# ── identity ──────────────────────────────────────────────────────────────── + + +def test_without_member_order_the_identity_is_unchanged() -> None: + assert GroupBy(["k"]).identity_structure() == ("GroupBy", ("k",)) + assert GroupBy(["k"], member_order="name").identity_structure() == ( + "GroupBy", + ("k",), + ) + assert "member_order" not in GroupBy(["k"]).to_config()["config"] + + +def test_member_order_is_part_of_the_identity() -> None: + source = _input("plain", "source", TAGS, LAYOUTS[0]) + default = source.group_by(["g", "k"]) + by_y = source.group_by(["g", "k"], member_order=["y"]) + by_x = source.group_by(["g", "k"], member_order=["x"]) + hashes = {s.content_hash() for s in (default, by_y, by_x)} + assert len(hashes) == 3 + assert len({s.pipeline_hash() for s in (default, by_y, by_x)}) == 3 + assert GroupBy(["k"], member_order=["y", "x"]).identity_structure() == ( + "GroupBy", + ("k",), + ("member_order", ("y", "x")), + ) + # The keyword ``name`` is the default, identity included. + named = source.group_by(["g", "k"], member_order="name") + assert named.content_hash() == default.content_hash() + + +@pytest.mark.parametrize("member_order", [None, ["y", "x"]]) +def test_the_config_round_trips(member_order: list[str] | None) -> None: + group_by = GroupBy(["g", "k"], member_order=member_order) + config = group_by.to_config() + if member_order is None: + assert config["config"] == {"by": ["g", "k"]} + else: + assert config["config"] == {"by": ["g", "k"], "member_order": member_order} + rebuilt = resolve_operator_from_config(config) + assert isinstance(rebuilt, GroupBy) + assert rebuilt.identity_structure() == group_by.identity_structure() + + +# ── refusals ──────────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + ("member_order", "error", "match"), + [ + ([], ValueError, "at least one"), + (["y", "y"], ValueError, "more than once"), + ({"x", "y"}, TypeError, "not a set"), + ([1], TypeError, "strings"), + (["k"], ValueError, "group key"), + ("canonical", ValueError, "'name'"), + ("x", ValueError, r"a list of tag names: member_order=\['x'\]"), + ], + ids=[ + "empty", + "repeated", + "set", + "not_a_string", + "key", + "unknown_keyword", + "bare_tag_name", + ], +) +def test_bad_member_orders_are_refused_at_construction( + member_order: Any, # pyright: ignore[reportExplicitAny] # the refusal is the test + error: type[Exception], + match: str, +) -> None: + with pytest.raises(error, match=match): + _ = GroupBy(["g", "k"], member_order=member_order) + + +def test_an_ordering_function_is_not_available_yet() -> None: + order: Callable[[frozenset[str]], list[str]] = sorted + with pytest.raises(NotImplementedError, match="ITL-1226"): + _ = GroupBy(["k"], member_order=order) # pyright: ignore[reportArgumentType] # reserved form + + +@pytest.mark.parametrize( + ("name", "match"), + [ + ("a", r"'a', which is a data column.*tag_by\('a'\)"), + ("nope", "'nope', which is not a tag"), + ], +) +def test_a_name_that_is_not_a_tag_of_the_input_is_refused( + name: str, match: str +) -> None: + source = _input("plain", "source", TAGS, LAYOUTS[0]) + with pytest.raises(InputValidationError, match=match): + _ = source.group_by(["g", "k"], member_order=[name]) + + +# ── NaN in a tag (ITL-1226, CodeRabbit on #434) ───────────────────────────── + +NAN = float("nan") +# ``x`` holds NaN at the top level, ``y`` inside a list; ``rid`` names each row, so +# reordering the rows leaves every record id as it is. +NAN_ROWS: list[dict[str, Any]] = [ + {"g": "a", "x": 0.0, "y": [1.0, NAN], "rid": "r1", "v": 1}, + {"g": "a", "x": NAN, "y": [NAN], "rid": "r2", "v": 2}, + {"g": "a", "x": 1.0, "y": [1.0, 0.0], "rid": "r3", "v": 3}, + {"g": "a", "x": NAN, "y": [1.0, None], "rid": "r4", "v": 4}, + {"g": "a", "x": None, "y": [NAN, 1.0], "rid": "r5", "v": 5}, +] + + +def _nan_source(rows: list[dict[str, Any]]) -> StreamBase: + table = pa.table( + { + "g": [r["g"] for r in rows], + "x": pa.array([r["x"] for r in rows], pa.float64()), + "y": pa.array([r["y"] for r in rows], pa.list_(pa.float64())), + "rid": [r["rid"] for r in rows], + "v": [r["v"] for r in rows], + } + ) + return op.sources.ArrowTableSource( + table, + tag_columns=["g", "x", "y"], + tag_only_columns=True, + source_id="s", + record_id_columns="rid", + ) + + +@pytest.mark.parametrize( + ("member_order", "expected"), + [ + # x: 0.0, 1.0, then the two NaN (tied; y breaks the tie), then null. + (None, [1, 3, 4, 2, 5]), + # y element by element: NaN after every number, null after NaN, and a + # shorter list before a longer one it starts. + (["y"], [3, 1, 4, 2, 5]), + ], + ids=["default", "member_order_y"], +) +def test_a_nan_tag_has_one_place_in_the_member_order( + member_order: list[str] | None, expected: list[int] +) -> None: + """NaN sorts after every number and before null, at any depth, whatever the row order. + + Without its own place a NaN compares with nothing, and the sort kept the input's + row order: the lists, folds and hashes then depended on it. + """ + seen: dict[str, Any] = {} + for rows in itertools.permutations(NAN_ROWS): + grouped = _nan_source(list(rows)).group_by(["g"], member_order=member_order) + assert grouped.as_table().column("v").to_pylist() == [expected] + fingerprint = _fingerprint(grouped, content_hash=True) + if not seen: + seen = fingerprint + assert fingerprint == seen + + +def test_a_nan_tag_orders_the_same_sync_and_async() -> None: + seen: list[str] = [] + for rows in (NAN_ROWS, NAN_ROWS[::-1], NAN_ROWS[2:] + NAN_ROWS[:2]): + for orchestrator in (SyncPipelineOrchestrator, AsyncPipelineOrchestrator): + job = PipelineJob(name="nan", database=InMemoryArrowDatabase()) + with job: + _ = _nan_source(rows).group_by(["g"], label="grouped") + job.run(orchestrator=orchestrator()) + out = job.nodes["grouped"].as_table(all_info=True) + assert out.column("v").to_pylist() == [[1, 3, 4, 2, 5]] + row = json.dumps(out.to_pylist(), sort_keys=True, default=str) + if not seen: + seen = [row] + assert row == seen[0], orchestrator.__name__ + + +def test_a_nan_group_key_is_a_group_of_its_own_row() -> None: + """Pins today's behaviour, unchanged here: NaN never equals NaN as a key. + + Each row whose key is NaN is a group of its own; the groups with a number come + first, in key order, then the NaN groups. + """ + nan = float("nan") + for values in ([0.0, nan, 1.0, nan], [nan, 1.0, nan, 0.0]): + source = op.sources.ArrowTableSource( + pa.table({"g": pa.array(values, pa.float64()), "v": [1, 2, 3, 4]}), + tag_columns=["g"], + tag_only_columns=True, + source_id="s", + ) + out = source.group_by(["g"]).as_table() + keys = out.column("g").to_pylist() + assert keys[:2] == [0.0, 1.0] + assert all(k != k for k in keys[2:]) and len(keys) == 4 + members = cast(list[list[int]], out.column("v").to_pylist()) + assert [len(m) for m in members] == [1, 1, 1, 1] diff --git a/tests/test_hashing/golden/cases.py b/tests/test_hashing/golden/cases.py index de1815e93..add9bcb9e 100644 --- a/tests/test_hashing/golden/cases.py +++ b/tests/test_hashing/golden/cases.py @@ -2565,3 +2565,89 @@ def semantic_annotated_functions() -> dict[str, Any]: out[f"{name}.info"] = hasher.hash_object(info).to_string() out[f"{name}.object"] = hasher.hash_object(fn).to_string() return out + + +# --------------------------------------------------------------------------- +# GroupBy over several tags and several data columns (ITL-1226) +# --------------------------------------------------------------------------- + +# Two non-key tags, ``x`` and ``y``, whose orders disagree; ``y`` and ``b`` hold nulls; +# the rows 1 and 2 share every tag, so only the record id orders them. +GROUP_MEMBER_ORDER_FIELDS = { + "g": (pa.large_string(), False, ["a", "a", "a", "a", "a", "a"]), + "k": (pa.int64(), False, [1, 1, 1, 2, 2, 2]), + "x": (pa.int64(), False, [2, 1, 1, 1, 2, 1]), + "y": (pa.large_string(), True, ["p", "q", "q", None, "p", "r"]), + "w": (pa.float64(), False, [0.5, 1.5, 2.5, 3.5, 4.5, 5.5]), + "n": (pa.int32(), True, [10, 20, 30, 40, None, 60]), + "s": (pa.large_string(), False, ["u", "v", "w", "x", "y", "z"]), +} + + +def _group_member_order_source(tag_columns: list[str], layout: list[str]) -> Any: # pyright: ignore[reportExplicitAny] # a source + """One set of rows, with the tags declared and the table laid out as given.""" + from orcapod.sources import ArrowTableSource + + fields = GROUP_MEMBER_ORDER_FIELDS + table = pa.table( + {name: pa.array(fields[name][2], fields[name][0]) for name in layout}, + schema=pa.schema( + [ + pa.field(name, fields[name][0], nullable=fields[name][1]) + for name in layout + ] + ), + ) + return ArrowTableSource( + table, + tag_columns=tag_columns, + tag_only_columns=True, + source_id="golden_group_member_order", + ) + + +GROUP_MEMBER_ORDER_ARRANGEMENTS: dict[str, tuple[list[str], list[str], list[str]]] = { + # name: (tag_columns, table layout, by) + "tags_first": ( + ["g", "k", "x", "y"], + ["g", "k", "x", "y", "w", "n", "s"], + ["g", "k"], + ), + "reversed": (["y", "x", "k", "g"], ["s", "n", "w", "y", "x", "k", "g"], ["k", "g"]), + "interleaved": ( + ["k", "y", "g", "x"], + ["w", "k", "s", "y", "n", "g", "x"], + ["g", "k"], + ), +} +"""Arrangements of one input whose ``GroupBy`` entries must be equal (ITL-1226): +``tests/test_hashing/golden/test_group_by_arrangements.py`` asserts it.""" + + +@case("pipeline", "group_by_arrangement") +def pipeline_group_by_arrangement() -> dict[str, Any]: + """Pin ``GroupBy`` over several tags and data columns, in several arrangements. + + The default member order (the non-key tags in name order, then the record id), + an explicit ``member_order``, and a grouped stream grouped again. Every + arrangement of the input's columns and of ``by`` pins the same values. + """ + from orcapod.operators import GroupBy + + out: dict[str, Any] = {} + for name, (tags, layout, by) in GROUP_MEMBER_ORDER_ARRANGEMENTS.items(): + source = _group_member_order_source(tags, layout) + out.update(_prefixed(f"default.{name}", stream_summary(GroupBy(by=by)(source)))) + out.update( + _prefixed( + f"member_order_y.{name}", + stream_summary(GroupBy(by=by, member_order=["y"])(source)), + ) + ) + out.update( + _prefixed( + f"grouped_again.{name}", + stream_summary(GroupBy(by=["g"])(GroupBy(by=by)(source))), + ) + ) + return out diff --git a/tests/test_hashing/golden/data/v1/pipeline-02.json b/tests/test_hashing/golden/data/v1/pipeline-02.json index d82150c16..194ba8743 100644 --- a/tests/test_hashing/golden/data/v1/pipeline-02.json +++ b/tests/test_hashing/golden/data/v1/pipeline-02.json @@ -36,5 +36,104 @@ "_tag_record_id::a4a91ac74561d231cff4ab05fdb7e199d259e8765f3af954e23d2c152b564b4f" ], "add_tags.operator[every_type].content_hash": "semantic_v1:6facb9a731655e8e6a695bbb96bb7902f6b212562f7df2b5207508ff3ab9e03e", - "add_tags.operator[every_type].pipeline_hash": "semantic_v1:6facb9a731655e8e6a695bbb96bb7902f6b212562f7df2b5207508ff3ab9e03e" + "add_tags.operator[every_type].pipeline_hash": "semantic_v1:6facb9a731655e8e6a695bbb96bb7902f6b212562f7df2b5207508ff3ab9e03e", + "group_by_arrangement.default.interleaved.content_hash": "semantic_v1:33994dc5eb4e5fdc0797c5d090fd91b3b8bfa7dd138763d944bbb390acb848d0", + "group_by_arrangement.default.interleaved.pipeline_hash": "semantic_v1:049a1fb3744027bea2a31ca70c960364adf513667b3c09f30441646c43b157f8", + "group_by_arrangement.default.interleaved.row[{\"g\":\"a\",\"k\":1}]._tag_record_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8": "group_v1:8c07899443ee6fcd1c5063e7b3b539372e18ac38101ddbb53cbb8c35a55bf877", + "group_by_arrangement.default.interleaved.row[{\"g\":\"a\",\"k\":1}]._tag_source_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8": "group_v1:16e676e585285bc478dcd6b8c2ecb19c8aeebc9e125505b32ac6be7283062a32", + "group_by_arrangement.default.interleaved.row[{\"g\":\"a\",\"k\":1}].data_hash": "arrow_v1:00000283c4201a810f7effa3ab65684519f59e751e2c1fcedbe8e1633d6c3b24e209bd", + "group_by_arrangement.default.interleaved.row[{\"g\":\"a\",\"k\":2}]._tag_record_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8": "group_v1:97aac73352f010c73664dd2e869a00e6a7e59644036afffa8850d67400652d09", + "group_by_arrangement.default.interleaved.row[{\"g\":\"a\",\"k\":2}]._tag_source_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8": "group_v1:16e676e585285bc478dcd6b8c2ecb19c8aeebc9e125505b32ac6be7283062a32", + "group_by_arrangement.default.interleaved.row[{\"g\":\"a\",\"k\":2}].data_hash": "arrow_v1:000002dfe264b0dcfd043bef99f03cbae4a6b0959b5dea7e3faf08824f84a8ef17e215", + "group_by_arrangement.default.interleaved.system_tag_columns": [ + "_tag_source_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8", + "_tag_record_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8" + ], + "group_by_arrangement.default.reversed.content_hash": "semantic_v1:33994dc5eb4e5fdc0797c5d090fd91b3b8bfa7dd138763d944bbb390acb848d0", + "group_by_arrangement.default.reversed.pipeline_hash": "semantic_v1:049a1fb3744027bea2a31ca70c960364adf513667b3c09f30441646c43b157f8", + "group_by_arrangement.default.reversed.row[{\"g\":\"a\",\"k\":1}]._tag_record_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8": "group_v1:8c07899443ee6fcd1c5063e7b3b539372e18ac38101ddbb53cbb8c35a55bf877", + "group_by_arrangement.default.reversed.row[{\"g\":\"a\",\"k\":1}]._tag_source_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8": "group_v1:16e676e585285bc478dcd6b8c2ecb19c8aeebc9e125505b32ac6be7283062a32", + "group_by_arrangement.default.reversed.row[{\"g\":\"a\",\"k\":1}].data_hash": "arrow_v1:00000283c4201a810f7effa3ab65684519f59e751e2c1fcedbe8e1633d6c3b24e209bd", + "group_by_arrangement.default.reversed.row[{\"g\":\"a\",\"k\":2}]._tag_record_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8": "group_v1:97aac73352f010c73664dd2e869a00e6a7e59644036afffa8850d67400652d09", + "group_by_arrangement.default.reversed.row[{\"g\":\"a\",\"k\":2}]._tag_source_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8": "group_v1:16e676e585285bc478dcd6b8c2ecb19c8aeebc9e125505b32ac6be7283062a32", + "group_by_arrangement.default.reversed.row[{\"g\":\"a\",\"k\":2}].data_hash": "arrow_v1:000002dfe264b0dcfd043bef99f03cbae4a6b0959b5dea7e3faf08824f84a8ef17e215", + "group_by_arrangement.default.reversed.system_tag_columns": [ + "_tag_source_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8", + "_tag_record_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8" + ], + "group_by_arrangement.default.tags_first.content_hash": "semantic_v1:33994dc5eb4e5fdc0797c5d090fd91b3b8bfa7dd138763d944bbb390acb848d0", + "group_by_arrangement.default.tags_first.pipeline_hash": "semantic_v1:049a1fb3744027bea2a31ca70c960364adf513667b3c09f30441646c43b157f8", + "group_by_arrangement.default.tags_first.row[{\"g\":\"a\",\"k\":1}]._tag_record_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8": "group_v1:8c07899443ee6fcd1c5063e7b3b539372e18ac38101ddbb53cbb8c35a55bf877", + "group_by_arrangement.default.tags_first.row[{\"g\":\"a\",\"k\":1}]._tag_source_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8": "group_v1:16e676e585285bc478dcd6b8c2ecb19c8aeebc9e125505b32ac6be7283062a32", + "group_by_arrangement.default.tags_first.row[{\"g\":\"a\",\"k\":1}].data_hash": "arrow_v1:00000283c4201a810f7effa3ab65684519f59e751e2c1fcedbe8e1633d6c3b24e209bd", + "group_by_arrangement.default.tags_first.row[{\"g\":\"a\",\"k\":2}]._tag_record_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8": "group_v1:97aac73352f010c73664dd2e869a00e6a7e59644036afffa8850d67400652d09", + "group_by_arrangement.default.tags_first.row[{\"g\":\"a\",\"k\":2}]._tag_source_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8": "group_v1:16e676e585285bc478dcd6b8c2ecb19c8aeebc9e125505b32ac6be7283062a32", + "group_by_arrangement.default.tags_first.row[{\"g\":\"a\",\"k\":2}].data_hash": "arrow_v1:000002dfe264b0dcfd043bef99f03cbae4a6b0959b5dea7e3faf08824f84a8ef17e215", + "group_by_arrangement.default.tags_first.system_tag_columns": [ + "_tag_source_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8", + "_tag_record_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8" + ], + "group_by_arrangement.grouped_again.interleaved.content_hash": "semantic_v1:2e82bc15f962fc2ed103e7a87d679881b8935b5d2342a98597599d5dd4c59217", + "group_by_arrangement.grouped_again.interleaved.pipeline_hash": "semantic_v1:1b13b4add9113d32ea8950cfd695c9dfb0cb960f72e891ba0012437e6782c58e", + "group_by_arrangement.grouped_again.interleaved.row[{\"g\":\"a\"}]._tag_record_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::049a1fb3744027bea2a31ca70c960364adf513667b3c09f30441646c43b157f8": "group_v1:0fb6d3b0875152e5728a67b1a53acf903ff1b2e5463ef866703f5bf7d39de6f9", + "group_by_arrangement.grouped_again.interleaved.row[{\"g\":\"a\"}]._tag_source_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::049a1fb3744027bea2a31ca70c960364adf513667b3c09f30441646c43b157f8": "group_v1:f5c2827c3bf542429f710e3f600266c761655fc041c5ca777a68f931b4ebfabe", + "group_by_arrangement.grouped_again.interleaved.row[{\"g\":\"a\"}].data_hash": "arrow_v1:00000205e0ff834a21a6cfd774cc736ab49e7b966de222bcb1af1804171adcde6554a3", + "group_by_arrangement.grouped_again.interleaved.system_tag_columns": [ + "_tag_source_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::049a1fb3744027bea2a31ca70c960364adf513667b3c09f30441646c43b157f8", + "_tag_record_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::049a1fb3744027bea2a31ca70c960364adf513667b3c09f30441646c43b157f8" + ], + "group_by_arrangement.grouped_again.reversed.content_hash": "semantic_v1:2e82bc15f962fc2ed103e7a87d679881b8935b5d2342a98597599d5dd4c59217", + "group_by_arrangement.grouped_again.reversed.pipeline_hash": "semantic_v1:1b13b4add9113d32ea8950cfd695c9dfb0cb960f72e891ba0012437e6782c58e", + "group_by_arrangement.grouped_again.reversed.row[{\"g\":\"a\"}]._tag_record_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::049a1fb3744027bea2a31ca70c960364adf513667b3c09f30441646c43b157f8": "group_v1:0fb6d3b0875152e5728a67b1a53acf903ff1b2e5463ef866703f5bf7d39de6f9", + "group_by_arrangement.grouped_again.reversed.row[{\"g\":\"a\"}]._tag_source_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::049a1fb3744027bea2a31ca70c960364adf513667b3c09f30441646c43b157f8": "group_v1:f5c2827c3bf542429f710e3f600266c761655fc041c5ca777a68f931b4ebfabe", + "group_by_arrangement.grouped_again.reversed.row[{\"g\":\"a\"}].data_hash": "arrow_v1:00000205e0ff834a21a6cfd774cc736ab49e7b966de222bcb1af1804171adcde6554a3", + "group_by_arrangement.grouped_again.reversed.system_tag_columns": [ + "_tag_source_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::049a1fb3744027bea2a31ca70c960364adf513667b3c09f30441646c43b157f8", + "_tag_record_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::049a1fb3744027bea2a31ca70c960364adf513667b3c09f30441646c43b157f8" + ], + "group_by_arrangement.grouped_again.tags_first.content_hash": "semantic_v1:2e82bc15f962fc2ed103e7a87d679881b8935b5d2342a98597599d5dd4c59217", + "group_by_arrangement.grouped_again.tags_first.pipeline_hash": "semantic_v1:1b13b4add9113d32ea8950cfd695c9dfb0cb960f72e891ba0012437e6782c58e", + "group_by_arrangement.grouped_again.tags_first.row[{\"g\":\"a\"}]._tag_record_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::049a1fb3744027bea2a31ca70c960364adf513667b3c09f30441646c43b157f8": "group_v1:0fb6d3b0875152e5728a67b1a53acf903ff1b2e5463ef866703f5bf7d39de6f9", + "group_by_arrangement.grouped_again.tags_first.row[{\"g\":\"a\"}]._tag_source_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::049a1fb3744027bea2a31ca70c960364adf513667b3c09f30441646c43b157f8": "group_v1:f5c2827c3bf542429f710e3f600266c761655fc041c5ca777a68f931b4ebfabe", + "group_by_arrangement.grouped_again.tags_first.row[{\"g\":\"a\"}].data_hash": "arrow_v1:00000205e0ff834a21a6cfd774cc736ab49e7b966de222bcb1af1804171adcde6554a3", + "group_by_arrangement.grouped_again.tags_first.system_tag_columns": [ + "_tag_source_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::049a1fb3744027bea2a31ca70c960364adf513667b3c09f30441646c43b157f8", + "_tag_record_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::049a1fb3744027bea2a31ca70c960364adf513667b3c09f30441646c43b157f8" + ], + "group_by_arrangement.member_order_y.interleaved.content_hash": "semantic_v1:6fbead157dda03d93842659d9317fcb3a3f7d6a63a818a1d582b41709c7c0857", + "group_by_arrangement.member_order_y.interleaved.pipeline_hash": "semantic_v1:217d48e4d36b3156a78fa296917976eb1ff00bf3fa82f63cbb3778a7eb1085c1", + "group_by_arrangement.member_order_y.interleaved.row[{\"g\":\"a\",\"k\":1}]._tag_record_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8": "group_v1:e728179de208af5490ea26a2e109bfad5878e0c1681232c81e61f846484d257e", + "group_by_arrangement.member_order_y.interleaved.row[{\"g\":\"a\",\"k\":1}]._tag_source_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8": "group_v1:16e676e585285bc478dcd6b8c2ecb19c8aeebc9e125505b32ac6be7283062a32", + "group_by_arrangement.member_order_y.interleaved.row[{\"g\":\"a\",\"k\":1}].data_hash": "arrow_v1:0000021c73412513e56fa02758fa9766402935bfad6b8833a7cd3668390450d36985a0", + "group_by_arrangement.member_order_y.interleaved.row[{\"g\":\"a\",\"k\":2}]._tag_record_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8": "group_v1:b1b330833a7e0d58dc74582079e759f5952b70e582e13a745120a0e36712bde4", + "group_by_arrangement.member_order_y.interleaved.row[{\"g\":\"a\",\"k\":2}]._tag_source_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8": "group_v1:16e676e585285bc478dcd6b8c2ecb19c8aeebc9e125505b32ac6be7283062a32", + "group_by_arrangement.member_order_y.interleaved.row[{\"g\":\"a\",\"k\":2}].data_hash": "arrow_v1:0000028eb8899531b475f6756bdf46d8834637915ec691ca020565db8fc85cf46adaa1", + "group_by_arrangement.member_order_y.interleaved.system_tag_columns": [ + "_tag_source_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8", + "_tag_record_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8" + ], + "group_by_arrangement.member_order_y.reversed.content_hash": "semantic_v1:6fbead157dda03d93842659d9317fcb3a3f7d6a63a818a1d582b41709c7c0857", + "group_by_arrangement.member_order_y.reversed.pipeline_hash": "semantic_v1:217d48e4d36b3156a78fa296917976eb1ff00bf3fa82f63cbb3778a7eb1085c1", + "group_by_arrangement.member_order_y.reversed.row[{\"g\":\"a\",\"k\":1}]._tag_record_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8": "group_v1:e728179de208af5490ea26a2e109bfad5878e0c1681232c81e61f846484d257e", + "group_by_arrangement.member_order_y.reversed.row[{\"g\":\"a\",\"k\":1}]._tag_source_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8": "group_v1:16e676e585285bc478dcd6b8c2ecb19c8aeebc9e125505b32ac6be7283062a32", + "group_by_arrangement.member_order_y.reversed.row[{\"g\":\"a\",\"k\":1}].data_hash": "arrow_v1:0000021c73412513e56fa02758fa9766402935bfad6b8833a7cd3668390450d36985a0", + "group_by_arrangement.member_order_y.reversed.row[{\"g\":\"a\",\"k\":2}]._tag_record_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8": "group_v1:b1b330833a7e0d58dc74582079e759f5952b70e582e13a745120a0e36712bde4", + "group_by_arrangement.member_order_y.reversed.row[{\"g\":\"a\",\"k\":2}]._tag_source_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8": "group_v1:16e676e585285bc478dcd6b8c2ecb19c8aeebc9e125505b32ac6be7283062a32", + "group_by_arrangement.member_order_y.reversed.row[{\"g\":\"a\",\"k\":2}].data_hash": "arrow_v1:0000028eb8899531b475f6756bdf46d8834637915ec691ca020565db8fc85cf46adaa1", + "group_by_arrangement.member_order_y.reversed.system_tag_columns": [ + "_tag_source_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8", + "_tag_record_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8" + ], + "group_by_arrangement.member_order_y.tags_first.content_hash": "semantic_v1:6fbead157dda03d93842659d9317fcb3a3f7d6a63a818a1d582b41709c7c0857", + "group_by_arrangement.member_order_y.tags_first.pipeline_hash": "semantic_v1:217d48e4d36b3156a78fa296917976eb1ff00bf3fa82f63cbb3778a7eb1085c1", + "group_by_arrangement.member_order_y.tags_first.row[{\"g\":\"a\",\"k\":1}]._tag_record_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8": "group_v1:e728179de208af5490ea26a2e109bfad5878e0c1681232c81e61f846484d257e", + "group_by_arrangement.member_order_y.tags_first.row[{\"g\":\"a\",\"k\":1}]._tag_source_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8": "group_v1:16e676e585285bc478dcd6b8c2ecb19c8aeebc9e125505b32ac6be7283062a32", + "group_by_arrangement.member_order_y.tags_first.row[{\"g\":\"a\",\"k\":1}].data_hash": "arrow_v1:0000021c73412513e56fa02758fa9766402935bfad6b8833a7cd3668390450d36985a0", + "group_by_arrangement.member_order_y.tags_first.row[{\"g\":\"a\",\"k\":2}]._tag_record_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8": "group_v1:b1b330833a7e0d58dc74582079e759f5952b70e582e13a745120a0e36712bde4", + "group_by_arrangement.member_order_y.tags_first.row[{\"g\":\"a\",\"k\":2}]._tag_source_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8": "group_v1:16e676e585285bc478dcd6b8c2ecb19c8aeebc9e125505b32ac6be7283062a32", + "group_by_arrangement.member_order_y.tags_first.row[{\"g\":\"a\",\"k\":2}].data_hash": "arrow_v1:0000028eb8899531b475f6756bdf46d8834637915ec691ca020565db8fc85cf46adaa1", + "group_by_arrangement.member_order_y.tags_first.system_tag_columns": [ + "_tag_source_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8", + "_tag_record_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8" + ] } diff --git a/tests/test_hashing/golden/test_group_by_arrangements.py b/tests/test_hashing/golden/test_group_by_arrangements.py new file mode 100644 index 000000000..ffaa74b7b --- /dev/null +++ b/tests/test_hashing/golden/test_group_by_arrangements.py @@ -0,0 +1,50 @@ +"""The golden ``GroupBy`` arrangements pin one set of values (ITL-1226). + +``pipeline_group_by_arrangement`` in ``cases.py`` pins ``GroupBy`` over one input in +several arrangements of its tag and data columns and of ``by``. GroupBy does not depend on +the arrangement, so every arrangement's entries must be equal; an explicit +``member_order`` gives other values. +""" + +from __future__ import annotations + +from typing import Any + +from tests.test_hashing.golden.cases import ( + GROUP_MEMBER_ORDER_ARRANGEMENTS, + pipeline_group_by_arrangement, +) + + +def _by_arrangement(entries: dict[str, Any], kind: str) -> dict[str, dict[str, Any]]: + """The entries of ``kind``, per arrangement, with the arrangement name removed.""" + out: dict[str, dict[str, Any]] = {} + for name in GROUP_MEMBER_ORDER_ARRANGEMENTS: + prefix = f"{kind}.{name}." + out[name] = { + key.removeprefix(prefix): value + for key, value in entries.items() + if key.startswith(prefix) + } + return out + + +def test_every_arrangement_pins_the_same_values() -> None: + entries = pipeline_group_by_arrangement() + for kind in ("default", "member_order_y", "grouped_again"): + per_arrangement = _by_arrangement(entries, kind) + first, *others = per_arrangement.values() + assert first + for other in others: + assert other == first, kind + + +def test_an_explicit_member_order_pins_other_values() -> None: + entries = pipeline_group_by_arrangement() + default = _by_arrangement(entries, "default")["tags_first"] + by_y = _by_arrangement(entries, "member_order_y")["tags_first"] + assert default.keys() == by_y.keys() + assert default["content_hash"] != by_y["content_hash"] + data_hashes = [k for k in default if k.endswith(".data_hash")] + assert data_hashes + assert all(default[k] != by_y[k] for k in data_hashes)