From d6377e044fbd2aa0ef1e35acfc1c8601734c74e8 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 17 Nov 2026 21:09:38 -0800 Subject: [PATCH 1/4] fix(group_by): all NaN keys form one group; a float key is emitted by its smallest bits (ITL-1234) GroupBy now groups the rows whose key is NaN into one group, as the joins match a NaN key; key tuples compare element by element with that rule. The NaN group sorts after every number and before null. A group whose members hold a float key with different bits (0.0 and -0.0, NaN with another sign or payload) emits, per key column, the member value with the smallest bit pattern at the column's own width, so neither the row order nor member_order decides the emitted key or its hashes. Golden: one case added (12 entries, additions only); no value moves. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 11 +- CLAUDE.md | 11 +- changelog.d/ITL-1234.md | 16 ++ docs/concepts/operators.md | 12 +- src/orcapod/contexts/data/v1.json | 4 +- src/orcapod/core/operators/group_by.py | 104 +++++++- .../operators/test_group_by_member_order.py | 24 +- .../operators/test_group_by_nan_keys.py | 232 ++++++++++++++++++ tests/test_hashing/golden/cases.py | 40 +++ .../golden/data/v1/pipeline-02.json | 15 ++ 10 files changed, 435 insertions(+), 34 deletions(-) create mode 100644 changelog.d/ITL-1234.md create mode 100644 tests/test_core/operators/test_group_by_nan_keys.py diff --git a/.zed/rules b/.zed/rules index 25da30d1b..1b5c41e0c 100644 --- a/.zed/rules +++ b/.zed/rules @@ -1180,7 +1180,7 @@ data. The rules: key orders its members by the first grouping's keys outside the coarser key, then by record id. An opt-in mode that keeps non-key tags is ITL-1071. - GroupBy never depends on how its input's columns are arranged (developer, November 17, 2026, ITL-1226). Members are ordered by the non-key tags in name order (each ascending, - nulls last), then the record-id and source-id columns; `member_order` (non-key TAG names + NaN after every number, nulls last), then the record-id and source-id columns; `member_order` (non-key TAG names only: a data column, a key, a repeat, a set or an empty list is refused; `"name"` is the default; an ordering function is reserved, refused for now) names tags to order by first, the rest follow in name order. A member is never ordered by a data value. The keys in `by` @@ -1190,6 +1190,15 @@ data. The rules: record ids, folds, source-info tokens and hashes; only the output's column order follows the input, as presentation. `TagBy`'s identity is its sorted names, and its new tags are placed in name order. +- `GroupBy` groups keys as the joins match them, except null (developer, November 17, 2026, + ITL-1234): all rows with a null key form one group, and so do all rows with a NaN key (the + grouping key holds every NaN as one object, `_grouping_key`); `0.0` equals `-0.0`. A NaN key + sorts after every number and before null, as in the member order (`_null_last_key`, which + also sorts a NaN tag after every number). The hashes read a float's bits, so a group emits, + for each float key column, its members' value with the smallest bit pattern at the column's + own width, float16, float32 or float64 (`_float_key_bits`, `_emitted_key_value`: `0.0` over + `-0.0`, the canonical NaN over any other): always a value a member holds, never the first + row's, never chosen by `member_order`. - Data source-info values may be str, None, or list[...], nested to any depth; Arrow/Python types are derived from the value, with None mapping to large_string. That rule has one definition, arrow_utils.source_info_arrow_type, and applies only to tokens supplied as diff --git a/CLAUDE.md b/CLAUDE.md index aa55e5b25..3072996a1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1226,7 +1226,7 @@ data. The rules: orders its members by the first grouping's keys outside the coarser key, then by record id. An opt-in mode that keeps non-key tags is ITL-1071. - GroupBy never depends on how its input's columns are arranged (developer, November 17, 2026, ITL-1226). Members are ordered by the non-key tags in name order (each ascending, - nulls last), then the record-id and source-id columns; `member_order` (non-key TAG names + NaN after every number, nulls last), then the record-id and source-id columns; `member_order` (non-key TAG names only: a data column, a key, a repeat, a set or an empty list is refused; `"name"` is the default; an ordering function is reserved, refused for now) names tags to order by first, the rest follow in name order. A member is never ordered by a data value. The keys in `by` @@ -1236,6 +1236,15 @@ data. The rules: record ids, folds, source-info tokens and hashes; only the output's column order follows the input, as presentation. `TagBy`'s identity is its sorted names, and its new tags are placed in name order. +- `GroupBy` groups keys as the joins match them, except null (developer, November 17, 2026, + ITL-1234): all rows with a null key form one group, and so do all rows with a NaN key (the + grouping key holds every NaN as one object, `_grouping_key`); `0.0` equals `-0.0`. A NaN key + sorts after every number and before null, as in the member order (`_null_last_key`, which + also sorts a NaN tag after every number). The hashes read a float's bits, so a group emits, + for each float key column, its members' value with the smallest bit pattern at the column's + own width, float16, float32 or float64 (`_float_key_bits`, `_emitted_key_value`: `0.0` over + `-0.0`, the canonical NaN over any other): always a value a member holds, never the first + row's, never chosen by `member_order`. - `Data` source-info values may be `str`, `None`, or `list[...]`, nested to any depth; the Arrow and Python types are derived from the value, with `None` mapping to `large_string`. That rule has one definition, `arrow_utils.source_info_arrow_type`, and applies only to diff --git a/changelog.d/ITL-1234.md b/changelog.d/ITL-1234.md new file mode 100644 index 000000000..6cbfb3af0 --- /dev/null +++ b/changelog.d/ITL-1234.md @@ -0,0 +1,16 @@ +### Changed + +- `GroupBy` puts all rows whose key is NaN into one group (ITL-1234), as the joins match a + NaN key; with several keys, the key tuples compare element by element with that rule. + Before, each such row was a group of its own, and those groups came out in the input's + row order. A NaN key sorts after every number and before null. Only an input with a NaN + key changes: its groups, their member lists, folded ids and hashes. + +### Fixed + +- `GroupBy` emits a float key whose members differ in bits as the one with the smallest + bit pattern at the column's own width (ITL-1234): `0.0` over `-0.0`, the canonical NaN + over a NaN with another sign or payload. Before, `0.0` and `-0.0` were already one + group, but the group emitted the key of the first row in input order, so its value and + the row's hashes depended on the row order. A group whose members agree emits their + value as before. diff --git a/docs/concepts/operators.md b/docs/concepts/operators.md index 4b73ea29c..e16a5a4f8 100644 --- a/docs/concepts/operators.md +++ b/docs/concepts/operators.md @@ -308,14 +308,22 @@ Ordering is deterministic by construction, because orcapod hashes the emitted li cache keys: - **Members** are sorted by the input's non-key tags, even though those tags are dropped. - The tags are taken in name order, each ascending with nulls last, and the internal record - id breaks ties. `member_order` names tags to sort by first, in its order; the other + The tags are taken in name order, each ascending with NaN after every number and nulls + last, and the internal record id breaks ties. `member_order` names tags to sort by first, in its order; the other non-key tags follow in name order. A member is never ordered by a data value: a name in `member_order` must be a tag of the input (a data-tag counts), and a data column is refused (make it a tag first with `tag_by`). - **Groups** are emitted sorted by their keys, taken in name order. The order of the names in `by` is not part of the operator: `GroupBy(by=["a", "b"])` and `GroupBy(by=["b", "a"])` are one operator. +- **Keys** group as the joins match them, except for null: rows with equal keys form one + group, all rows with a null key form one group, and so do all rows with a NaN key (NaN + equals NaN here; `0.0` equals `-0.0`). A NaN key sorts after every number and before null. + When the members of a group hold a float key with different bits (`0.0` and `-0.0`, or + NaN with another sign or payload), the group emits the one with the smallest bit pattern + at the column's own width (`float16`, `float32` or `float64`), for each key column: `0.0` + over `-0.0`, and the canonical NaN over any other. The emitted key is always a value a + member holds, and neither the order of the rows nor `member_order` decides it. - **No arrangement of the input changes the result.** The order of its tags, the order of its data columns and the layout of its table leave every value, record id and hash of the output unchanged. A reordered input produces a byte-identical output table. The output diff --git a/src/orcapod/contexts/data/v1.json b/src/orcapod/contexts/data/v1.json index d2645e10f..fe8d91736 100644 --- a/src/orcapod/contexts/data/v1.json +++ b/src/orcapod/contexts/data/v1.json @@ -254,10 +254,10 @@ "date": "2026-11-17", "release": null, "added": { - "pipeline-02.json": "1ec3cbbe09bbb353366d1c8f2573881467d1c8e1ae53a20f8df4e7f43bbfa74e" + "pipeline-02.json": "4295ff9963cd99f27edc8b9edecf103b2f94976eb3b8fdf27ff3bc8ad1d89930" }, "previous": "6cdd29cdc6fa98df322931e2ab0ff9df151fdb47d3381adecd9f58fd55d2fb22", - "digest": "fa4c504d4eb289138f85029217770c25c45ebea9e7dd8855c42c83b9c515d37c" + "digest": "b274da918590e154385368314e79a2e2ff3b5973b9c261cbd533dfe07baf1f9e" } ] }, diff --git a/src/orcapod/core/operators/group_by.py b/src/orcapod/core/operators/group_by.py index 6d1e31967..5344055a9 100644 --- a/src/orcapod/core/operators/group_by.py +++ b/src/orcapod/core/operators/group_by.py @@ -50,6 +50,72 @@ def _null_last_key(value: Any) -> tuple[Any, ...]: return (0, value) +# The grouping form of a NaN key value: one shared object, so every row whose key +# is NaN falls into one group (ITL-1234), as the joins match a NaN key +# (``match_keys``). ``0.0`` and ``-0.0`` already fall into one group: they are equal. +_NAN_KEY = object() + + +def _grouping_key(key: tuple[Any, ...]) -> tuple[Any, ...]: + """Return a group key in the form that groups rows: every NaN as one value. + + Args: + key: The values of the ``by`` columns of one row, in the order of ``by``. + + Returns: + The key with each NaN replaced by one shared object. + """ + return tuple( + _NAN_KEY if isinstance(value, float) and math.isnan(value) else value + for value in key + ) + + +def _float_key_bits(column: "pa.ChunkedArray[Any]") -> list[int] | None: + """Return the bit pattern of each value of a float key column, at its own width. + + Args: + column: A key column of the input table. + + Returns: + For a ``float16``, ``float32`` or ``float64`` column, each value's IEEE 754 + bits as an unsigned integer of the column's width (``0`` for a null: a null + key's group holds nulls only, so its bits never decide); otherwise ``None``. + """ + arrow_type = column.type + if not pa.types.is_floating(arrow_type): + return None + unsigned = {16: pa.uint16(), 32: pa.uint32(), 64: pa.uint64()}[arrow_type.bit_width] + return [ + 0 if bits is None else bits + for bits in column.combine_chunks().view(unsigned).to_pylist() + ] + + +def _emitted_key_value(values: Sequence[Any], bits: Sequence[int] | None) -> Any: + """Return the value a group emits for one key column, whatever its row order. + + The members of a group hold equal key values, but equal floats may differ in + their bits (``0.0`` and ``-0.0``, NaN with another sign or payload), and the + hashes of the emitted row read the bits. So a float key is the members' value + with the smallest bit pattern at the column's own width (ITL-1234), chosen per + key column: the value is always one a member holds, a group whose members agree + keeps it, and a mixed group emits ``0.0`` over ``-0.0`` and the canonical quiet + NaN over any other NaN. Neither the row order nor ``member_order`` decides it. + + Args: + values: The key column's value in each member of one group (at least one). + bits: The bit pattern of each of those values, from ``_float_key_bits``; + ``None`` for a column that is not a float. + + Returns: + The key value the group emits. + """ + if bits is None: + return values[0] + return values[min(range(len(values)), key=bits.__getitem__)] + + GROUP_BY_FOLD_LABEL = "group_v1" """Label of the folds of ``GroupBy``: the folded source id and record id are ``"group_v1:"`` (ITL-1154), and so is the one source-info token of a data-tag @@ -177,6 +243,15 @@ class GroupBy(UnaryOperator): reordered input or a reordered ``by`` must produce an identical output. + Rows with equal keys form one group, as the joins match keys, except that all + rows with a null key form one group too. So do all rows with a NaN key, and + ``0.0`` equals ``-0.0`` (ITL-1234). A NaN key sorts after every number and + before null. A float key that the members hold with different bits is + emitted, per key column, as the one with the smallest bit pattern at the + column's own width (``0.0`` over ``-0.0``, the canonical NaN over any other), + so the emitted key is a value a member holds and never depends on the row + order or on ``member_order``. + Tag tuples are expected to be unique within a stream, but nothing enforces that. If the input does contain duplicate tag tuples, the members sharing a tuple are separated only by ``record_id``, which is fixed when the source @@ -394,15 +469,34 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: sorted(c for c in system_tag_columns if c.startswith(prefix)) ) - groups: dict[tuple[Any, ...], list[dict[str, Any]]] = {} - for row in table.to_pylist(): - groups.setdefault(tuple(row[c] for c in self.by), []).append(row) + # Every NaN key is one value here (ITL-1234), so the rows whose key is NaN + # form one group, as the joins match them. + rows = table.to_pylist() + groups: dict[tuple[Any, ...], list[int]] = {} + for index, row in enumerate(rows): + key = _grouping_key(tuple(row[c] for c in self.by)) + groups.setdefault(key, []).append(index) + key_bits = {c: _float_key_bits(table.column(c)) for c in self.by} + keyed_groups: list[tuple[tuple[Any, ...], list[dict[str, Any]]]] = [] + for indices in groups.values(): + emitted = tuple( + _emitted_key_value( + [rows[i][c] for i in indices], + None + if (bits := key_bits[c]) is None + else [bits[i] for i in indices], + ) + for c in self.by + ) + keyed_groups.append((emitted, [rows[i] for i in indices])) grouped_rows: list[dict[str, Any]] = [] # Emit groups in key order rather than first-appearance order, so a - # reordered input produces a byte-identical output table. + # reordered input produces a byte-identical output table. The keys are in + # name order (``by`` is sorted); a NaN key sorts after every number and + # before null, as a member's tag does. for key, members in sorted( - groups.items(), key=lambda kv: tuple(_null_last_key(v) for v in kv[0]) + keyed_groups, key=lambda km: tuple(_null_last_key(v) for v in km[0]) ): if sort_columns: members.sort( diff --git a/tests/test_core/operators/test_group_by_member_order.py b/tests/test_core/operators/test_group_by_member_order.py index 3bc81e901..04ea3d7e3 100644 --- a/tests/test_core/operators/test_group_by_member_order.py +++ b/tests/test_core/operators/test_group_by_member_order.py @@ -20,7 +20,7 @@ import itertools import json from collections.abc import Callable -from typing import Any, cast +from typing import Any import pyarrow as pa import pytest @@ -451,25 +451,3 @@ def test_a_nan_tag_orders_the_same_sync_and_async() -> None: if not seen: seen = [row] assert row == seen[0], orchestrator.__name__ - - -def test_a_nan_group_key_is_a_group_of_its_own_row() -> None: - """Pins today's behaviour, unchanged here: NaN never equals NaN as a key. - - Each row whose key is NaN is a group of its own; the groups with a number come - first, in key order, then the NaN groups. - """ - nan = float("nan") - for values in ([0.0, nan, 1.0, nan], [nan, 1.0, nan, 0.0]): - source = op.sources.ArrowTableSource( - pa.table({"g": pa.array(values, pa.float64()), "v": [1, 2, 3, 4]}), - tag_columns=["g"], - tag_only_columns=True, - source_id="s", - ) - out = source.group_by(["g"]).as_table() - keys = out.column("g").to_pylist() - assert keys[:2] == [0.0, 1.0] - assert all(k != k for k in keys[2:]) and len(keys) == 4 - members = cast(list[list[int]], out.column("v").to_pylist()) - assert [len(m) for m in members] == [1, 1, 1, 1] diff --git a/tests/test_core/operators/test_group_by_nan_keys.py b/tests/test_core/operators/test_group_by_nan_keys.py new file mode 100644 index 000000000..112ef6882 --- /dev/null +++ b/tests/test_core/operators/test_group_by_nan_keys.py @@ -0,0 +1,232 @@ +"""A NaN group key (ITL-1234). + +Developer decision, November 17, 2026: all rows whose ``by`` key is NaN form one group, +as the joins match a NaN key; before, each such row was a group of its own, in the +input's row order. A NaN key sorts after every number and before null, as a NaN tag +does in the member order. ``0.0`` and ``-0.0`` are one key. The hashes of a grouped row +read a float's bits, so a group emits, for each float key, its members' value with the +smallest bit pattern: the emitted key never depends on which row came first. + +Every test runs every order of the input rows: the values, the bits of every float, +the record ids, the folds and every hash must be the same for all of them. +""" + +from __future__ import annotations + +import itertools +import json +import struct +from typing import Any + +import pyarrow as pa +import pytest + +import orcapod as op +from orcapod.core.streams.base import StreamBase +from orcapod.databases import InMemoryArrowDatabase +from orcapod.pipeline import ( + AsyncPipelineOrchestrator, + PipelineJob, + SyncPipelineOrchestrator, +) +from orcapod.protocols.core_protocols import StreamProtocol + + +# Each float key value by name, as its bit pattern at each width: the canonical quiet +# NaN, NaN with the sign bit set, NaN with a payload, -0.0, 0.0 and 1.0. +BITS: dict[str, dict[int, int]] = { + "nan": {16: 0x7E00, 32: 0x7FC00000, 64: 0x7FF8000000000000}, + "-nan": {16: 0xFE00, 32: 0xFFC00000, 64: 0xFFF8000000000000}, + "payload_nan": {16: 0x7E01, 32: 0x7FC00001, 64: 0x7FF8000000000001}, + "-0": {16: 0x8000, 32: 0x80000000, 64: 0x8000000000000000}, + "0": {16: 0x0, 32: 0x0, 64: 0x0}, + "1": {16: 0x3C00, 32: 0x3F800000, 64: 0x3FF0000000000000}, +} +FLOAT = {16: pa.float16(), 32: pa.float32(), 64: pa.float64()} +UNSIGNED = {16: pa.uint16(), 32: pa.uint32(), 64: pa.uint64()} +WIDTHS = [16, 32, 64] + +# One key column ``k``; ``t`` and ``u`` are non-key tags that order the members; ``rid`` +# names each row, so a reordered input keeps every record id. +ONE_KEY_ROWS: list[dict[str, Any]] = [ + {"k": "nan", "t": 2, "u": 1, "rid": "r1", "v": 1}, + {"k": "-nan", "t": 1, "u": 2, "rid": "r2", "v": 2}, + {"k": "payload_nan", "t": 3, "u": 0, "rid": "r3", "v": 3}, + {"k": "-0", "t": 1, "u": 1, "rid": "r4", "v": 4}, + {"k": "0", "t": 0, "u": 2, "rid": "r5", "v": 5}, + {"k": None, "t": 0, "u": 0, "rid": "r6", "v": 6}, +] + +# Two key columns, both floats. +TWO_KEY_ROWS: list[dict[str, Any]] = [ + {"g": "0", "k": "nan", "t": 1, "u": 0, "rid": "r1", "v": 1}, + {"g": "0", "k": "-nan", "t": 0, "u": 1, "rid": "r2", "v": 2}, + {"g": "nan", "k": "nan", "t": 1, "u": 0, "rid": "r3", "v": 3}, + {"g": "-0", "k": "1", "t": 0, "u": 0, "rid": "r4", "v": 4}, + {"g": "-nan", "k": None, "t": 0, "u": 0, "rid": "r5", "v": 5}, + {"g": "payload_nan", "k": "-nan", "t": 0, "u": 1, "rid": "r6", "v": 6}, +] + + +def _source(rows: list[dict[str, Any]], keys: list[str], width: int) -> StreamBase: + columns: dict[str, Any] = { + name: pa.array( + [None if r[name] is None else BITS[r[name]][width] for r in rows], + UNSIGNED[width], + ).view(FLOAT[width]) + for name in keys + } + for name in ("t", "u", "rid", "v"): + columns[name] = [r[name] for r in rows] + return op.sources.ArrowTableSource( + pa.table(columns), + tag_columns=[*keys, "t", "u"], + tag_only_columns=True, + source_id="s", + record_id_columns="rid", + ) + + +def _bits(value: float) -> int: + bits: int = struct.unpack(" Any: + """Replace every float by its bit pattern, so a comparison sees NaN and -0.0.""" + if isinstance(value, float): + return f"float:{_bits(value):016x}" + if isinstance(value, list): + return [_with_bits(v) for v in value] + if isinstance(value, dict): + return {k: _with_bits(v) for k, v in value.items()} + return value + + +def _fingerprint(stream: StreamProtocol) -> dict[str, Any]: + """The rows (floats by their bits), the row hashes and the stream's hashes.""" + return { + "rows": [ + json.dumps(_with_bits(row), sort_keys=True, default=str) + for row in stream.as_table(all_info=True).to_pylist() + ], + "row_hashes": [ + (tag.content_hash().to_string(), data.content_hash().to_string()) + for tag, data in stream.iter_data() + ], + "content_hash": stream.content_hash().to_string(), + "pipeline_hash": stream.pipeline_hash().to_string(), + } + + +def _key_bits(stream: StreamProtocol, name: str, width: int) -> list[int | None]: + """The emitted key values of one column, as bit patterns at the column's width.""" + column = stream.as_table().column(name).combine_chunks() + assert column.type == FLOAT[width] + return column.view(UNSIGNED[width]).to_pylist() + + +@pytest.mark.parametrize("width", WIDTHS) +@pytest.mark.parametrize( + ("member_order", "members"), + [ + # 0.0 group by t (r5, r4); NaN group by t (r2, r1, r3); null group. + (None, [[5, 4], [2, 1, 3], [6]]), + # by u first: 0.0 group (r4, r5); NaN group (r3, r1, r2). + (["u"], [[4, 5], [3, 1, 2], [6]]), + ], + ids=["default", "member_order_u"], +) +def test_nan_keys_form_one_group_in_every_row_order( + member_order: list[str] | None, members: list[list[int]], width: int +) -> None: + """Three NaN keys with different bits are one group, after 0.0 and before null.""" + seen: dict[str, Any] = {} + for rows in itertools.permutations(ONE_KEY_ROWS): + grouped = _source(list(rows), ["k"], width).group_by( + "k", member_order=member_order + ) + assert grouped.as_table().column("v").to_pylist() == members + # 0.0 over -0.0 and the canonical NaN over the others, whichever came first. + assert _key_bits(grouped, "k", width) == [ + BITS["0"][width], + BITS["nan"][width], + None, + ] + fingerprint = _fingerprint(grouped) + if not seen: + seen = fingerprint + assert fingerprint == seen + + +@pytest.mark.parametrize("width", WIDTHS) +@pytest.mark.parametrize( + ("member_order", "members"), + [ + (None, [[4], [2, 1], [6, 3], [5]]), + (["u"], [[4], [1, 2], [3, 6], [5]]), + ], + ids=["default", "member_order_u"], +) +def test_two_keys_compare_element_wise_in_every_row_order( + member_order: list[str] | None, members: list[list[int]], width: int +) -> None: + """Key tuples compare element by element, NaN equal to NaN, in name order. + + Each key column picks its own emitted value. + """ + seen: dict[str, Any] = {} + for rows in itertools.permutations(TWO_KEY_ROWS): + # The order of ``by`` is not identity: either order gives the same result. + by = ["k", "g"] if rows[0]["rid"] < "r4" else ["g", "k"] + grouped = _source(list(rows), ["g", "k"], width).group_by( + by, member_order=member_order + ) + assert grouped.as_table().column("v").to_pylist() == members + # A group whose members agree emits their value (-0.0 and the sign-set NaN + # here); a mixed group emits the smallest bit pattern. + assert _key_bits(grouped, "g", width) == [ + BITS[name][width] for name in ("-0", "0", "nan", "-nan") + ] + assert _key_bits(grouped, "k", width) == [ + BITS["1"][width], + BITS["nan"][width], + BITS["nan"][width], + None, + ] + fingerprint = _fingerprint(grouped) + if not seen: + seen = fingerprint + assert fingerprint == seen + + +@pytest.mark.parametrize("width", [32, 64]) +def test_nan_keys_group_the_same_sync_and_async(width: int) -> None: + seen: list[str] = [] + for rows in (ONE_KEY_ROWS, ONE_KEY_ROWS[::-1], ONE_KEY_ROWS[3:] + ONE_KEY_ROWS[:3]): + for orchestrator in (SyncPipelineOrchestrator, AsyncPipelineOrchestrator): + job = PipelineJob(name="nan_keys", database=InMemoryArrowDatabase()) + with job: + _ = _source(list(rows), ["k"], width).group_by("k", label="grouped") + job.run(orchestrator=orchestrator()) + out = job.nodes["grouped"].as_table(all_info=True) + assert out.column("v").to_pylist() == [[5, 4], [2, 1, 3], [6]] + keys = out.column("k").combine_chunks().view(UNSIGNED[width]) + assert keys.to_pylist() == [BITS["0"][width], BITS["nan"][width], None] + fingerprint = json.dumps( + _with_bits(out.to_pylist()), sort_keys=True, default=str + ) + if not seen: + seen = [fingerprint] + assert fingerprint == seen[0], orchestrator.__name__ + + +@pytest.mark.parametrize("width", WIDTHS) +@pytest.mark.parametrize("name", ["-nan", "payload_nan", "-0"]) +def test_a_group_whose_members_agree_keeps_their_key_bits( + name: str, width: int +) -> None: + """The emitted key is always a value a member holds: nothing is rewritten.""" + rows = [dict(r, k=name) for r in ONE_KEY_ROWS[:2]] + grouped = _source(rows, ["k"], width).group_by("k") + assert _key_bits(grouped, "k", width) == [BITS[name][width]] diff --git a/tests/test_hashing/golden/cases.py b/tests/test_hashing/golden/cases.py index add9bcb9e..9771c30ca 100644 --- a/tests/test_hashing/golden/cases.py +++ b/tests/test_hashing/golden/cases.py @@ -2651,3 +2651,43 @@ def pipeline_group_by_arrangement() -> dict[str, Any]: ) ) return out + + +# GroupBy over a float key with NaN and -0.0 (ITL-1234) +# --------------------------------------------------------------------------- + + +@case("pipeline", "group_by_nan_keys") +def pipeline_group_by_nan_keys() -> dict[str, Any]: + """Pin ``GroupBy`` over a float key whose NaN and zero values differ in bits. + + The three NaN keys (the canonical quiet NaN, one with the sign bit set, one with + a payload) form one group, which emits the canonical NaN; ``0.0`` and ``-0.0`` + form one group, which emits ``0.0``; the null keys form one group. + """ + from orcapod.operators import GroupBy + from orcapod.sources import ArrowTableSource + + key_bits = [ + 0xFFF8000000000000, + 0x8000000000000000, + 0x7FF8000000000001, + None, + 0x0, + 0x7FF8000000000000, + None, + ] + table = pa.table( + { + "k": pa.array(key_bits, pa.uint64()).view(pa.float64()), + "t": pa.array([1, 2, 3, 4, 5, 6, 7], pa.int64()), + "v": pa.array([10, 20, 30, 40, 50, 60, 70], pa.int64()), + } + ) + source = ArrowTableSource( + table, + tag_columns=["k", "t"], + tag_only_columns=True, + source_id="golden_group_by_nan_keys", + ) + return stream_summary(GroupBy(by=["k"])(source)) diff --git a/tests/test_hashing/golden/data/v1/pipeline-02.json b/tests/test_hashing/golden/data/v1/pipeline-02.json index 194ba8743..268c8da0b 100644 --- a/tests/test_hashing/golden/data/v1/pipeline-02.json +++ b/tests/test_hashing/golden/data/v1/pipeline-02.json @@ -135,5 +135,20 @@ "group_by_arrangement.member_order_y.tags_first.system_tag_columns": [ "_tag_source_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8", "_tag_record_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8" + ], + "group_by_nan_keys.content_hash": "semantic_v1:440a76d5318ec3d55c8cbdf569f468168ae35ef4ff3251e57f1ee1b173f835a7", + "group_by_nan_keys.pipeline_hash": "semantic_v1:757b8297a21932d70dd5f816742c2fb0eb159bde455f83373eb3f9a31591d46b", + "group_by_nan_keys.row[{\"k\":0.0}]._tag_record_id::b9b2a71e04d7acbbbcb6d4e9ed74068893840cbed90077e3567c6851b533a656::b9b2a71e04d7acbbbcb6d4e9ed74068893840cbed90077e3567c6851b533a656": "group_v1:a129e550a9e72a9c72e074e0ed3afffce39ada58073e6e1dcf018b03d309da55", + "group_by_nan_keys.row[{\"k\":0.0}]._tag_source_id::b9b2a71e04d7acbbbcb6d4e9ed74068893840cbed90077e3567c6851b533a656::b9b2a71e04d7acbbbcb6d4e9ed74068893840cbed90077e3567c6851b533a656": "group_v1:006df9f09c1fa47b2eb819117849cfc1a1781b3a244b3a05fe0dc07fdb8af1a2", + "group_by_nan_keys.row[{\"k\":0.0}].data_hash": "arrow_v1:000002529ddb22d5950e5c74cf80d618ffc83f8cf705a56f197403ec4e1835a06ba677", + "group_by_nan_keys.row[{\"k\":NaN}]._tag_record_id::b9b2a71e04d7acbbbcb6d4e9ed74068893840cbed90077e3567c6851b533a656::b9b2a71e04d7acbbbcb6d4e9ed74068893840cbed90077e3567c6851b533a656": "group_v1:15a58a6c8160f20635066df45354c75ee2e2446220d8218f3513910379604b62", + "group_by_nan_keys.row[{\"k\":NaN}]._tag_source_id::b9b2a71e04d7acbbbcb6d4e9ed74068893840cbed90077e3567c6851b533a656::b9b2a71e04d7acbbbcb6d4e9ed74068893840cbed90077e3567c6851b533a656": "group_v1:7600c5c59642d9cc035a04f3d2694fe97220904595fdca39214ab49f21b42047", + "group_by_nan_keys.row[{\"k\":NaN}].data_hash": "arrow_v1:0000024fb6c35ca785c5d07b688027a823d5c9c565ceadadf5d3648926d84df0755a22", + "group_by_nan_keys.row[{\"k\":null}]._tag_record_id::b9b2a71e04d7acbbbcb6d4e9ed74068893840cbed90077e3567c6851b533a656::b9b2a71e04d7acbbbcb6d4e9ed74068893840cbed90077e3567c6851b533a656": "group_v1:c15b4d7880e5fd711b2b799ad3b08acdbefb04c8ca67c51f284cf5cb1c9f1628", + "group_by_nan_keys.row[{\"k\":null}]._tag_source_id::b9b2a71e04d7acbbbcb6d4e9ed74068893840cbed90077e3567c6851b533a656::b9b2a71e04d7acbbbcb6d4e9ed74068893840cbed90077e3567c6851b533a656": "group_v1:006df9f09c1fa47b2eb819117849cfc1a1781b3a244b3a05fe0dc07fdb8af1a2", + "group_by_nan_keys.row[{\"k\":null}].data_hash": "arrow_v1:0000028a9bb897af2967d6055081dad273123417a319e33b18b6c44c68551b23f89b8b", + "group_by_nan_keys.system_tag_columns": [ + "_tag_source_id::b9b2a71e04d7acbbbcb6d4e9ed74068893840cbed90077e3567c6851b533a656::b9b2a71e04d7acbbbcb6d4e9ed74068893840cbed90077e3567c6851b533a656", + "_tag_record_id::b9b2a71e04d7acbbbcb6d4e9ed74068893840cbed90077e3567c6851b533a656::b9b2a71e04d7acbbbcb6d4e9ed74068893840cbed90077e3567c6851b533a656" ] } From db50cf3f0a39741d32c45b4271d6fb2df6172a8a Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 17 Nov 2026 21:34:57 -0800 Subject: [PATCH 2/4] fix(group_by): emit a key from the input's Arrow column, bits read only for a zero or NaN key (ITL-1234) Copilot review of #441. A float32 signaling NaN key came back quiet through a Python float, so the emitted key was not a value a member holds: each key column is now taken from the input's Arrow column at the chosen member row. Bits are read only for a group of two or more members whose key is zero or NaN. Tests: signaling NaN per width, alone and mixed, tag and data-tag key, sync == async. Golden: tag hashes, key bits and a float32 data-tag variant (27 entries added, none changed). Docs say that the smallest bit pattern can be a signaling NaN; the changelog says stored rows are not replaced. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 8 +- CLAUDE.md | 8 +- changelog.d/ITL-1234.md | 15 ++- docs/concepts/operators.md | 6 +- src/orcapod/contexts/data/v1.json | 4 +- src/orcapod/core/operators/group_by.py | 118 ++++++++++-------- .../operators/test_group_by_nan_keys.py | 95 +++++++++++++- tests/test_hashing/golden/cases.py | 51 +++++++- .../golden/data/v1/pipeline-02.json | 30 +++++ 9 files changed, 259 insertions(+), 76 deletions(-) diff --git a/.zed/rules b/.zed/rules index 1b5c41e0c..bea2ff403 100644 --- a/.zed/rules +++ b/.zed/rules @@ -1196,9 +1196,11 @@ data. The rules: sorts after every number and before null, as in the member order (`_null_last_key`, which also sorts a NaN tag after every number). The hashes read a float's bits, so a group emits, for each float key column, its members' value with the smallest bit pattern at the column's - own width, float16, float32 or float64 (`_float_key_bits`, `_emitted_key_value`: `0.0` over - `-0.0`, the canonical NaN over any other): always a value a member holds, never the first - row's, never chosen by `member_order`. + own width, float16, float32 or float64 (`_emitted_key_row`: `0.0` over `-0.0`, a positive NaN + over a negative one; a signaling NaN has smaller bits than the canonical quiet NaN, so it + wins), taken from the input's Arrow column, never through Python (a float32 signaling NaN + comes back quiet from Python): always a value a member holds, never the first row's, never + chosen by `member_order`. Bits are read only for a group whose key is zero or NaN. - Data source-info values may be str, None, or list[...], nested to any depth; Arrow/Python types are derived from the value, with None mapping to large_string. That rule has one definition, arrow_utils.source_info_arrow_type, and applies only to tokens supplied as diff --git a/CLAUDE.md b/CLAUDE.md index 3072996a1..10e50c757 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1242,9 +1242,11 @@ data. The rules: sorts after every number and before null, as in the member order (`_null_last_key`, which also sorts a NaN tag after every number). The hashes read a float's bits, so a group emits, for each float key column, its members' value with the smallest bit pattern at the column's - own width, float16, float32 or float64 (`_float_key_bits`, `_emitted_key_value`: `0.0` over - `-0.0`, the canonical NaN over any other): always a value a member holds, never the first - row's, never chosen by `member_order`. + own width, float16, float32 or float64 (`_emitted_key_row`: `0.0` over `-0.0`, a positive NaN + over a negative one; a signaling NaN has smaller bits than the canonical quiet NaN, so it + wins), taken from the input's Arrow column, never through Python (a float32 signaling NaN + comes back quiet from Python): always a value a member holds, never the first row's, never + chosen by `member_order`. Bits are read only for a group whose key is zero or NaN. - `Data` source-info values may be `str`, `None`, or `list[...]`, nested to any depth; the Arrow and Python types are derived from the value, with `None` mapping to `large_string`. That rule has one definition, `arrow_utils.source_info_arrow_type`, and applies only to diff --git a/changelog.d/ITL-1234.md b/changelog.d/ITL-1234.md index 6cbfb3af0..dfcb13df9 100644 --- a/changelog.d/ITL-1234.md +++ b/changelog.d/ITL-1234.md @@ -4,13 +4,16 @@ NaN key; with several keys, the key tuples compare element by element with that rule. Before, each such row was a group of its own, and those groups came out in the input's row order. A NaN key sorts after every number and before null. Only an input with a NaN - key changes: its groups, their member lists, folded ids and hashes. + key changes: its groups, their member lists, folded ids and hashes. Stored rows of such + a `GroupBy` and of the nodes after it are not replaced: delete those tables or attach a + new database before running it with this version (`clear_cache()` clears memory only). ### Fixed - `GroupBy` emits a float key whose members differ in bits as the one with the smallest - bit pattern at the column's own width (ITL-1234): `0.0` over `-0.0`, the canonical NaN - over a NaN with another sign or payload. Before, `0.0` and `-0.0` were already one - group, but the group emitted the key of the first row in input order, so its value and - the row's hashes depended on the row order. A group whose members agree emits their - value as before. + bit pattern at the column's own width (ITL-1234): `0.0` over `-0.0`, a positive NaN over + a negative one, and a signaling NaN (smaller bits) over the canonical quiet NaN. Before, + `0.0` and `-0.0` were already one group, but the group emitted the key of the first row + in input order, so its value and the row's hashes depended on the row order. The key is + now taken from the input's Arrow column: a `float32` signaling NaN key is emitted with + its bits (it came back quiet). A group whose members agree emits their value as before. diff --git a/docs/concepts/operators.md b/docs/concepts/operators.md index e16a5a4f8..3a8b08f7c 100644 --- a/docs/concepts/operators.md +++ b/docs/concepts/operators.md @@ -322,8 +322,10 @@ cache keys: When the members of a group hold a float key with different bits (`0.0` and `-0.0`, or NaN with another sign or payload), the group emits the one with the smallest bit pattern at the column's own width (`float16`, `float32` or `float64`), for each key column: `0.0` - over `-0.0`, and the canonical NaN over any other. The emitted key is always a value a - member holds, and neither the order of the rows nor `member_order` decides it. + over `-0.0`, and a positive NaN over a negative one. A signaling NaN has smaller bits than + the canonical quiet NaN, so when a member holds one, it is the key emitted. The emitted key + is always a value a member holds, bit for bit, and neither the order of the rows nor + `member_order` decides it. - **No arrangement of the input changes the result.** The order of its tags, the order of its data columns and the layout of its table leave every value, record id and hash of the output unchanged. A reordered input produces a byte-identical output table. The output diff --git a/src/orcapod/contexts/data/v1.json b/src/orcapod/contexts/data/v1.json index fe8d91736..9c6400fc0 100644 --- a/src/orcapod/contexts/data/v1.json +++ b/src/orcapod/contexts/data/v1.json @@ -254,10 +254,10 @@ "date": "2026-11-17", "release": null, "added": { - "pipeline-02.json": "4295ff9963cd99f27edc8b9edecf103b2f94976eb3b8fdf27ff3bc8ad1d89930" + "pipeline-02.json": "7778b4c69ff706debbcbd9fc67dbe5b723df56252158be8253f8751326a5bd41" }, "previous": "6cdd29cdc6fa98df322931e2ab0ff9df151fdb47d3381adecd9f58fd55d2fb22", - "digest": "b274da918590e154385368314e79a2e2ff3b5973b9c261cbd533dfe07baf1f9e" + "digest": "838e8b0023619f2192415496893e3c4b35981980e388c6a617202d9cb1991d40" } ] }, diff --git a/src/orcapod/core/operators/group_by.py b/src/orcapod/core/operators/group_by.py index 5344055a9..41ea9a77e 100644 --- a/src/orcapod/core/operators/group_by.py +++ b/src/orcapod/core/operators/group_by.py @@ -4,7 +4,7 @@ import math from collections.abc import Collection, Sequence, Set -from typing import Any, ClassVar, TYPE_CHECKING, get_origin +from typing import Any, ClassVar, TYPE_CHECKING, cast, get_origin from orcapod.core.operators.base import UnaryOperator from orcapod.core.data_tags import stream_data_tags @@ -71,49 +71,46 @@ def _grouping_key(key: tuple[Any, ...]) -> tuple[Any, ...]: ) -def _float_key_bits(column: "pa.ChunkedArray[Any]") -> list[int] | None: - """Return the bit pattern of each value of a float key column, at its own width. - - Args: - column: A key column of the input table. - - Returns: - For a ``float16``, ``float32`` or ``float64`` column, each value's IEEE 754 - bits as an unsigned integer of the column's width (``0`` for a null: a null - key's group holds nulls only, so its bits never decide); otherwise ``None``. - """ - arrow_type = column.type - if not pa.types.is_floating(arrow_type): - return None - unsigned = {16: pa.uint16(), 32: pa.uint32(), 64: pa.uint64()}[arrow_type.bit_width] - return [ - 0 if bits is None else bits - for bits in column.combine_chunks().view(unsigned).to_pylist() - ] - - -def _emitted_key_value(values: Sequence[Any], bits: Sequence[int] | None) -> Any: - """Return the value a group emits for one key column, whatever its row order. +def _emitted_key_row(column: "pa.ChunkedArray[Any]", indices: Sequence[int]) -> int: + """Return the input row whose value of one key column a group emits. The members of a group hold equal key values, but equal floats may differ in - their bits (``0.0`` and ``-0.0``, NaN with another sign or payload), and the - hashes of the emitted row read the bits. So a float key is the members' value - with the smallest bit pattern at the column's own width (ITL-1234), chosen per - key column: the value is always one a member holds, a group whose members agree - keeps it, and a mixed group emits ``0.0`` over ``-0.0`` and the canonical quiet - NaN over any other NaN. Neither the row order nor ``member_order`` decides it. + their bits: ``0.0`` and ``-0.0``, and NaN with another sign or payload (equal + floats of any other value have the same bits). The hashes of the emitted row + read the bits, so for a zero or NaN key the group emits the member value with + the smallest bit pattern at the column's own width (ITL-1234), chosen per key + column. That is ``0.0`` over ``-0.0``, a positive NaN over a negative one, and + the NaN with the smallest payload; a signaling NaN has smaller bits than the + canonical quiet NaN, so it is the one emitted when a member holds it. A group + whose members agree emits their value. Neither the row order nor + ``member_order`` decides it, and the caller takes the value from the Arrow + column, so its bits are kept exactly. Args: - values: The key column's value in each member of one group (at least one). - bits: The bit pattern of each of those values, from ``_float_key_bits``; - ``None`` for a column that is not a float. + column: A key column of the input table. + indices: The rows of one group, at least one, all with equal keys. Returns: - The key value the group emits. + The row, one of ``indices``, whose value the group emits for this column. """ - if bits is None: - return values[0] - return values[min(range(len(values)), key=bits.__getitem__)] + first = indices[0] + arrow_type = column.type + if len(indices) == 1 or not pa.types.is_floating(arrow_type): + return first + value = column[first].as_py() + if value is None or (value != 0.0 and not math.isnan(value)): + return first + # The unsigned integer of the float's width, to read its bits; no value of the + # group is null, since a null key equals no float. + unsigned = {16: pa.uint16, 32: pa.uint32, 64: pa.uint64}[arrow_type.bit_width]() + bits = cast( + list[int], + column.take(pa.array(indices, pa.int64())) + .combine_chunks() + .view(unsigned) + .to_pylist(), + ) + return indices[min(range(len(indices)), key=bits.__getitem__)] GROUP_BY_FOLD_LABEL = "group_v1" @@ -248,9 +245,10 @@ class GroupBy(UnaryOperator): ``0.0`` equals ``-0.0`` (ITL-1234). A NaN key sorts after every number and before null. A float key that the members hold with different bits is emitted, per key column, as the one with the smallest bit pattern at the - column's own width (``0.0`` over ``-0.0``, the canonical NaN over any other), - so the emitted key is a value a member holds and never depends on the row - order or on ``member_order``. + column's own width (``0.0`` over ``-0.0``, a positive NaN over a negative + one, a signaling NaN over the canonical quiet NaN), taken from the input's + Arrow column, so the emitted key is a value a member holds, bit for bit, and + never depends on the row order or on ``member_order``. Tag tuples are expected to be unique within a stream, but nothing enforces that. If the input does contain duplicate tag tuples, the members sharing @@ -476,28 +474,29 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: for index, row in enumerate(rows): key = _grouping_key(tuple(row[c] for c in self.by)) groups.setdefault(key, []).append(index) - key_bits = {c: _float_key_bits(table.column(c)) for c in self.by} - keyed_groups: list[tuple[tuple[Any, ...], list[dict[str, Any]]]] = [] - for indices in groups.values(): - emitted = tuple( - _emitted_key_value( - [rows[i][c] for i in indices], - None - if (bits := key_bits[c]) is None - else [bits[i] for i in indices], - ) - for c in self.by + # For each group, the row whose value each key column emits. + keyed_groups: list[tuple[tuple[int, ...], list[dict[str, Any]]]] = [ + ( + tuple(_emitted_key_row(table.column(c), indices) for c in self.by), + [rows[i] for i in indices], ) - keyed_groups.append((emitted, [rows[i] for i in indices])) + for indices in groups.values() + ] grouped_rows: list[dict[str, Any]] = [] # Emit groups in key order rather than first-appearance order, so a # reordered input produces a byte-identical output table. The keys are in # name order (``by`` is sorted); a NaN key sorts after every number and # before null, as a member's tag does. - for key, members in sorted( - keyed_groups, key=lambda km: tuple(_null_last_key(v) for v in km[0]) + key_rows: list[tuple[int, ...]] = [] + for group_key_rows, members in sorted( + keyed_groups, + key=lambda km: tuple( + _null_last_key(rows[r][c]) for r, c in zip(km[0], self.by) + ), ): + key_rows.append(group_key_rows) + key = tuple(rows[r][c] for r, c in zip(group_key_rows, self.by)) if sort_columns: members.sort( key=lambda r: tuple(_null_last_key(r[c]) for c in sort_columns) @@ -531,6 +530,17 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: stream.data_context.type_converter, ) + # Each key column holds the chosen member's value as the input's Arrow + # column has it, never through Python: a conversion can change a float's + # bits (a float32 signaling NaN comes back quiet). + for position, c in enumerate(self.by): + chosen = pa.array([r[position] for r in key_rows], pa.int64()) + grouped_table = grouped_table.set_column( + grouped_table.schema.get_field_index(c), + grouped_table.schema.field(c), + table.column(c).take(chosen), + ) + n_char = self.orcapod_config.hashing.system_tag_n_char grouped_table = arrow_utils.append_to_system_tags( grouped_table, stream.pipeline_hash().to_hex(n_char) diff --git a/tests/test_core/operators/test_group_by_nan_keys.py b/tests/test_core/operators/test_group_by_nan_keys.py index 112ef6882..ee6e1bcdc 100644 --- a/tests/test_core/operators/test_group_by_nan_keys.py +++ b/tests/test_core/operators/test_group_by_nan_keys.py @@ -33,11 +33,13 @@ # Each float key value by name, as its bit pattern at each width: the canonical quiet -# NaN, NaN with the sign bit set, NaN with a payload, -0.0, 0.0 and 1.0. +# NaN, NaN with the sign bit set, NaN with a payload, a signaling NaN (smaller bits than +# the quiet one), -0.0, 0.0 and 1.0. BITS: dict[str, dict[int, int]] = { "nan": {16: 0x7E00, 32: 0x7FC00000, 64: 0x7FF8000000000000}, "-nan": {16: 0xFE00, 32: 0xFFC00000, 64: 0xFFF8000000000000}, "payload_nan": {16: 0x7E01, 32: 0x7FC00001, 64: 0x7FF8000000000001}, + "signaling_nan": {16: 0x7C01, 32: 0x7F800001, 64: 0x7FF0000000000001}, "-0": {16: 0x8000, 32: 0x80000000, 64: 0x8000000000000000}, "0": {16: 0x0, 32: 0x0, 64: 0x0}, "1": {16: 0x3C00, 32: 0x3F800000, 64: 0x3FF0000000000000}, @@ -68,7 +70,14 @@ ] -def _source(rows: list[dict[str, Any]], keys: list[str], width: int) -> StreamBase: +def _source( + rows: list[dict[str, Any]], + keys: list[str], + width: int, + *, + data_tag_key: bool = False, +) -> StreamBase: + """The rows as a source; with ``data_tag_key``, the keys are data-tags too.""" columns: dict[str, Any] = { name: pa.array( [None if r[name] is None else BITS[r[name]][width] for r in rows], @@ -81,7 +90,7 @@ def _source(rows: list[dict[str, Any]], keys: list[str], width: int) -> StreamBa return op.sources.ArrowTableSource( pa.table(columns), tag_columns=[*keys, "t", "u"], - tag_only_columns=True, + tag_only_columns=["t", "u"] if data_tag_key else True, source_id="s", record_id_columns="rid", ) @@ -222,11 +231,87 @@ def test_nan_keys_group_the_same_sync_and_async(width: int) -> None: @pytest.mark.parametrize("width", WIDTHS) -@pytest.mark.parametrize("name", ["-nan", "payload_nan", "-0"]) +@pytest.mark.parametrize("name", ["-nan", "payload_nan", "signaling_nan", "-0"]) def test_a_group_whose_members_agree_keeps_their_key_bits( name: str, width: int ) -> None: - """The emitted key is always a value a member holds: nothing is rewritten.""" + """The emitted key is always a value a member holds, bit for bit. + + A ``float32`` signaling NaN comes back quiet through a Python float, so the key is + taken from the Arrow column. + """ rows = [dict(r, k=name) for r in ONE_KEY_ROWS[:2]] grouped = _source(rows, ["k"], width).group_by("k") assert _key_bits(grouped, "k", width) == [BITS[name][width]] + + +# A signaling NaN among the NaN keys: its bits are the smallest, so it is emitted. +SIGNALING_ROWS: list[dict[str, Any]] = [ + {"k": "nan", "t": 1, "u": 0, "rid": "r1", "v": 1}, + {"k": "signaling_nan", "t": 2, "u": 0, "rid": "r2", "v": 2}, + {"k": "-nan", "t": 3, "u": 0, "rid": "r3", "v": 3}, + {"k": "payload_nan", "t": 4, "u": 0, "rid": "r4", "v": 4}, + {"k": "0", "t": 0, "u": 0, "rid": "r5", "v": 5}, +] + + +@pytest.mark.parametrize("data_tag_key", [False, True], ids=["tag_key", "data_tag_key"]) +@pytest.mark.parametrize("width", WIDTHS) +def test_a_signaling_nan_key_is_emitted_with_its_bits( + width: int, data_tag_key: bool +) -> None: + """In every row order, the NaN group emits the signaling NaN, bit for bit. + + With a data-tag key, the key's data hash and folded source-info token are the + same for every row order too. + """ + seen: dict[str, Any] = {} + for rows in itertools.permutations(SIGNALING_ROWS): + grouped = _source(list(rows), ["k"], width, data_tag_key=data_tag_key) + grouped = grouped.group_by("k") + assert grouped.as_table().column("v").to_pylist() == [[5], [1, 2, 3, 4]] + assert _key_bits(grouped, "k", width) == [ + BITS["0"][width], + BITS["signaling_nan"][width], + ] + fingerprint = _fingerprint(grouped) + if not seen: + seen = fingerprint + assert fingerprint == seen + + +@pytest.mark.parametrize("data_tag_key", [False, True], ids=["tag_key", "data_tag_key"]) +@pytest.mark.parametrize("width", WIDTHS) +def test_a_signaling_nan_key_groups_the_same_sync_and_async( + width: int, data_tag_key: bool +) -> None: + seen: list[str] = [] + for rows in (SIGNALING_ROWS, SIGNALING_ROWS[::-1]): + for orchestrator in (SyncPipelineOrchestrator, AsyncPipelineOrchestrator): + job = PipelineJob(name="signaling_nan", database=InMemoryArrowDatabase()) + with job: + _ = _source( + list(rows), ["k"], width, data_tag_key=data_tag_key + ).group_by("k", label="grouped") + job.run(orchestrator=orchestrator()) + node = job.nodes["grouped"] + out = node.as_table(all_info=True) + keys = out.column("k").combine_chunks().view(UNSIGNED[width]) + assert keys.to_pylist() == [BITS["0"][width], BITS["signaling_nan"][width]] + fingerprint = json.dumps( + { + "rows": _with_bits(out.to_pylist()), + "hashes": [ + ( + tag.content_hash().to_string(), + data.content_hash().to_string(), + ) + for tag, data in node.iter_data() + ], + }, + sort_keys=True, + default=str, + ) + if not seen: + seen = [fingerprint] + assert fingerprint == seen[0], orchestrator.__name__ diff --git a/tests/test_hashing/golden/cases.py b/tests/test_hashing/golden/cases.py index 9771c30ca..14defcd82 100644 --- a/tests/test_hashing/golden/cases.py +++ b/tests/test_hashing/golden/cases.py @@ -2690,4 +2690,53 @@ def pipeline_group_by_nan_keys() -> dict[str, Any]: tag_only_columns=True, source_id="golden_group_by_nan_keys", ) - return stream_summary(GroupBy(by=["k"])(source)) + grouped = GroupBy(by=["k"])(source) + out = stream_summary(grouped) + out.update(_group_key_identity(grouped, pa.uint64())) + + # A data-tag key of float32 (the key is data too): a signaling NaN, whose bits are + # the smallest of the NaN group, is emitted bit for bit; the key's data hash and + # its folded source-info token are pinned with the rest. + data_tag_bits = [0x7FC00000, 0x80000000, 0x7F800001, None, 0x0, 0xFFC00000] + data_tag_table = pa.table( + { + "k": pa.array(data_tag_bits, pa.uint32()).view(pa.float32()), + "t": pa.array([1, 2, 3, 4, 5, 6], pa.int64()), + "v": pa.array([10, 20, 30, 40, 50, 60], pa.int64()), + } + ) + data_tag_source = ArrowTableSource( + data_tag_table, + tag_columns=["k", "t"], + tag_only_columns=["t"], + source_id="golden_group_by_nan_keys_data_tag", + ) + data_tag_grouped = GroupBy(by=["k"])(data_tag_source) + out.update( + _prefixed( + "data_tag_key", + { + **stream_summary(data_tag_grouped), + **_group_key_identity(data_tag_grouped, pa.uint32()), + }, + ) + ) + return out + + +def _group_key_identity(stream: Any, unsigned: Any) -> dict[str, Any]: # pyright: ignore[reportExplicitAny] # a stream and an Arrow type + """Per row of a grouping on ``k``: the tag hash, the key's bits and its source info. + + ``stream_summary`` keys a row by its tag values, where every NaN reads ``NaN``; + the bits and the tag hash pin which NaN (or which zero) the group emits. + """ + out: dict[str, Any] = {} + bits = stream.as_table().column("k").combine_chunks().view(unsigned).to_pylist() + for (tag, data), key_bits in zip(stream.iter_data(), bits, strict=True): + row = _row_key(tag.as_dict()) + out[f"row[{row}].tag_hash"] = tag.content_hash().to_string() + out[f"row[{row}].key_bits"] = None if key_bits is None else f"{key_bits:x}" + source_info = data.as_dict(all_info=True).get("_source_k") + if source_info is not None: + out[f"row[{row}]._source_k"] = _jsonable(source_info) + return out diff --git a/tests/test_hashing/golden/data/v1/pipeline-02.json b/tests/test_hashing/golden/data/v1/pipeline-02.json index 268c8da0b..f542d4746 100644 --- a/tests/test_hashing/golden/data/v1/pipeline-02.json +++ b/tests/test_hashing/golden/data/v1/pipeline-02.json @@ -137,16 +137,46 @@ "_tag_record_id::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8::009b87c48e0540870894b8e1434a47f6e9150c21d7a401ecd9b791bdec215ee8" ], "group_by_nan_keys.content_hash": "semantic_v1:440a76d5318ec3d55c8cbdf569f468168ae35ef4ff3251e57f1ee1b173f835a7", + "group_by_nan_keys.data_tag_key.content_hash": "semantic_v1:94b46d0e4613d29dfe5f06623a86552f8e872f5eba3fa0d76ea3693c9e1c0b9c", + "group_by_nan_keys.data_tag_key.pipeline_hash": "semantic_v1:07ef40fee39a17a870e25d24a13d72d9c27195b8f17cc030717d980561ac3c94", + "group_by_nan_keys.data_tag_key.row[{\"k\":0.0}]._source_k": "group_v1:aabaa4e051213c68eeb096b4e38e9720d0f9531198dde40939113f3c8bc07416", + "group_by_nan_keys.data_tag_key.row[{\"k\":0.0}]._tag_record_id::955dd62c8087caae0a03dbe9bee3e42ca10d0673586b3fb3f2c3429913df3f86::955dd62c8087caae0a03dbe9bee3e42ca10d0673586b3fb3f2c3429913df3f86": "group_v1:a129e550a9e72a9c72e074e0ed3afffce39ada58073e6e1dcf018b03d309da55", + "group_by_nan_keys.data_tag_key.row[{\"k\":0.0}]._tag_source_id::955dd62c8087caae0a03dbe9bee3e42ca10d0673586b3fb3f2c3429913df3f86::955dd62c8087caae0a03dbe9bee3e42ca10d0673586b3fb3f2c3429913df3f86": "group_v1:9a86e4293f06b800ba18b6120618af58a9d2443ac6aabfc23a867911ac83d531", + "group_by_nan_keys.data_tag_key.row[{\"k\":0.0}].data_hash": "arrow_v1:0000026eb6abde2a57e28a9af20033f401e7d5be56590255423dd38578ea4b8bc6d8fa", + "group_by_nan_keys.data_tag_key.row[{\"k\":0.0}].key_bits": "0", + "group_by_nan_keys.data_tag_key.row[{\"k\":0.0}].tag_hash": "arrow_v1:000002c3c7a7994cff84945af2d6431f5c326caaa9730c66c8bfea219445e4e7e6588e", + "group_by_nan_keys.data_tag_key.row[{\"k\":NaN}]._source_k": "group_v1:b799bbb52cdc4a7496fa1f9213f4c079fe579069a166f8f7c8f25584f7b49e43", + "group_by_nan_keys.data_tag_key.row[{\"k\":NaN}]._tag_record_id::955dd62c8087caae0a03dbe9bee3e42ca10d0673586b3fb3f2c3429913df3f86::955dd62c8087caae0a03dbe9bee3e42ca10d0673586b3fb3f2c3429913df3f86": "group_v1:15a58a6c8160f20635066df45354c75ee2e2446220d8218f3513910379604b62", + "group_by_nan_keys.data_tag_key.row[{\"k\":NaN}]._tag_source_id::955dd62c8087caae0a03dbe9bee3e42ca10d0673586b3fb3f2c3429913df3f86::955dd62c8087caae0a03dbe9bee3e42ca10d0673586b3fb3f2c3429913df3f86": "group_v1:4a54373ba212a3ac8a5bb554667e191d09090baf16c3c43c06d9999f4df5c487", + "group_by_nan_keys.data_tag_key.row[{\"k\":NaN}].data_hash": "arrow_v1:000002ca3551166efdb59cabd35a1c5e605d234fce59c5972bc18dcf917b6c3b0602a5", + "group_by_nan_keys.data_tag_key.row[{\"k\":NaN}].key_bits": "7f800001", + "group_by_nan_keys.data_tag_key.row[{\"k\":NaN}].tag_hash": "arrow_v1:000002586e50f76d93b46583813186e82a5b179d9aaf2ecdf67e5a7da55e7b5083c098", + "group_by_nan_keys.data_tag_key.row[{\"k\":null}]._source_k": "group_v1:95c9de130390ceb327414d685bb28e39b88536215ff54bf3419a0eb831be63f8", + "group_by_nan_keys.data_tag_key.row[{\"k\":null}]._tag_record_id::955dd62c8087caae0a03dbe9bee3e42ca10d0673586b3fb3f2c3429913df3f86::955dd62c8087caae0a03dbe9bee3e42ca10d0673586b3fb3f2c3429913df3f86": "group_v1:e94b9ef076656ab5b5f91eece564251d16ab77e88be2db5262283ffabe061f2f", + "group_by_nan_keys.data_tag_key.row[{\"k\":null}]._tag_source_id::955dd62c8087caae0a03dbe9bee3e42ca10d0673586b3fb3f2c3429913df3f86::955dd62c8087caae0a03dbe9bee3e42ca10d0673586b3fb3f2c3429913df3f86": "group_v1:d47fca34430e43831cbbda6a8bc924808c01fbd2ce82830d868fe564323f0923", + "group_by_nan_keys.data_tag_key.row[{\"k\":null}].data_hash": "arrow_v1:000002b80cd3a5b8f1963552a06ceb7ecce1c32c72b533558a6a6dbbdfbe1d7b12db38", + "group_by_nan_keys.data_tag_key.row[{\"k\":null}].key_bits": null, + "group_by_nan_keys.data_tag_key.row[{\"k\":null}].tag_hash": "arrow_v1:0000020d9462d917b515c59d08a022c6328fa90c882ad01a9afd0568720d5fe047541e", + "group_by_nan_keys.data_tag_key.system_tag_columns": [ + "_tag_source_id::955dd62c8087caae0a03dbe9bee3e42ca10d0673586b3fb3f2c3429913df3f86::955dd62c8087caae0a03dbe9bee3e42ca10d0673586b3fb3f2c3429913df3f86", + "_tag_record_id::955dd62c8087caae0a03dbe9bee3e42ca10d0673586b3fb3f2c3429913df3f86::955dd62c8087caae0a03dbe9bee3e42ca10d0673586b3fb3f2c3429913df3f86" + ], "group_by_nan_keys.pipeline_hash": "semantic_v1:757b8297a21932d70dd5f816742c2fb0eb159bde455f83373eb3f9a31591d46b", "group_by_nan_keys.row[{\"k\":0.0}]._tag_record_id::b9b2a71e04d7acbbbcb6d4e9ed74068893840cbed90077e3567c6851b533a656::b9b2a71e04d7acbbbcb6d4e9ed74068893840cbed90077e3567c6851b533a656": "group_v1:a129e550a9e72a9c72e074e0ed3afffce39ada58073e6e1dcf018b03d309da55", "group_by_nan_keys.row[{\"k\":0.0}]._tag_source_id::b9b2a71e04d7acbbbcb6d4e9ed74068893840cbed90077e3567c6851b533a656::b9b2a71e04d7acbbbcb6d4e9ed74068893840cbed90077e3567c6851b533a656": "group_v1:006df9f09c1fa47b2eb819117849cfc1a1781b3a244b3a05fe0dc07fdb8af1a2", "group_by_nan_keys.row[{\"k\":0.0}].data_hash": "arrow_v1:000002529ddb22d5950e5c74cf80d618ffc83f8cf705a56f197403ec4e1835a06ba677", + "group_by_nan_keys.row[{\"k\":0.0}].key_bits": "0", + "group_by_nan_keys.row[{\"k\":0.0}].tag_hash": "arrow_v1:0000025651fc4f2e407c80d32ec94f8b7c9266e23136d476d7f80618055473c366e04f", "group_by_nan_keys.row[{\"k\":NaN}]._tag_record_id::b9b2a71e04d7acbbbcb6d4e9ed74068893840cbed90077e3567c6851b533a656::b9b2a71e04d7acbbbcb6d4e9ed74068893840cbed90077e3567c6851b533a656": "group_v1:15a58a6c8160f20635066df45354c75ee2e2446220d8218f3513910379604b62", "group_by_nan_keys.row[{\"k\":NaN}]._tag_source_id::b9b2a71e04d7acbbbcb6d4e9ed74068893840cbed90077e3567c6851b533a656::b9b2a71e04d7acbbbcb6d4e9ed74068893840cbed90077e3567c6851b533a656": "group_v1:7600c5c59642d9cc035a04f3d2694fe97220904595fdca39214ab49f21b42047", "group_by_nan_keys.row[{\"k\":NaN}].data_hash": "arrow_v1:0000024fb6c35ca785c5d07b688027a823d5c9c565ceadadf5d3648926d84df0755a22", + "group_by_nan_keys.row[{\"k\":NaN}].key_bits": "7ff8000000000000", + "group_by_nan_keys.row[{\"k\":NaN}].tag_hash": "arrow_v1:0000026c964d20115d230210455bca1605a00a2fab6412b4de1081e8ee180eb28ba70b", "group_by_nan_keys.row[{\"k\":null}]._tag_record_id::b9b2a71e04d7acbbbcb6d4e9ed74068893840cbed90077e3567c6851b533a656::b9b2a71e04d7acbbbcb6d4e9ed74068893840cbed90077e3567c6851b533a656": "group_v1:c15b4d7880e5fd711b2b799ad3b08acdbefb04c8ca67c51f284cf5cb1c9f1628", "group_by_nan_keys.row[{\"k\":null}]._tag_source_id::b9b2a71e04d7acbbbcb6d4e9ed74068893840cbed90077e3567c6851b533a656::b9b2a71e04d7acbbbcb6d4e9ed74068893840cbed90077e3567c6851b533a656": "group_v1:006df9f09c1fa47b2eb819117849cfc1a1781b3a244b3a05fe0dc07fdb8af1a2", "group_by_nan_keys.row[{\"k\":null}].data_hash": "arrow_v1:0000028a9bb897af2967d6055081dad273123417a319e33b18b6c44c68551b23f89b8b", + "group_by_nan_keys.row[{\"k\":null}].key_bits": null, + "group_by_nan_keys.row[{\"k\":null}].tag_hash": "arrow_v1:0000028511fb665d99c302face48678883c792c7ca0904c03075a5563ca42a6ce7804a", "group_by_nan_keys.system_tag_columns": [ "_tag_source_id::b9b2a71e04d7acbbbcb6d4e9ed74068893840cbed90077e3567c6851b533a656::b9b2a71e04d7acbbbcb6d4e9ed74068893840cbed90077e3567c6851b533a656", "_tag_record_id::b9b2a71e04d7acbbbcb6d4e9ed74068893840cbed90077e3567c6851b533a656::b9b2a71e04d7acbbbcb6d4e9ed74068893840cbed90077e3567c6851b533a656" From 9b71556e509293a0c72c611a431315cab675a4ba Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 17 Nov 2026 22:02:32 -0800 Subject: [PATCH 3/4] fix(group_by): take only float-storage keys from Arrow; read a logical float key's storage bits (ITL-1234) Copilot's second review of #441. (A) A key that is not stored as a float goes back to the value rebuilt from Python, as on main: two time64[ns] values within one microsecond read as one datetime.time, and taking one member's Arrow value made the key depend on the row order (ITL-1250 groups such keys by their exact value). (B) A logical type stored as a float is chosen by the bits of its storage, and keeps its type on the output. Tests first (both failed on db50cf3f): a float-backed logical key in every row order and sync == async; time64[ns] pinned to main's key. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 7 +- CLAUDE.md | 7 +- changelog.d/ITL-1234.md | 3 +- docs/concepts/operators.md | 11 +- src/orcapod/core/operators/group_by.py | 53 +++++--- .../operators/test_group_by_nan_keys.py | 117 +++++++++++++++++- 6 files changed, 175 insertions(+), 23 deletions(-) diff --git a/.zed/rules b/.zed/rules index bea2ff403..0abc30fd5 100644 --- a/.zed/rules +++ b/.zed/rules @@ -1200,7 +1200,12 @@ data. The rules: over a negative one; a signaling NaN has smaller bits than the canonical quiet NaN, so it wins), taken from the input's Arrow column, never through Python (a float32 signaling NaN comes back quiet from Python): always a value a member holds, never the first row's, never - chosen by `member_order`. Bits are read only for a group whose key is zero or NaN. + chosen by `member_order`. Bits are read only for a group whose key is zero or NaN. A logical + type stored as a float is a float key, read through its storage (`_float_storage`). Every + other key is rebuilt from its Python value, as before: equal Python values that differ in + Arrow (`time64[ns]` within a microsecond) are one group with one rebuilt key; grouping by + the exact value is ITL-1250. The stance on `0.0`/`-0.0` and NaN for the whole library + (match by value, as Polars does; show the smallest bit pattern; rewrite nothing) is ITL-1251. - Data source-info values may be str, None, or list[...], nested to any depth; Arrow/Python types are derived from the value, with None mapping to large_string. That rule has one definition, arrow_utils.source_info_arrow_type, and applies only to tokens supplied as diff --git a/CLAUDE.md b/CLAUDE.md index 10e50c757..e511cfc3b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1246,7 +1246,12 @@ data. The rules: over a negative one; a signaling NaN has smaller bits than the canonical quiet NaN, so it wins), taken from the input's Arrow column, never through Python (a float32 signaling NaN comes back quiet from Python): always a value a member holds, never the first row's, never - chosen by `member_order`. Bits are read only for a group whose key is zero or NaN. + chosen by `member_order`. Bits are read only for a group whose key is zero or NaN. A logical + type stored as a float is a float key, read through its storage (`_float_storage`). Every + other key is rebuilt from its Python value, as before: equal Python values that differ in + Arrow (`time64[ns]` within a microsecond) are one group with one rebuilt key; grouping by + the exact value is ITL-1250. The stance on `0.0`/`-0.0` and NaN for the whole library + (match by value, as Polars does; show the smallest bit pattern; rewrite nothing) is ITL-1251. - `Data` source-info values may be `str`, `None`, or `list[...]`, nested to any depth; the Arrow and Python types are derived from the value, with `None` mapping to `large_string`. That rule has one definition, `arrow_utils.source_info_arrow_type`, and applies only to diff --git a/changelog.d/ITL-1234.md b/changelog.d/ITL-1234.md index dfcb13df9..51e42d311 100644 --- a/changelog.d/ITL-1234.md +++ b/changelog.d/ITL-1234.md @@ -16,4 +16,5 @@ `0.0` and `-0.0` were already one group, but the group emitted the key of the first row in input order, so its value and the row's hashes depended on the row order. The key is now taken from the input's Arrow column: a `float32` signaling NaN key is emitted with - its bits (it came back quiet). A group whose members agree emits their value as before. + its bits (it came back quiet). A logical type stored as a float is chosen by the bits of + its storage. A group whose members agree emits their value as before. diff --git a/docs/concepts/operators.md b/docs/concepts/operators.md index 3a8b08f7c..b5d42a0e2 100644 --- a/docs/concepts/operators.md +++ b/docs/concepts/operators.md @@ -321,11 +321,12 @@ cache keys: equals NaN here; `0.0` equals `-0.0`). A NaN key sorts after every number and before null. When the members of a group hold a float key with different bits (`0.0` and `-0.0`, or NaN with another sign or payload), the group emits the one with the smallest bit pattern - at the column's own width (`float16`, `float32` or `float64`), for each key column: `0.0` - over `-0.0`, and a positive NaN over a negative one. A signaling NaN has smaller bits than - the canonical quiet NaN, so when a member holds one, it is the key emitted. The emitted key - is always a value a member holds, bit for bit, and neither the order of the rows nor - `member_order` decides it. + at the column's own width (`float16`, `float32` or `float64`; a logical type stored as a + float is read through its storage), for each key column: `0.0` over `-0.0`, and a + positive NaN over a negative one. A signaling NaN has smaller bits than the canonical + quiet NaN, so when a member holds one, it is the key emitted. The emitted key is always a + value a member holds, bit for bit, and neither the order of the rows nor `member_order` + decides it. - **No arrangement of the input changes the result.** The order of its tags, the order of its data columns and the layout of its table leave every value, record id and hash of the output unchanged. A reordered input produces a byte-identical output table. The output diff --git a/src/orcapod/core/operators/group_by.py b/src/orcapod/core/operators/group_by.py index 41ea9a77e..05e319344 100644 --- a/src/orcapod/core/operators/group_by.py +++ b/src/orcapod/core/operators/group_by.py @@ -71,6 +71,27 @@ def _grouping_key(key: tuple[Any, ...]) -> tuple[Any, ...]: ) +def _float_storage(arrow_type: "pa.DataType") -> "pa.DataType | None": + """Return the float type a column is stored as, or ``None`` when it is not a float. + + A logical (extension) type over float storage, such as a temperature stored as + ``float64``, is stored as that float. + + Args: + arrow_type: The Arrow type of a key column. + + Returns: + The ``float16``, ``float32`` or ``float64`` type of the column's storage, or + ``None``. + """ + storage = ( + arrow_type.storage_type + if isinstance(arrow_type, pa.ExtensionType) + else arrow_type + ) + return storage if pa.types.is_floating(storage) else None + + def _emitted_key_row(column: "pa.ChunkedArray[Any]", indices: Sequence[int]) -> int: """Return the input row whose value of one key column a group emits. @@ -94,22 +115,19 @@ def _emitted_key_row(column: "pa.ChunkedArray[Any]", indices: Sequence[int]) -> The row, one of ``indices``, whose value the group emits for this column. """ first = indices[0] - arrow_type = column.type - if len(indices) == 1 or not pa.types.is_floating(arrow_type): + storage_type = _float_storage(column.type) + if len(indices) == 1 or storage_type is None: return first - value = column[first].as_py() + members = column.take(pa.array(indices, pa.int64())).combine_chunks() + if isinstance(members, pa.ExtensionArray): + members = members.storage + value = cast(float | None, members[0].as_py()) if value is None or (value != 0.0 and not math.isnan(value)): return first # The unsigned integer of the float's width, to read its bits; no value of the # group is null, since a null key equals no float. - unsigned = {16: pa.uint16, 32: pa.uint32, 64: pa.uint64}[arrow_type.bit_width]() - bits = cast( - list[int], - column.take(pa.array(indices, pa.int64())) - .combine_chunks() - .view(unsigned) - .to_pylist(), - ) + unsigned = {16: pa.uint16, 32: pa.uint32, 64: pa.uint64}[storage_type.bit_width]() + bits = cast(list[int], members.view(unsigned).to_pylist()) return indices[min(range(len(indices)), key=bits.__getitem__)] @@ -530,10 +548,17 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: stream.data_context.type_converter, ) - # Each key column holds the chosen member's value as the input's Arrow - # column has it, never through Python: a conversion can change a float's - # bits (a float32 signaling NaN comes back quiet). + # A float key column (a float, or a logical type stored as one) holds the + # chosen member's value as the input's Arrow column has it, never through + # Python: a conversion can change a float's bits (a float32 signaling NaN + # comes back quiet). Every other key keeps the value rebuilt from Python, + # as before: its members may differ in Arrow although equal in Python (two + # time64[ns] values within one microsecond read as one ``datetime.time``), + # and taking one member's would make the key depend on the row order. + # Grouping such keys by their exact value is ITL-1250. for position, c in enumerate(self.by): + if _float_storage(table.column(c).type) is None: + continue chosen = pa.array([r[position] for r in key_rows], pa.int64()) grouped_table = grouped_table.set_column( grouped_table.schema.get_field_index(c), diff --git a/tests/test_core/operators/test_group_by_nan_keys.py b/tests/test_core/operators/test_group_by_nan_keys.py index ee6e1bcdc..9eea2301f 100644 --- a/tests/test_core/operators/test_group_by_nan_keys.py +++ b/tests/test_core/operators/test_group_by_nan_keys.py @@ -16,7 +16,7 @@ import itertools import json import struct -from typing import Any +from typing import Any, cast import pyarrow as pa import pytest @@ -24,6 +24,7 @@ import orcapod as op from orcapod.core.streams.base import StreamBase from orcapod.databases import InMemoryArrowDatabase +from orcapod.logical_types import make_arrow_extension_type from orcapod.pipeline import ( AsyncPipelineOrchestrator, PipelineJob, @@ -315,3 +316,117 @@ def test_a_signaling_nan_key_groups_the_same_sync_and_async( if not seen: seen = [fingerprint] assert fingerprint == seen[0], orchestrator.__name__ + + +# ── a float-backed logical type, and a key that is not a float ────────────── + +# A logical (extension) type over float64 storage, like ``example.celsius`` in +# docs/concepts/logical-type-identifiers.md. +FLOAT_BACKED_CLASS = make_arrow_extension_type( + "test.group_by_float_key", pa.float64(), b"v1" +) +# The made class takes no arguments; the stubs see pyarrow's base constructor. +FLOAT_BACKED = FLOAT_BACKED_CLASS() # pyright: ignore[reportCallIssue] + +LOGICAL_ROWS: list[dict[str, Any]] = [ + {"k": "nan", "rid": "r1", "v": 1}, + {"k": "-nan", "rid": "r2", "v": 2}, + {"k": "payload_nan", "rid": "r3", "v": 3}, + {"k": "-0", "rid": "r4", "v": 4}, + {"k": "0", "rid": "r5", "v": 5}, + {"k": None, "rid": "r6", "v": 6}, +] + + +def _logical_source(rows: list[dict[str, Any]]) -> StreamBase: + storage = pa.array( + [None if r["k"] is None else BITS[r["k"]][64] for r in rows], pa.uint64() + ).view(pa.float64()) + table = pa.table( + { + "k": pa.ExtensionArray.from_storage(FLOAT_BACKED, storage), + "rid": [r["rid"] for r in rows], + "v": [r["v"] for r in rows], + } + ) + return op.sources.ArrowTableSource( + table, + tag_columns=["k"], + tag_only_columns=True, + source_id="s", + record_id_columns="rid", + ) + + +def _logical_key_bits(table: pa.Table) -> list[int | None]: + column = table.column("k").combine_chunks() + assert column.type == FLOAT_BACKED # the output keeps the logical type + assert isinstance(column, pa.ExtensionArray) + return column.storage.view(pa.uint64()).to_pylist() + + +def test_a_float_backed_logical_key_emits_the_smallest_bits() -> None: + """The bits are read from the extension's float storage, in every row order.""" + seen: dict[str, Any] = {} + for rows in itertools.permutations(LOGICAL_ROWS): + grouped = _logical_source(list(rows)).group_by("k") + table = grouped.as_table() + assert table.column("v").to_pylist() == [[4, 5], [1, 2, 3], [6]] + assert _logical_key_bits(table) == [BITS["0"][64], BITS["nan"][64], None] + fingerprint = _fingerprint(grouped) + if not seen: + seen = fingerprint + assert fingerprint == seen + + +# The pipeline's Polars round trip reads the unregistered test type as its storage. +@pytest.mark.filterwarnings("ignore:Extension type 'test.group_by_float_key'") +def test_a_float_backed_logical_key_groups_the_same_sync_and_async() -> None: + seen: list[str] = [] + for rows in (LOGICAL_ROWS, LOGICAL_ROWS[::-1]): + for orchestrator in (SyncPipelineOrchestrator, AsyncPipelineOrchestrator): + job = PipelineJob(name="logical_key", database=InMemoryArrowDatabase()) + with job: + _ = _logical_source(list(rows)).group_by("k", label="grouped") + job.run(orchestrator=orchestrator()) + node = job.nodes["grouped"] + out = node.as_table(all_info=True) + assert _logical_key_bits(out) == [BITS["0"][64], BITS["nan"][64], None] + fingerprint = json.dumps( + { + "rows": _with_bits(out.to_pylist()), + "hashes": [ + ( + tag.content_hash().to_string(), + data.content_hash().to_string(), + ) + for tag, data in node.iter_data() + ], + }, + sort_keys=True, + default=str, + ) + if not seen: + seen = [fingerprint] + assert fingerprint == seen[0], orchestrator.__name__ + + +def test_a_time64_ns_key_is_rebuilt_from_its_python_value() -> None: + """Pins today's grouping of a key that is not a float, whatever the row order. + + ``GroupBy`` compares keys as Python values, and a ``time64[ns]`` value reads as a + ``datetime.time`` of microseconds: 1 ns and 2 ns form one group, which emits + the Python value rebuilt (0 ns), as on ``main``. Grouping by the exact value is + ITL-1250. Only a float key is taken from the Arrow column. + """ + for values in ([1, 2], [2, 1]): + table = pa.table( + {"k": pa.array(values, pa.int64()).cast(pa.time64("ns")), "v": values} + ) + source = op.sources.ArrowTableSource( + table, tag_columns=["k"], tag_only_columns=True, source_id="s" + ) + out = source.group_by("k").as_table() + assert out.column("k").cast(pa.int64()).to_pylist() == [0] + members = cast(list[list[int]], out.column("v").to_pylist()) + assert sorted(members[0]) == sorted(values) From 1db0583f640fdda6f540ead36396ff2058eded4c Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 17 Nov 2026 22:21:16 -0800 Subject: [PATCH 4/4] fix(group_by): check a group's first key before gathering it; changelog names mixed signed-zero groups (ITL-1234) Copilot (r4609854563): only a zero or NaN key can differ in bits among equal values, so the first member's value is read before the group is gathered; a group of any other float key is no longer copied. CodeRabbit (r4609830225): the changed inputs and the stored-rows guidance now include a group whose members mix 0.0 and -0.0. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1234.md | 10 ++++++---- src/orcapod/core/operators/group_by.py | 14 +++++++++++--- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/changelog.d/ITL-1234.md b/changelog.d/ITL-1234.md index 51e42d311..4f2448757 100644 --- a/changelog.d/ITL-1234.md +++ b/changelog.d/ITL-1234.md @@ -3,10 +3,12 @@ - `GroupBy` puts all rows whose key is NaN into one group (ITL-1234), as the joins match a NaN key; with several keys, the key tuples compare element by element with that rule. Before, each such row was a group of its own, and those groups came out in the input's - row order. A NaN key sorts after every number and before null. Only an input with a NaN - key changes: its groups, their member lists, folded ids and hashes. Stored rows of such - a `GroupBy` and of the nodes after it are not replaced: delete those tables or attach a - new database before running it with this version (`clear_cache()` clears memory only). + row order. A NaN key sorts after every number and before null. Only two kinds of input + change: one with a NaN key (its groups, member lists, folded ids and hashes), and one + with a group whose members mix `0.0` and `-0.0` (its key and hashes; see Fixed). Stored + rows of such a `GroupBy` and of the nodes after it are not replaced: delete those tables + or attach a new database before running it with this version (`clear_cache()` clears + memory only). ### Fixed diff --git a/src/orcapod/core/operators/group_by.py b/src/orcapod/core/operators/group_by.py index 05e319344..078f059b1 100644 --- a/src/orcapod/core/operators/group_by.py +++ b/src/orcapod/core/operators/group_by.py @@ -118,12 +118,20 @@ def _emitted_key_row(column: "pa.ChunkedArray[Any]", indices: Sequence[int]) -> storage_type = _float_storage(column.type) if len(indices) == 1 or storage_type is None: return first + # Only a zero or NaN key can differ in bits among equal values, so the first + # member's value decides whether the group is gathered at all. + scalar = column[first] + if isinstance(scalar, pa.ExtensionScalar): + # A null extension scalar has no storage scalar. + storage = scalar.value + value = None if storage is None else cast(float | None, storage.as_py()) + else: + value = cast(float | None, scalar.as_py()) + if value is None or (value != 0.0 and not math.isnan(value)): + return first members = column.take(pa.array(indices, pa.int64())).combine_chunks() if isinstance(members, pa.ExtensionArray): members = members.storage - value = cast(float | None, members[0].as_py()) - if value is None or (value != 0.0 and not math.isnan(value)): - return first # The unsigned integer of the float's width, to read its bits; no value of the # group is null, since a null key equals no float. unsigned = {16: pa.uint16, 32: pa.uint32, 64: pa.uint64}[storage_type.bit_width]()