diff --git a/changelog.d/ITL-1046.md b/changelog.d/ITL-1046.md new file mode 100644 index 000000000..71d1bd3ef --- /dev/null +++ b/changelog.d/ITL-1046.md @@ -0,0 +1,3 @@ +### Fixed + +- `Index` after `Batch` or `GroupBy` now gives the extracted element its own provenance token (ITL-1046). The `_source_` value was the Python text of the whole token list followed by `[i]` (`"['src::x=1::x', 'src::x=2::x'][0]"`), which no resolver can parse. It is now the member's token (`'src::x=1::x'`), one list level removed at every depth (a column aggregated twice gives the inner list), and null when the row is skipped. A scalar token keeps the `[i]` form. No hash, record id or golden value changes. Tables already written keep the old token text; upgrading stored data is the job of the migration tool (ITL-1048). diff --git a/design_issues/ITL-1046-IX1.md b/design_issues/ITL-1046-IX1.md new file mode 100644 index 000000000..9c13481d9 --- /dev/null +++ b/design_issues/ITL-1046-IX1.md @@ -0,0 +1,21 @@ +## `src/orcapod/core/operators/index.py` — Index and Pick provenance tokens + +### IX1 — Index after Batch or GroupBy wrote the text of a whole token list plus `[i]` +**Status:** resolved +**Severity:** medium +**Issue:** ITL-1046 + +After `Batch` or `GroupBy` the source-info token of a column is a list, one token per member. +`Index` built the element's token as `f"{old_src}[{i}]"`, which for a list is the Python text +of the whole list followed by the index (`"['src::x=1::x', 'src::x=2::x'][0]"`). No resolver +can parse it. + +**Fix:** when the incoming token is a list, `Index` takes the element's own token +(`old_src[i]`, negative indices as in Python), one list level removed at every depth: a +column aggregated twice gives the inner list. A scalar token keeps the `[i]` form. +The token's Arrow type follows `arrow_utils.source_info_arrow_type` (`large_string` after one +aggregation, `large_list` after two); zero-row and non-empty results agree, as do +the sync and async paths. No hash, record id or golden value depends on the token (a `Data` +content hash excludes source info; the pipeline hash is schema-only). `Pick` is not affected: +its column is a `dict`, which no aggregation produces. Stored tables written with the old text +keep it; upgrading them is the migration tool's job (ITL-1048). diff --git a/src/orcapod/core/operators/index.py b/src/orcapod/core/operators/index.py index 2d15383c5..c2c00e8f6 100644 --- a/src/orcapod/core/operators/index.py +++ b/src/orcapod/core/operators/index.py @@ -9,10 +9,11 @@ from orcapod.channels import ReadableChannel, WritableChannel from orcapod.core.operators.base import UnaryOperator +from orcapod.system_constants import constants from orcapod.core.streams.arrow_table_stream import ArrowTableStream from orcapod.errors import InputValidationError from orcapod.protocols.core_protocols import DataProtocol, StreamProtocol, TagProtocol -from orcapod.types import ColumnConfig, ContentHash, Schema +from orcapod.types import ColumnConfig, ContentHash, Schema, SourceInfoValue from orcapod.utils import arrow_utils from orcapod.utils.lazy_module import LazyModule @@ -286,7 +287,15 @@ def _process_one( ) old_src = data.source_info().get(self.column) - new_src = f"{old_src}[{self.i}]" if old_src else None + new_src: SourceInfoValue + if isinstance(old_src, list): + # A many->one operator (Batch, GroupBy) left one token per member: the + # element's provenance is its own token, one list level removed (the + # inner list after two aggregations). ``effective_i`` is in range for + # the value, and a token list holds one entry per member. + new_src = old_src[effective_i] if effective_i < len(old_src) else None + else: + new_src = f"{old_src}[{self.i}]" if old_src else None element_field = pa.field( self.out or self.column, @@ -354,8 +363,11 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: # The schema is the input's Arrow schema (large-normalized, as for # a non-empty result) with only the projected column changed. tag_columns, _ = stream.keys() + # The source-info columns are part of it: the projected column's token + # type is the input's with one list level removed, as for a non-empty + # result (ITL-1046), and every other token column keeps its input type. input_schema = arrow_utils.normalize_schema_to_large_types( - stream.as_table().schema + stream.as_table(columns={"source": True}).schema ) element_field = pa.field( self.out or self.column, @@ -364,11 +376,28 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: ), nullable=self._output_nullable(), ) + token_name = f"{constants.SOURCE_PREFIX}{self.column}" + input_token = input_schema.field(token_name) + input_token_type = input_token.type + token_type = ( + input_token_type.value_type + if pa.types.is_large_list(input_token_type) + else input_token_type + ) + token_field = pa.field( + f"{constants.SOURCE_PREFIX}{self.out or self.column}", + token_type, + nullable=input_token.nullable, + ) if self.out is None: position = input_schema.get_field_index(self.column) output_schema = input_schema.set(position, element_field) # pyright: ignore[reportUnknownMemberType] # stubs leave Field unparametrized + output_schema = output_schema.set( + output_schema.get_field_index(token_name), token_field + ) # pyright: ignore[reportUnknownMemberType] # stubs leave Field unparametrized else: output_schema = input_schema.append(element_field) # pyright: ignore[reportUnknownMemberType] # stubs leave Field unparametrized + output_schema = output_schema.append(token_field) # pyright: ignore[reportUnknownMemberType] # stubs leave Field unparametrized return ArrowTableStream( output_schema.empty_table(), tag_columns=tag_columns, diff --git a/superpowers/plans/2026-11-13-itl-1046-index-list-source-token.md b/superpowers/plans/2026-11-13-itl-1046-index-list-source-token.md new file mode 100644 index 000000000..9e807cf70 --- /dev/null +++ b/superpowers/plans/2026-11-13-itl-1046-index-list-source-token.md @@ -0,0 +1,25 @@ +# ITL-1046 plan: Index after Batch/GroupBy writes the element's own token + +Worker notes (a successor can continue from here). + +## Bug +`Index._process_one`: `new_src = f"{old_src}[{i}]" if old_src else None`. When `old_src` is a +list (token of a many->one stream) the f-string stringifies the list. + +## Fix +- Scalar token: unchanged (`f"{old_src}[{i}]"`, null when falsy). +- List token: `old_src[effective_i]` (the element's own token; `effective_i` handles negatives; the + index is in range, since out-of-range rows are skipped before this point). Nested lists: the + element is the inner list, so one level of nesting is removed. A list token shorter than the + value (cannot happen from Batch/GroupBy) yields null. +- Type: `Data.with_source_info` types the token from the Python value + (`arrow_utils.source_info_arrow_type`), so one level is removed by construction. +- Pick: never sees a list token for a dict column from Batch? (Batch makes `list[dict]`, Pick needs + a dict). Checked below. + +## Tests (public interface, inputs not in tag order) +tests/test_core/operators/test_index_source_token.py: Batch->Index, GroupBy->Index, two +aggregations->Index, out-of-range (skip) , negative index, zero-row, async == sync. + +## Out of scope +Stored tables already written keep old tokens (migration tool, ITL-1048). diff --git a/tests/test_core/operators/test_index_source_token.py b/tests/test_core/operators/test_index_source_token.py new file mode 100644 index 000000000..79a1c93de --- /dev/null +++ b/tests/test_core/operators/test_index_source_token.py @@ -0,0 +1,155 @@ +"""Index after an aggregation takes the element's own provenance token (ITL-1046). + +After ``Batch`` or ``GroupBy`` a column's source-info token is a list, one token per +member. ``Index`` must give the extracted element the token of that member, not the text +of the whole list followed by ``[i]``. A scalar token keeps the ``[i]`` form. +""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import pyarrow as pa + +from orcapod.channels import Channel +from orcapod.core.operators import Batch, GroupBy, Index +from orcapod.core.sources import ArrowTableSource +from orcapod.protocols.core_protocols import StreamProtocol + +SRC = "src" + + +def _source() -> StreamProtocol: + """Five rows, deliberately not in tag order (ids 3, 1, 2, 5, 4).""" + table = pa.table( + { + "k": pa.array(["b", "a", "b", "a", "b"]), + "id": pa.array([3, 1, 2, 5, 4], pa.int64()), + "x": pa.array([30, 10, 20, 50, 40], pa.int64()), + } + ) + return ArrowTableSource(table, tag_columns=["k", "id"], source_id=SRC) + + +def _tokens(stream: StreamProtocol, column: str) -> list[Any]: + table = stream.as_table(columns={"source": True}) + return table.column(f"_source_{column}").to_pylist() + + +def _token_type(stream: StreamProtocol, column: str) -> pa.DataType: + return ( + stream.as_table(columns={"source": True}).schema.field(f"_source_{column}").type + ) + + +def test_batch_then_index_takes_the_members_own_token() -> None: + batched = Batch(2)(_source()) + members = _tokens(batched, "x") + result = Index("x", 0, out="first")(batched) + + assert _tokens(result, "first") == [m[0] for m in members] + assert all(isinstance(t, str) and "[" not in t for t in _tokens(result, "first")) + assert _token_type(result, "first") == pa.large_string() + # the untouched column keeps its list token + assert _tokens(result, "x") == members + + +def test_batch_then_index_negative_index() -> None: + batched = Batch(2)(_source()) + members = _tokens(batched, "x") + result = Index("x", -1, out="last")(batched) + assert _tokens(result, "last") == [m[-1] for m in members] + + +def test_batch_then_index_in_place_replaces_the_token() -> None: + batched = Batch(2)(_source()) + members = _tokens(batched, "x") + result = Index("x", 1)(batched.select_data_columns("x")) + # row 3 is a batch of one: index 1 is out of range, so the row is skipped + assert _tokens(result, "x") == [m[1] for m in members if len(m) > 1] + + +def test_out_of_range_index_skips_the_row() -> None: + batched = Batch(2)(_source()) + result = Index("x", 1, out="second")(batched) + assert result.as_table().num_rows == 2 + assert all(isinstance(t, str) for t in _tokens(result, "second")) + + +def test_group_by_then_index_takes_the_members_own_token() -> None: + grouped = GroupBy(["k"])(_source()) + members = _tokens(grouped, "x") + result = Index("x", 0, out="first")(grouped) + + assert _tokens(result, "first") == [m[0] for m in members] + assert _token_type(result, "first") == pa.large_string() + + +def test_two_aggregations_then_index_gives_the_inner_list() -> None: + twice = Batch(2)(GroupBy(["k"])(_source())) + members = _tokens(twice, "x") # list of lists of tokens + assert all(isinstance(m[0], list) for m in members) + result = Index("x", 0, out="first")(twice) + + assert _tokens(result, "first") == [m[0] for m in members] + assert _token_type(result, "first") == pa.large_list(pa.large_string()) + + +def test_zero_row_result_has_the_same_token_type_as_a_non_empty_one() -> None: + batched = Batch(2)(_source()) + non_empty = Index("x", 0, out="first")(batched) + empty = Index("x", 5, out="first")(batched) + assert empty.as_table().num_rows == 0 + assert _token_type(empty, "first") == _token_type(non_empty, "first") + + +def test_scalar_token_keeps_the_indexed_form() -> None: + table = pa.table( + { + "id": pa.array([2, 1], pa.int64()), + "v": pa.array([[1, 2], [3, 4]], pa.list_(pa.int64())), + }, + schema=pa.schema( + [ + pa.field("id", pa.int64(), nullable=False), + pa.field("v", pa.list_(pa.int64()), nullable=False), + ] + ), + ) + source = ArrowTableSource(table, tag_columns=["id"], source_id=SRC) + before = _tokens(source, "v") + result = Index("v", 1, out="second")(source) + assert _tokens(result, "second") == [f"{t}[1]" for t in before] + + +def test_async_path_yields_the_same_tokens_as_the_sync_path() -> None: + batched = Batch(2)(_source()) + op = Index("x", 0, out="first") + sync_tokens = _tokens(op(batched), "first") + + async def run() -> list[Any]: + op.validate_inputs(batched) + source = Channel(buffer_size=16) + out = Channel(buffer_size=16) + for tag, data in batched.iter_data(): + await source.writer.send((tag, data)) + await source.writer.close() + await op.async_execute( + [source.reader], out.writer, input_pipeline_hashes=[batched.pipeline_hash()] + ) + return [data.source_info()["first"] for _, data in await out.reader.collect()] + + assert asyncio.run(run()) == sync_tokens + + +def test_zero_row_result_has_every_token_type_of_a_non_empty_one() -> None: + # Two aggregations: the projected token is one list level, the others keep theirs. + twice = Batch(2)(GroupBy(["k"])(_source())) + for out in ("first", None): + non_empty = Index("x", 0, out=out)(twice) + empty = Index("x", 9, out=out)(twice) + assert empty.as_table().num_rows == 0 + empty_schema = empty.as_table(columns={"source": True}).schema + full_schema = non_empty.as_table(columns={"source": True}).schema + assert empty_schema == full_schema