Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions changelog.d/ITL-1046.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
### Fixed

- `Index` after `Batch` or `GroupBy` now gives the extracted element its own provenance token (ITL-1046). The `_source_<column>` value was the Python text of the whole token list followed by `[i]` (`"['src::x=1::x', 'src::x=2::x'][0]"`), which no resolver can parse. It is now the member's token (`'src::x=1::x'`), one list level removed at every depth (a column aggregated twice gives the inner list), and null when the row is skipped. A scalar token keeps the `<token>[i]` form. No hash, record id or golden value changes. Tables already written keep the old token text; upgrading stored data is the job of the migration tool (ITL-1048).
21 changes: 21 additions & 0 deletions design_issues/ITL-1046-IX1.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
## `src/orcapod/core/operators/index.py` — Index and Pick provenance tokens

### IX1 — Index after Batch or GroupBy wrote the text of a whole token list plus `[i]`
**Status:** resolved
**Severity:** medium
**Issue:** ITL-1046

After `Batch` or `GroupBy` the source-info token of a column is a list, one token per member.
`Index` built the element's token as `f"{old_src}[{i}]"`, which for a list is the Python text
of the whole list followed by the index (`"['src::x=1::x', 'src::x=2::x'][0]"`). No resolver
can parse it.

**Fix:** when the incoming token is a list, `Index` takes the element's own token
(`old_src[i]`, negative indices as in Python), one list level removed at every depth: a
column aggregated twice gives the inner list. A scalar token keeps the `<token>[i]` form.
The token's Arrow type follows `arrow_utils.source_info_arrow_type` (`large_string` after one
aggregation, `large_list<large_string>` after two); zero-row and non-empty results agree, as do
the sync and async paths. No hash, record id or golden value depends on the token (a `Data`
content hash excludes source info; the pipeline hash is schema-only). `Pick` is not affected:
its column is a `dict`, which no aggregation produces. Stored tables written with the old text
keep it; upgrading them is the migration tool's job (ITL-1048).
35 changes: 32 additions & 3 deletions src/orcapod/core/operators/index.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,11 @@

from orcapod.channels import ReadableChannel, WritableChannel
from orcapod.core.operators.base import UnaryOperator
from orcapod.system_constants import constants
from orcapod.core.streams.arrow_table_stream import ArrowTableStream
from orcapod.errors import InputValidationError
from orcapod.protocols.core_protocols import DataProtocol, StreamProtocol, TagProtocol
from orcapod.types import ColumnConfig, ContentHash, Schema
from orcapod.types import ColumnConfig, ContentHash, Schema, SourceInfoValue
from orcapod.utils import arrow_utils
from orcapod.utils.lazy_module import LazyModule

Expand Down Expand Up @@ -286,7 +287,15 @@ def _process_one(
)

old_src = data.source_info().get(self.column)
new_src = f"{old_src}[{self.i}]" if old_src else None
new_src: SourceInfoValue
if isinstance(old_src, list):
# A many->one operator (Batch, GroupBy) left one token per member: the
# element's provenance is its own token, one list level removed (the
# inner list after two aggregations). ``effective_i`` is in range for
# the value, and a token list holds one entry per member.
new_src = old_src[effective_i] if effective_i < len(old_src) else None
else:
new_src = f"{old_src}[{self.i}]" if old_src else None

element_field = pa.field(
self.out or self.column,
Expand Down Expand Up @@ -354,8 +363,11 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol:
# The schema is the input's Arrow schema (large-normalized, as for
# a non-empty result) with only the projected column changed.
tag_columns, _ = stream.keys()
# The source-info columns are part of it: the projected column's token
# type is the input's with one list level removed, as for a non-empty
# result (ITL-1046), and every other token column keeps its input type.
input_schema = arrow_utils.normalize_schema_to_large_types(
stream.as_table().schema
stream.as_table(columns={"source": True}).schema
)
element_field = pa.field(
self.out or self.column,
Expand All @@ -364,11 +376,28 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol:
),
nullable=self._output_nullable(),
)
token_name = f"{constants.SOURCE_PREFIX}{self.column}"
input_token = input_schema.field(token_name)
input_token_type = input_token.type
token_type = (
input_token_type.value_type
if pa.types.is_large_list(input_token_type)
else input_token_type
)
token_field = pa.field(
f"{constants.SOURCE_PREFIX}{self.out or self.column}",
token_type,
nullable=input_token.nullable,
)
if self.out is None:
position = input_schema.get_field_index(self.column)
output_schema = input_schema.set(position, element_field) # pyright: ignore[reportUnknownMemberType] # stubs leave Field unparametrized
output_schema = output_schema.set(
output_schema.get_field_index(token_name), token_field
) # pyright: ignore[reportUnknownMemberType] # stubs leave Field unparametrized
else:
output_schema = input_schema.append(element_field) # pyright: ignore[reportUnknownMemberType] # stubs leave Field unparametrized
output_schema = output_schema.append(token_field) # pyright: ignore[reportUnknownMemberType] # stubs leave Field unparametrized
return ArrowTableStream(
output_schema.empty_table(),
tag_columns=tag_columns,
Expand Down
25 changes: 25 additions & 0 deletions superpowers/plans/2026-11-13-itl-1046-index-list-source-token.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# ITL-1046 plan: Index after Batch/GroupBy writes the element's own token

Worker notes (a successor can continue from here).

## Bug
`Index._process_one`: `new_src = f"{old_src}[{i}]" if old_src else None`. When `old_src` is a
list (token of a many->one stream) the f-string stringifies the list.

## Fix
- Scalar token: unchanged (`f"{old_src}[{i}]"`, null when falsy).
- List token: `old_src[effective_i]` (the element's own token; `effective_i` handles negatives; the
index is in range, since out-of-range rows are skipped before this point). Nested lists: the
element is the inner list, so one level of nesting is removed. A list token shorter than the
value (cannot happen from Batch/GroupBy) yields null.
- Type: `Data.with_source_info` types the token from the Python value
(`arrow_utils.source_info_arrow_type`), so one level is removed by construction.
- Pick: never sees a list token for a dict column from Batch? (Batch makes `list[dict]`, Pick needs
a dict). Checked below.

## Tests (public interface, inputs not in tag order)
tests/test_core/operators/test_index_source_token.py: Batch->Index, GroupBy->Index, two
aggregations->Index, out-of-range (skip) , negative index, zero-row, async == sync.

## Out of scope
Stored tables already written keep old tokens (migration tool, ITL-1048).
155 changes: 155 additions & 0 deletions tests/test_core/operators/test_index_source_token.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
"""Index after an aggregation takes the element's own provenance token (ITL-1046).

After ``Batch`` or ``GroupBy`` a column's source-info token is a list, one token per
member. ``Index`` must give the extracted element the token of that member, not the text
of the whole list followed by ``[i]``. A scalar token keeps the ``<token>[i]`` form.
"""

from __future__ import annotations

import asyncio
from typing import Any

import pyarrow as pa

from orcapod.channels import Channel
from orcapod.core.operators import Batch, GroupBy, Index
from orcapod.core.sources import ArrowTableSource
from orcapod.protocols.core_protocols import StreamProtocol

SRC = "src"


def _source() -> StreamProtocol:
"""Five rows, deliberately not in tag order (ids 3, 1, 2, 5, 4)."""
table = pa.table(
{
"k": pa.array(["b", "a", "b", "a", "b"]),
"id": pa.array([3, 1, 2, 5, 4], pa.int64()),
"x": pa.array([30, 10, 20, 50, 40], pa.int64()),
}
)
return ArrowTableSource(table, tag_columns=["k", "id"], source_id=SRC)


def _tokens(stream: StreamProtocol, column: str) -> list[Any]:
table = stream.as_table(columns={"source": True})
return table.column(f"_source_{column}").to_pylist()


def _token_type(stream: StreamProtocol, column: str) -> pa.DataType:
return (
stream.as_table(columns={"source": True}).schema.field(f"_source_{column}").type
)


def test_batch_then_index_takes_the_members_own_token() -> None:
batched = Batch(2)(_source())
members = _tokens(batched, "x")
result = Index("x", 0, out="first")(batched)

assert _tokens(result, "first") == [m[0] for m in members]
assert all(isinstance(t, str) and "[" not in t for t in _tokens(result, "first"))
assert _token_type(result, "first") == pa.large_string()
# the untouched column keeps its list token
assert _tokens(result, "x") == members


def test_batch_then_index_negative_index() -> None:
batched = Batch(2)(_source())
members = _tokens(batched, "x")
result = Index("x", -1, out="last")(batched)
assert _tokens(result, "last") == [m[-1] for m in members]


def test_batch_then_index_in_place_replaces_the_token() -> None:
batched = Batch(2)(_source())
members = _tokens(batched, "x")
result = Index("x", 1)(batched.select_data_columns("x"))
# row 3 is a batch of one: index 1 is out of range, so the row is skipped
assert _tokens(result, "x") == [m[1] for m in members if len(m) > 1]


def test_out_of_range_index_skips_the_row() -> None:
batched = Batch(2)(_source())
result = Index("x", 1, out="second")(batched)
assert result.as_table().num_rows == 2
assert all(isinstance(t, str) for t in _tokens(result, "second"))


def test_group_by_then_index_takes_the_members_own_token() -> None:
grouped = GroupBy(["k"])(_source())
members = _tokens(grouped, "x")
result = Index("x", 0, out="first")(grouped)

assert _tokens(result, "first") == [m[0] for m in members]
assert _token_type(result, "first") == pa.large_string()


def test_two_aggregations_then_index_gives_the_inner_list() -> None:
twice = Batch(2)(GroupBy(["k"])(_source()))
members = _tokens(twice, "x") # list of lists of tokens
assert all(isinstance(m[0], list) for m in members)
result = Index("x", 0, out="first")(twice)

assert _tokens(result, "first") == [m[0] for m in members]
assert _token_type(result, "first") == pa.large_list(pa.large_string())


def test_zero_row_result_has_the_same_token_type_as_a_non_empty_one() -> None:
batched = Batch(2)(_source())
non_empty = Index("x", 0, out="first")(batched)
empty = Index("x", 5, out="first")(batched)
assert empty.as_table().num_rows == 0
assert _token_type(empty, "first") == _token_type(non_empty, "first")


def test_scalar_token_keeps_the_indexed_form() -> None:
table = pa.table(
{
"id": pa.array([2, 1], pa.int64()),
"v": pa.array([[1, 2], [3, 4]], pa.list_(pa.int64())),
},
schema=pa.schema(
[
pa.field("id", pa.int64(), nullable=False),
pa.field("v", pa.list_(pa.int64()), nullable=False),
]
),
)
source = ArrowTableSource(table, tag_columns=["id"], source_id=SRC)
before = _tokens(source, "v")
result = Index("v", 1, out="second")(source)
assert _tokens(result, "second") == [f"{t}[1]" for t in before]


def test_async_path_yields_the_same_tokens_as_the_sync_path() -> None:
batched = Batch(2)(_source())
op = Index("x", 0, out="first")
sync_tokens = _tokens(op(batched), "first")

async def run() -> list[Any]:
op.validate_inputs(batched)
source = Channel(buffer_size=16)
out = Channel(buffer_size=16)
for tag, data in batched.iter_data():
await source.writer.send((tag, data))
await source.writer.close()
await op.async_execute(
[source.reader], out.writer, input_pipeline_hashes=[batched.pipeline_hash()]
)
return [data.source_info()["first"] for _, data in await out.reader.collect()]

assert asyncio.run(run()) == sync_tokens


def test_zero_row_result_has_every_token_type_of_a_non_empty_one() -> None:
# Two aggregations: the projected token is one list level, the others keep theirs.
twice = Batch(2)(GroupBy(["k"])(_source()))
for out in ("first", None):
non_empty = Index("x", 0, out=out)(twice)
empty = Index("x", 9, out=out)(twice)
assert empty.as_table().num_rows == 0
empty_schema = empty.as_table(columns={"source": True}).schema
full_schema = non_empty.as_table(columns={"source": True}).schema
assert empty_schema == full_schema
Loading