From dcf5c69e6396381743119a0dd28016d36231f123 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 16:22:18 -0800 Subject: [PATCH 01/12] fix(operator_node): a zero-row async run keeps the sync path's columns and Arrow types (ITL-1080) The async operator node taps its input channels (first row, and the first row with a None source-info token per column) and, on a run with no output row, applies the operator to its inputs rebuilt with no rows, exactly as the sync orchestrator rebuilds them (StaticOutputOperatorPod._materialize_upstream, now shared). It no longer builds the result from the Python output schema. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/operator_node.py | 77 ++++++++++++++++--- .../core/operators/static_output_pod.py | 53 +++++++++++++ src/orcapod/pipeline/sync_orchestrator.py | 49 ++---------- ...026-11-13-itl-1080-sync-async-same-rows.md | 75 ++++++++++++++++++ .../test_operator_node_async_readback.py | 76 +++++++++++++++--- 5 files changed, 266 insertions(+), 64 deletions(-) create mode 100644 superpowers/plans/2026-11-13-itl-1080-sync-async-same-rows.md diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index c7c357f3d..f2c4d784f 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -23,7 +23,7 @@ from typing import TYPE_CHECKING, Any, Literal, cast from orcapod import contexts -from orcapod.channels import Channel, ReadableChannel, WritableChannel +from orcapod.channels import Channel, ChannelClosed, ReadableChannel, WritableChannel from orcapod.config import OrcapodConfig from orcapod.core.operators.static_output_pod import StaticOutputOperatorPod from orcapod.core.streams.arrow_table_stream import ArrowTableStream @@ -79,6 +79,55 @@ pc = LazyModule("pyarrow.compute") +class _InputTap: + """An input channel reader that keeps the rows needed to rebuild its columns. + + An async operator node hands its input channels to the operator, so it sees + no input row itself. When the run produces no output row, the node rebuilds + each input with no rows and applies the operator to those, as the sync path + applies it to the rebuilt upstream rows (ITL-1080). Those columns and types + are what ``StaticOutputOperatorPod._rows_to_table`` makes of the input's + rows, and the only thing in them that depends on more than one row is the + nullability of a source-info column (nullable when some row's token is + ``None``). So the tap keeps the first row, and the first row with a ``None`` + token for each source-info column, never the whole input. + + Attributes: + rows: The kept rows, in arrival order. + """ + + def __init__(self, reader: ReadableChannel[tuple[TagProtocol, DataProtocol]]): + self._reader = reader + self.rows: list[tuple[TagProtocol, DataProtocol]] = [] + self._null_sources: set[str] = set() + + def _keep(self, item: tuple[TagProtocol, DataProtocol]) -> None: + """Keep ``item`` if it is the first row or has a new ``None`` token.""" + nulls = {name for name, token in item[1].source_info().items() if token is None} + if not self.rows or not nulls <= self._null_sources: + self.rows.append(item) + self._null_sources |= nulls + + async def receive(self) -> tuple[TagProtocol, DataProtocol]: + """Receive the next row from the wrapped reader, keeping it if needed.""" + item = await self._reader.receive() + self._keep(item) + return item + + def __aiter__(self) -> "_InputTap": + return self + + async def __anext__(self) -> tuple[TagProtocol, DataProtocol]: + try: + return await self.receive() + except ChannelClosed: + raise StopAsyncIteration + + async def collect(self) -> list[tuple[TagProtocol, DataProtocol]]: + """Drain every remaining row.""" + return [item async for item in self] + + # --------------------------------------------------------------------------- # OperatorNodeBase — shared base (no DB) # --------------------------------------------------------------------------- @@ -1716,11 +1765,12 @@ async def forward() -> None: hashes = [s.pipeline_hash() for s in self._input_streams] operator = cast(StaticOutputOperatorPod, self._operator) + taps = [_InputTap(reader) for reader in inputs] try: async with asyncio.TaskGroup() as tg: tg.create_task( operator.async_execute( - inputs, + taps, intermediate.writer, input_pipeline_hashes=hashes, ) @@ -1742,14 +1792,23 @@ async def forward() -> None: if should_store: self._store_output_stream(stream) else: - # A completed run with no rows keeps a typed zero-row result, as the - # sync path does, so the node reads as computed and empty (not as - # never run). Nothing is stored for it. - self._cached_output_stream = ArrowTableStream( - self._make_empty_table(), - tag_columns=self.keys()[0], - data_context=self.data_context, + # A completed run with no rows keeps the operator's result over its + # inputs rebuilt with no rows, exactly as the sync orchestrator + # rebuilds them, so the node reads as computed and empty with the + # sync path's columns and Arrow types (ITL-1080). An input that + # delivered no row is rebuilt from its Python output schema, as on + # the sync path. Nothing is stored for it. + empty_result = operator.process( + *( + StaticOutputOperatorPod._materialize_upstream( + tap.rows, upstream, zero_rows=True + ) + for tap, upstream in zip(taps, self._input_streams, strict=True) + ) ) + # Computed, as execute() computes it, so the node reads as run. + _ = list(empty_result.iter_data()) + self._cached_output_stream = empty_result self._update_modified_time() diff --git a/src/orcapod/core/operators/static_output_pod.py b/src/orcapod/core/operators/static_output_pod.py index 081209815..d2bc05dd1 100644 --- a/src/orcapod/core/operators/static_output_pod.py +++ b/src/orcapod/core/operators/static_output_pod.py @@ -278,6 +278,59 @@ def _materialize_to_stream( # pass through, one value per row. return ArrowTableStream(table, tag_columns=tag_keys) + @staticmethod + def _materialize_upstream( + rows: Sequence[tuple[TagProtocol, DataProtocol]], + upstream: StreamProtocol, + *, + zero_rows: bool = False, + ) -> StreamProtocol: + """Rebuild the rows an upstream node delivered as that node's stream. + + This is how an operator node's input is built in a job: the sync + orchestrator rebuilds every upstream's buffered rows with it, and an + async operator node rebuilds its inputs with it when its run produced + no row (``zero_rows``). The stream takes the upstream's producer and + upstreams, so its identity and pipeline hash are the upstream's and + system-tag column names come out as they would on the upstream itself. + + With rows, the table is ``_rows_to_table(rows)``: every row keeps its + own system tags and source info. Without rows, the table is built from + the upstream's Python output schema (with system tags and source info), + which cannot recover Arrow types the rows would have had (ITL-912). + + Args: + rows: The rows the upstream delivered; may be empty. + upstream: The upstream node or stream the rows came from. + zero_rows: Keep the table's columns and types but none of its rows. + + Returns: + A stream over the rows (none when ``zero_rows``), in order. + """ + from orcapod.utils import arrow_utils + + if rows: + table, tag_keys = StaticOutputOperatorPod._rows_to_table(rows) + if zero_rows: + table = table.slice(0, 0) + else: + # Build an empty table preserving declared field nullability. + tag_schema, data_schema = upstream.output_schema( + columns={"system_tags": True, "source": True} + ) + table = arrow_utils.make_empty_table( + {**tag_schema, **data_schema}, upstream.data_context.type_converter + ) + tag_keys = upstream.keys()[0] + # No ``source_info`` argument: the table's own ``_source_*`` columns + # pass through, one value per row. + return ArrowTableStream( + table, + tag_columns=tag_keys, + producer=upstream.producer, + upstreams=upstream.upstreams, + ) + @staticmethod def _materialize_input_stream( rows: Sequence[tuple[TagProtocol, DataProtocol]], diff --git a/src/orcapod/pipeline/sync_orchestrator.py b/src/orcapod/pipeline/sync_orchestrator.py index 981706352..e6efa36a3 100644 --- a/src/orcapod/pipeline/sync_orchestrator.py +++ b/src/orcapod/pipeline/sync_orchestrator.py @@ -12,7 +12,6 @@ from orcapod.errors import StoreError, TypeIdentityError from orcapod.pipeline.result import OrchestratorResult -from orcapod.utils import arrow_utils from orcapod.utils.store_errors import keep_store_error from orcapod.protocols.node_protocols import ( is_function_node, @@ -187,58 +186,22 @@ def _gather_upstream_multi( @staticmethod def _materialize_as_stream(buf: list[tuple[Any, Any]], upstream_node: Any) -> Any: - """Wrap a (tag, data) buffer as an ArrowTableStream. + """Wrap a (tag, data) buffer as the upstream node's stream. - Builds the table with ``StaticOutputOperatorPod._rows_to_table``, as - ``StaticOutputOperatorPod._materialize_to_stream`` does, so every row - keeps its own system tags and source info. + ``StaticOutputOperatorPod._materialize_upstream`` builds it, the one + definition an async operator node shares when it rebuilds its inputs + for a run that produced no row. Args: buf: List of (tag, data) tuples. - upstream_node: The node that produced this buffer (used to - determine tag column names). + upstream_node: The node that produced this buffer. Returns: An ArrowTableStream. """ - from orcapod.core.streams.arrow_table_stream import ArrowTableStream - - if not buf: - # Build an empty stream preserving declared field nullability. - tag_schema, data_schema = upstream_node.output_schema( - columns={"system_tags": True, "source": True} - ) - type_converter = upstream_node.data_context.type_converter - empty_table = arrow_utils.make_empty_table( - {**tag_schema, **data_schema}, type_converter - ) - tag_keys = upstream_node.keys()[0] - return ArrowTableStream( - empty_table, - tag_columns=tag_keys, - producer=upstream_node.producer, - upstreams=upstream_node.upstreams, - ) - from orcapod.core.operators.static_output_pod import StaticOutputOperatorPod - # Each row keeps its own system tags and source info. - full_table, user_tag_keys = StaticOutputOperatorPod._rows_to_table(buf) - - # Pass the upstream node's producer and upstreams so the - # materialized stream inherits the correct identity_structure - # and pipeline_identity_structure (via StreamBase delegation). - # This ensures downstream operators produce correct system tag - # column names (which embed pipeline hashes of their inputs). - producer = upstream_node.producer - upstreams = upstream_node.upstreams - - return ArrowTableStream( - full_table, - tag_columns=user_tag_keys, - producer=producer, - upstreams=upstreams, - ) + return StaticOutputOperatorPod._materialize_upstream(buf, upstream_node) @staticmethod def _gc_buffers( diff --git a/superpowers/plans/2026-11-13-itl-1080-sync-async-same-rows.md b/superpowers/plans/2026-11-13-itl-1080-sync-async-same-rows.md new file mode 100644 index 000000000..c3b0beff8 --- /dev/null +++ b/superpowers/plans/2026-11-13-itl-1080-sync-async-same-rows.md @@ -0,0 +1,75 @@ +# ITL-1080 Sync and async give the same rows — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Sync and async runs give identical rows: the zero-row async operator node keeps the +sync path's columns and Arrow types, and (once the developer decides the meta rule) every +operator treats input `Data` meta columns alike on both paths. + +**Architecture:** The zero-row fix lives in `OperatorJobNode.async_execute`: it taps each input +channel (first row, plus the first row with a `None` source-info token per column), and on a +run with no output rows rebuilds zero-row inputs exactly as the sync orchestrator materializes +its buffers (one shared helper, `StaticOutputOperatorPod._materialize_upstream`), then keeps +`operator.process(*inputs)` as the result, as `execute()` does. No channel protocol or +`async_execute` signature change. The meta fix (rule B, pending developer decision) drops +input meta in every operator on both paths. + +**Tech Stack:** Python 3.12, PyArrow, pytest via `uv run`. + +--- + +## Status / notes for a successor + +- Diagnosis sent to coordinator 2026-11-13 (scratch notes in the worker transcript): + - All 8 meta xfails: Data/Tag content hashes identical on both paths; only meta columns differ. + - Main's Join DROPS meta (ITL-854/857), not keeps. Rule B (no operator carries input meta) + recommended; coordinator agrees, **developer decision pending: do not change operator meta + handling until answered.** + - Q2 (zero-row node) approved: implement now, test first, nothing stored for zero rows. +- Step-4 matrix (scratchpad `matrix.py`): source → fn pod → 17 operators → fn pod, sync vs async + job, in-memory store: no difference. MergeJoin over a dict column raises TypeError on BOTH + paths (report, out of scope). + +## Task 1: Zero-row async operator node (approved) + +**Files:** +- Modify: `src/orcapod/core/operators/static_output_pod.py` (add `_materialize_upstream`) +- Modify: `src/orcapod/pipeline/sync_orchestrator.py` (`_materialize_as_stream` delegates) +- Modify: `src/orcapod/core/nodes/operator_node.py` (`async_execute`: input taps, zero-row result) +- Test: `tests/test_pipeline/test_operator_node_async_readback.py` + +- [ ] Remove the strict xfail on + `test_operator_node_with_no_rows_keeps_columns_and_types_after_async_run`, parametrize it over + zero-row cases (filter no match, DropNulls dropping all, Join/SemiJoin/MergeJoin no match, + operator after an input that delivered no rows), comparing `as_table(all_info=True)` schema + and rows and `as_table()` of sync vs async runs. Run: fails. +- [ ] Add `_materialize_upstream(rows, upstream, *, zero_rows)` beside `_rows_to_table`: the body + of the sync orchestrator's `_materialize_as_stream`, optionally sliced to zero rows. Make the + sync orchestrator call it (no behaviour change). +- [ ] In `OperatorJobNode.async_execute`, wrap each input reader in `_InputTap` (keeps the first + row and, per source-info column, the first row whose token is `None`). With no output rows, + `self._cached_output_stream = operator.process(*[_materialize_upstream(tap.rows, upstream, + zero_rows=True) ...])`. +- [ ] Run targeted tests: readback, equivalence, sync orchestrator, golden. Commit. + +## Task 2: Meta rule B (BLOCKED on developer decision) + +- [ ] Prepare failing tests over both meta origins (raw `ArrowTableStream` with `__` columns; + `CachedFunctionPod` stream rows with `__computed`) for every operator, sync and async: + output `Data.meta_columns == ()` and identical schemas on both paths. +- [ ] After approval: drop input meta in each operator (sync: input tables via the Join helper + `_without_meta`, moved to the operator base; predictions drop `__` fields; async: per-row + operators drop `Data` meta). Remove the 8 META_XFAIL marks; update + `test_drop_nulls::test_no_argument_leaves_meta_columns_alone`, docs/concepts/operators.md. +- [ ] Golden suite unchanged. + +## Task 3: Equivalence matrix coverage + +- [ ] `CASES` in `test_async_sync_equivalence.py` already holds SemiJoin, MapTags, MapData, the + four selections, DropNulls, Index, Pick. Add the automatic select in front of a function pod + (sync vs async job, stored records) if not covered by `test_sync_async_stored_records.py`. + +## Task 4: Finish + +- [ ] `changelog.d/ITL-1080.md`, `design_issues/ITL-1080-SA1.md`. +- [ ] Merge origin/main, push, PR "Fixes ITL-1080", CodeRabbit, CI on 3.11/3.12/3.13. diff --git a/tests/test_pipeline/test_operator_node_async_readback.py b/tests/test_pipeline/test_operator_node_async_readback.py index d29f5fd26..bf5b2d073 100644 --- a/tests/test_pipeline/test_operator_node_async_readback.py +++ b/tests/test_pipeline/test_operator_node_async_readback.py @@ -488,23 +488,75 @@ def test_operator_node_with_no_rows_reads_back_empty_after_async_run( assert list(async_node.iter_data()) == [] -@pytest.mark.xfail( - strict=True, - reason=( - "ITL-858/ITL-912: no row reaches the async node, so it reads back the table " - "built from the Python output schema: int8 becomes int64, and the system-tag " - "and source-info columns are missing" +ZERO_ROW_CASES = [ + NO_MATCH, + Case( + "drop_nulls_drops_every_row", + lambda a, b, c: DropNulls(["s"])( + PolarsFilter(constraints={"id": 2})(a), label="op" + ), + 0, + ), + Case( + "join_no_match", + lambda a, b, c: Join()( + PolarsFilter(constraints={"id": 1})(a), + PolarsFilter(constraints={"id": 2})(b), + label="op", + ), + 0, ), + Case( + "semi_join_no_match", + lambda a, b, c: SemiJoin()( + PolarsFilter(constraints={"id": 1})(a), + PolarsFilter(constraints={"id": 2})(b), + label="op", + ), + 0, + ), + Case( + "merge_join_no_match", + lambda a, b, c: MergeJoin()( + PolarsFilter(constraints={"id": 1})(a), + PolarsFilter(constraints={"id": 2})(c), + label="op", + ), + 0, + ), + Case( + "filter_after_operator", + lambda a, b, c: PolarsFilter(constraints={"grp": "none"})( + SelectDataColumns(["x"])(a), label="op" + ), + 0, + ), + # An input that delivers no row at all: both paths build that input from the + # upstream's Python output schema (ITL-912), so they agree with each other. + *AFTER_NO_ROWS_CASES[:4], +] + + +@pytest.mark.parametrize( + "case", ZERO_ROW_CASES, ids=[case.name for case in ZERO_ROW_CASES] ) def test_operator_node_with_no_rows_keeps_columns_and_types_after_async_run( - tmp_path: Path, + case: Case, tmp_path: Path ) -> None: - """A zero-row async result should keep the sync path's columns and Arrow types.""" - sync_node, async_node = _sync_and_async_nodes(NO_MATCH, tmp_path) + """A zero-row async result keeps the sync path's columns and Arrow types (ITL-1080). - assert async_node.as_table(all_info=True).schema == ( - sync_node.as_table(all_info=True).schema - ) + The node has no output row to read a type from; it rebuilds its inputs with no rows, + as the sync orchestrator builds them, and keeps the operator's result over those. + """ + sync_node, async_node = _sync_and_async_nodes(case, tmp_path) + + sync_table = sync_node.as_table(all_info=True) + async_table = async_node.as_table(all_info=True) + assert sync_table.num_rows == async_table.num_rows == 0 + assert async_table.schema == sync_table.schema + assert async_node.as_table().schema == sync_node.as_table().schema + assert async_node.keys(all_info=True) == sync_node.keys(all_info=True) + assert list(async_node.iter_data()) == [] AFTER_OPERATOR_CASES = [ From 8e88660aafdf65651bf7ceb15f3827ffbc954ba1 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 16:23:36 -0800 Subject: [PATCH 02/12] test(wip): rule B tests: no operator carries input Data meta (ITL-1080) Co-Authored-By: Claude Opus 5.5 --- .../operators/test_async_sync_equivalence.py | 138 ++++++++++++++++++ 1 file changed, 138 insertions(+) diff --git a/tests/test_core/operators/test_async_sync_equivalence.py b/tests/test_core/operators/test_async_sync_equivalence.py index da3790e47..cf883a9f0 100644 --- a/tests/test_core/operators/test_async_sync_equivalence.py +++ b/tests/test_core/operators/test_async_sync_equivalence.py @@ -1328,3 +1328,141 @@ def test_async_null_logical_key_matches_nothing_as_the_sync_path_does() -> None: for operands in ((left, right), (right, left)): expected = sync_rows(operator(), *operands) assert_equivalent(expected, async_rows(operator(), *operands)) + + +# ── no operator carries its inputs' Data meta columns (ITL-1080, rule B) ──── + + +def _assert_no_input_meta(rows: list[Row]) -> None: + """No output row carries a Data meta column.""" + for _, data in rows: + assert data.meta_columns == () + assert not [ + name + for name in data.as_table(all_info=True).column_names + if "__m" in name or "__computed" in name + ] + + +@pytest.mark.parametrize( + "case", [pytest.param(_meta_case(c), id=c.label) for c in CASES] +) +def test_no_operator_carries_raw_stream_meta(case: Case) -> None: + """A raw ``ArrowTableStream``'s ``__`` columns reach no operator output, on either path. + + Join's rule (ITL-854/857), generalized to every operator: meta columns describe how + an input row was produced; an operator's output row was produced by the operator. + The prediction leaves the ``__`` names out as well. + """ + sources = case.make_sources() + expected = sync_rows(case.make_op(), *sources) + actual = async_rows(case.make_op(), *sources) + _assert_no_input_meta(expected) + _assert_no_input_meta(actual) + _, data_schema = case.make_op().output_schema(*sources) + assert not [name for name in data_schema if name.startswith("__")] + + +def cached_meta_source( + source_id: str, offset: int = 0, mergeable: bool = False +) -> StreamProtocol: + """A cached function pod's output whose every row's ``Data`` carries ``__computed``. + + Tags ``k: int16`` and ``t: int8``; data ``y: int``, ``yl: list[int]``, + ``yd: dict[str, int]`` and nullable ``yn: int | None``; only ``y`` and ``yl`` when + ``mergeable``, the columns ``MergeJoin`` can merge (it cannot sort dicts or nulls). + """ + source = _source( + { + "k": pa.array([1, 2, 3], pa.int16()), + "t": pa.array([5, 5, 6], pa.int8()), + "a": pa.array([1, 2, 3], pa.int8()), + }, + tags=("k", "t"), + source_id=source_id, + ) + + def outputs(a: int) -> tuple[int, list[int], dict[str, int], int | None]: + return a + offset, [a, a + 1], {"q": a}, None if a == 2 else a + + def merged_outputs(a: int) -> tuple[int, list[int]]: + return a + offset, [a, a + 1] + + function_pod = FunctionPod( + PythonDataFunction( # pyright: ignore[reportArgumentType] # PythonDataFunction is not yet typed as matching DataFunctionProtocol (baselined in other tests); it works at runtime + merged_outputs if mergeable else outputs, + output_keys=("y", "yl") if mergeable else ("y", "yl", "yd", "yn"), + function_name=f"outputs_{source_id}", + ) + ) + pod = CachedFunctionPod(function_pod, result_database=InMemoryArrowDatabase()) + stream = pod(source) + assert all(data.meta_columns for _, data in stream.iter_data()) + return stream + + +CACHED_META_CASES = [ + Case("join", Join, lambda: (cached_meta_source("p"), source_b(nested=False))), + Case( + "merge_join", + MergeJoin, + lambda: ( + cached_meta_source("p", mergeable=True), + cached_meta_source("q", 10, mergeable=True), + ), + ), + Case("semijoin", SemiJoin, lambda: (cached_meta_source("p"), source_b())), + Case("batch_2", lambda: Batch(batch_size=2), lambda: (cached_meta_source("p"),)), + Case("batch_all", Batch, lambda: (cached_meta_source("p"),)), + Case("group_by", lambda: GroupBy(["t"]), lambda: (cached_meta_source("p"),)), + Case( + "polars_filter", + lambda: PolarsFilter(constraints={"t": 5}), + lambda: (cached_meta_source("p"),), + ), + Case("map_tags", lambda: MapTags({"t": "tb"}), lambda: (cached_meta_source("p"),)), + Case("map_data", lambda: MapData({"y": "w"}), lambda: (cached_meta_source("p"),)), + Case( + "select_tag_columns", + lambda: SelectTagColumns(["k"]), + lambda: (cached_meta_source("p"),), + ), + Case( + "select_data_columns", + lambda: SelectDataColumns(["y"]), + lambda: (cached_meta_source("p"),), + ), + Case( + "drop_tag_columns", + lambda: DropTagColumns(["t"]), + lambda: (cached_meta_source("p"),), + ), + Case( + "drop_data_columns", + lambda: DropDataColumns(["yl"]), + lambda: (cached_meta_source("p"),), + ), + Case("index", lambda: Index("yl", 0), lambda: (cached_meta_source("p"),)), + Case("pick", lambda: Pick("yd", "q"), lambda: (cached_meta_source("p"),)), + Case("drop_nulls", lambda: DropNulls("yn"), lambda: (cached_meta_source("p"),)), + Case("drop_nulls_all", DropNulls, lambda: (cached_meta_source("p"),)), +] + + +@pytest.mark.parametrize( + "case", CACHED_META_CASES, ids=[c.label for c in CACHED_META_CASES] +) +def test_no_operator_carries_cached_pod_meta(case: Case) -> None: + """A cached pod's ``__computed`` reaches no operator output, on either path. + + The stream hides the column in ``as_table()`` but every row's ``Data`` carries it, + so a per-row async path used to keep it while the sync path lost it. + """ + sources = case.make_sources() + expected = sync_rows(case.make_op(), *sources) + actual = async_rows(case.make_op(), *sources) + assert expected + _assert_no_input_meta(expected) + _assert_no_input_meta(actual) + assert_equivalent(expected, actual) + assert_same_meta(expected, actual) From 296ba5121420666d15b3597a8626ca33dcc17519 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 16:36:02 -0800 Subject: [PATCH 03/12] wip(operators): rule B, no operator carries its inputs' Data meta columns (ITL-1080) Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/operators/base.py | 22 ++- src/orcapod/core/operators/batch.py | 2 + .../core/operators/column_selection.py | 5 + src/orcapod/core/operators/drop_nulls.py | 2 + src/orcapod/core/operators/index.py | 2 + src/orcapod/core/operators/join.py | 77 +++------- src/orcapod/core/operators/mappers.py | 3 + src/orcapod/core/operators/meta_columns.py | 136 ++++++++++++++++++ src/orcapod/core/operators/pick.py | 2 + src/orcapod/core/operators/semijoin.py | 2 + .../operators/test_async_sync_equivalence.py | 36 +---- tests/test_core/operators/test_drop_nulls.py | 8 +- 12 files changed, 199 insertions(+), 98 deletions(-) create mode 100644 src/orcapod/core/operators/meta_columns.py diff --git a/src/orcapod/core/operators/base.py b/src/orcapod/core/operators/base.py index 7fa5a2b08..f31276679 100644 --- a/src/orcapod/core/operators/base.py +++ b/src/orcapod/core/operators/base.py @@ -5,6 +5,10 @@ from collections.abc import Collection, Sequence from typing import TYPE_CHECKING, Any +from orcapod.core.operators.meta_columns import ( + without_input_meta, + without_meta_fields, +) from orcapod.channels import ReadableChannel, WritableChannel from orcapod.core.operators.static_output_pod import StaticOutputOperatorPod from orcapod.errors import InputValidationError @@ -136,7 +140,8 @@ def validate_inputs(self, *streams: StreamProtocol) -> None: def static_process(self, *streams: StreamProtocol) -> StreamProtocol: """Forward to ``unary_static_process`` with the single input stream.""" stream = streams[0] - return self.unary_static_process(stream) + # No operator carries its input's Data meta columns (ITL-1080). + return without_input_meta(self.unary_static_process(stream), streams) def output_schema( self, @@ -145,7 +150,11 @@ def output_schema( all_info: bool = False, ) -> tuple[Schema, Schema]: stream = streams[0] - return self.unary_output_schema(stream, columns=columns, all_info=all_info) + tag_schema, data_schema = self.unary_output_schema( + stream, columns=columns, all_info=all_info + ) + # The prediction of ``without_input_meta`` in ``static_process``. + return tag_schema, without_meta_fields(data_schema) def argument_symmetry(self, streams: Collection[StreamProtocol]) -> ArgumentGroup: # return single stream as a tuple @@ -231,7 +240,10 @@ def is_commutative(self) -> bool: def static_process(self, *streams: StreamProtocol) -> StreamProtocol: """Forward to ``binary_static_process`` with two input streams.""" left_stream, right_stream = streams - return self.binary_static_process(left_stream, right_stream) + # No operator carries its inputs' Data meta columns (ITL-1080). + return without_input_meta( + self.binary_static_process(left_stream, right_stream), streams + ) def output_schema( self, @@ -240,9 +252,11 @@ def output_schema( all_info: bool = False, ) -> tuple[Schema, Schema]: left_stream, right_stream = streams - return self.binary_output_schema( + tag_schema, data_schema = self.binary_output_schema( left_stream, right_stream, columns=columns, all_info=all_info ) + # The prediction of ``without_input_meta`` in ``static_process``. + return tag_schema, without_meta_fields(data_schema) def validate_inputs(self, *streams: StreamProtocol) -> None: if len(streams) != 2: diff --git a/src/orcapod/core/operators/batch.py b/src/orcapod/core/operators/batch.py index 2fc0fe32b..80932f7ea 100644 --- a/src/orcapod/core/operators/batch.py +++ b/src/orcapod/core/operators/batch.py @@ -3,6 +3,7 @@ from collections.abc import Sequence from typing import TYPE_CHECKING, Any +from orcapod.core.operators.meta_columns import WithoutMetaWriter from orcapod.channels import ReadableChannel, WritableChannel from orcapod.core.operators.base import UnaryOperator from orcapod.core.streams import ArrowTableStream @@ -173,6 +174,7 @@ async def async_execute( ValueError: If ``input_pipeline_hashes`` does not have one hash per input. """ try: + output = WithoutMetaWriter(output) if len(input_pipeline_hashes) != len(inputs): raise ValueError( f"input_pipeline_hashes length ({len(input_pipeline_hashes)}) " diff --git a/src/orcapod/core/operators/column_selection.py b/src/orcapod/core/operators/column_selection.py index f3ed67c56..97a4c7933 100644 --- a/src/orcapod/core/operators/column_selection.py +++ b/src/orcapod/core/operators/column_selection.py @@ -2,6 +2,7 @@ from collections.abc import Collection, Mapping, Sequence from typing import TYPE_CHECKING, Any +from orcapod.core.operators.meta_columns import WithoutMetaWriter from orcapod.channels import ReadableChannel, WritableChannel from orcapod.core.operators.base import UnaryOperator from orcapod.core.streams import ArrowTableStream @@ -127,6 +128,7 @@ async def async_execute( pipeline hash. """ try: + output = WithoutMetaWriter(output) tags_to_drop: list[str] | None = None async for tag, data in inputs[0]: if tags_to_drop is None: @@ -255,6 +257,7 @@ async def async_execute( pipeline hash. """ try: + output = WithoutMetaWriter(output) pkts_to_drop: list[str] | None = None async for tag, data in inputs[0]: if pkts_to_drop is None: @@ -378,6 +381,7 @@ async def async_execute( pipeline hash. """ try: + output = WithoutMetaWriter(output) effective_drops: list[str] | None = None async for tag, data in inputs[0]: if effective_drops is None: @@ -508,6 +512,7 @@ async def async_execute( pipeline hash. """ try: + output = WithoutMetaWriter(output) effective_drops: list[str] | None = None async for tag, data in inputs[0]: if effective_drops is None: diff --git a/src/orcapod/core/operators/drop_nulls.py b/src/orcapod/core/operators/drop_nulls.py index 437fbfcff..e5f5ce584 100644 --- a/src/orcapod/core/operators/drop_nulls.py +++ b/src/orcapod/core/operators/drop_nulls.py @@ -7,6 +7,7 @@ from collections.abc import Collection, Sequence from typing import TYPE_CHECKING, Any +from orcapod.core.operators.meta_columns import WithoutMetaWriter from orcapod.channels import ReadableChannel, WritableChannel from orcapod.core.datagrams import Data, Tag from orcapod.core.operators.base import UnaryOperator @@ -331,6 +332,7 @@ async def async_execute( pipeline hash. """ try: + output = WithoutMetaWriter(output) async for tag, data in inputs[0]: names = self._target_columns(tag.keys(), data.keys()) tag_table = self._narrow_table( diff --git a/src/orcapod/core/operators/index.py b/src/orcapod/core/operators/index.py index c2c00e8f6..ed8e5a477 100644 --- a/src/orcapod/core/operators/index.py +++ b/src/orcapod/core/operators/index.py @@ -7,6 +7,7 @@ from collections.abc import Sequence from typing import TYPE_CHECKING, Any, cast +from orcapod.core.operators.meta_columns import WithoutMetaWriter from orcapod.channels import ReadableChannel, WritableChannel from orcapod.core.operators.base import UnaryOperator from orcapod.system_constants import constants @@ -430,6 +431,7 @@ async def async_execute( pipeline hash. """ try: + output = WithoutMetaWriter(output) async for tag, data in inputs[0]: result = self._process_one(tag, data) if result is not None: diff --git a/src/orcapod/core/operators/join.py b/src/orcapod/core/operators/join.py index c53f07b9a..520f5f40e 100644 --- a/src/orcapod/core/operators/join.py +++ b/src/orcapod/core/operators/join.py @@ -3,6 +3,12 @@ from collections.abc import Callable, Collection, Mapping, Sequence from typing import TYPE_CHECKING, Any +from orcapod.core.operators.meta_columns import ( + WithoutMetaWriter, + without_input_meta, + without_meta_columns, + without_meta_fields, +) from orcapod.channels import ReadableChannel, WritableChannel from orcapod.core.operators.match_keys import frozen_scalar from orcapod.core.operators.base import ( @@ -67,7 +73,10 @@ def output_schema( if len(streams) == 1: # if single stream, simply return the output schema of the single input stream - return streams[0].output_schema(columns=columns, all_info=all_info) + tag_schema, data_schema = streams[0].output_schema( + columns=columns, all_info=all_info + ) + return tag_schema, without_meta_fields(data_schema) tag_schema, data_schema = self._joined_user_schemas(*streams) @@ -91,7 +100,7 @@ def _joined_user_schemas(self, *streams: StreamProtocol) -> tuple[Schema, Schema System tags are excluded: the join renames them, and ``_predict_system_tag_schema`` computes them separately. Meta (``__``-prefixed) columns are excluded too: Join drops them - (``_without_meta``), including the ones a raw ``ArrowTableStream`` + (``meta_columns.without_meta_columns``), including the ones a raw ``ArrowTableStream`` lists as data columns in its ``output_schema``. Args: @@ -107,10 +116,10 @@ def _joined_user_schemas(self, *streams: StreamProtocol) -> tuple[Schema, Schema ordered_streams = self.order_input_streams(*streams) stream = ordered_streams[0] tag_schema, data_schema = stream.output_schema() - data_schema = self._without_meta_fields(data_schema) + data_schema = without_meta_fields(data_schema) for other_stream in ordered_streams[1:]: other_tag_schema, other_data_schema = other_stream.output_schema() - other_data_schema = self._without_meta_fields(other_data_schema) + other_data_schema = without_meta_fields(other_data_schema) tag_schema = schema_utils.union_schemas(tag_schema, other_tag_schema) intersection_data_schema = schema_utils.intersection_schemas( data_schema, other_data_schema @@ -173,14 +182,14 @@ def static_process(self, *streams: StreamProtocol) -> StreamProtocol: The result holds every input's tag and data columns, source info and system tags (renamed per input). It holds none of the inputs' meta - (``__``-prefixed) columns, which Join drops (``_without_meta``). + (``__``-prefixed) columns, which Join drops (``meta_columns.without_meta_columns``). Raises: InputValidationError: If two inputs share a data column, or a shared tag key has different Arrow types in two inputs. """ if len(streams) == 1: - return streams[0] + return without_input_meta(streams[0], streams) # Canonically order streams by pipeline_hash for deterministic # system tag column names regardless of input order (Join is commutative) @@ -196,7 +205,7 @@ def static_process(self, *streams: StreamProtocol) -> StreamProtocol: stream = ordered_streams[0] tag_keys, _ = [set(k) for k in stream.keys()] - table = self._without_meta( + table = without_meta_columns( stream.as_table(columns={"source": True, "system_tags": True}) ) # trick to get cartesian product; use int8 explicitly so that zero-row @@ -211,7 +220,7 @@ def static_process(self, *streams: StreamProtocol) -> StreamProtocol: for idx, next_stream in enumerate(ordered_streams[1:], start=1): next_tag_keys, _ = next_stream.keys() - next_table = self._without_meta( + next_table = without_meta_columns( next_stream.as_table(columns={"source": True, "system_tags": True}) ) next_table = arrow_utils.append_to_system_tags( @@ -278,55 +287,6 @@ def static_process(self, *streams: StreamProtocol) -> StreamProtocol: tag_columns=tuple(tag_order), ) - @staticmethod - def _without_meta(table: "pa.Table") -> "pa.Table": - """Drop the meta (``__``-prefixed) columns from an input table. - - Join keeps none of its inputs' meta columns. They describe how one - input row was produced (a cached pod's ``__computed``, the result - cache's ``__pf_*`` and ``__pod_ts``), nothing downstream of a Join - reads them, and inputs routinely share their names. A raw - ``ArrowTableStream`` returns its ``__`` columns from ``as_table`` - whatever the column configuration, together with a source-info - column for each (``_source___x``), so both are dropped by name. - - Args: - table: An input's table. - - Returns: - The table without its meta columns and their source-info columns. - """ - meta_source_prefix = f"{constants.SOURCE_PREFIX}{constants.META_PREFIX}" - meta = [ - name - for name in table.column_names - if name.startswith((constants.META_PREFIX, meta_source_prefix)) - ] - return table.drop_columns(meta) if meta else table - - @staticmethod - def _without_meta_fields(schema: Schema) -> Schema: - """Drop the meta (``__``-prefixed) fields from a data schema. - - The prediction counterpart of ``_without_meta``: a raw - ``ArrowTableStream`` lists its ``__`` columns as data columns. - - Args: - schema: An input's data schema. - - Returns: - The schema without its meta fields. - """ - if not any(name.startswith(constants.META_PREFIX) for name in schema): - return schema - return Schema( - { - name: data_type - for name, data_type in schema.items() - if not name.startswith(constants.META_PREFIX) - } - ) - # ------------------------------------------------------------------ # Async execution # ------------------------------------------------------------------ @@ -399,6 +359,7 @@ async def async_execute( ValueError: If ``input_pipeline_hashes`` does not have one hash per input. """ try: + output = WithoutMetaWriter(output) if len(input_pipeline_hashes) != len(inputs): raise ValueError( f"input_pipeline_hashes length ({len(input_pipeline_hashes)}) " @@ -833,7 +794,7 @@ def _merge_data(left: DataProtocol, right: DataProtocol) -> DataProtocol: Every column keeps the Arrow type it already has, and the source info of both sides is carried over. Neither side's meta columns are, as in - ``static_process`` (``_without_meta``). + ``static_process`` (``meta_columns.without_meta_columns``). Args: left: The canonically earlier data (in a staggered join, the diff --git a/src/orcapod/core/operators/mappers.py b/src/orcapod/core/operators/mappers.py index 48db5c234..4060dc8ce 100644 --- a/src/orcapod/core/operators/mappers.py +++ b/src/orcapod/core/operators/mappers.py @@ -1,6 +1,7 @@ from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Any +from orcapod.core.operators.meta_columns import WithoutMetaWriter from orcapod.channels import ReadableChannel, WritableChannel from orcapod.core.operators.base import UnaryOperator from orcapod.core.streams import ArrowTableStream @@ -140,6 +141,7 @@ async def async_execute( pipeline hash. """ try: + output = WithoutMetaWriter(output) rename_map: dict[str, str] | None = None unmapped: list[str] | None = None async for tag, data in inputs[0]: @@ -289,6 +291,7 @@ async def async_execute( pipeline hash. """ try: + output = WithoutMetaWriter(output) rename_map: dict[str, str] | None = None unmapped: list[str] | None = None async for tag, data in inputs[0]: diff --git a/src/orcapod/core/operators/meta_columns.py b/src/orcapod/core/operators/meta_columns.py new file mode 100644 index 000000000..652eaf96d --- /dev/null +++ b/src/orcapod/core/operators/meta_columns.py @@ -0,0 +1,136 @@ +"""No operator carries its inputs' Data meta columns, on either path (ITL-1080). + +Meta columns (``__``-prefixed: a cached pod's ``__computed``, the result cache's +``__pf_*`` and ``__pod_ts``) describe how one input row was produced. An operator's +output row is produced by the operator, so it carries none of them: ``Join`` set the +rule (ITL-854, ITL-857) and every operator follows it, on the sync path (a table, its +prediction) and on the async path (each row's ``Data``) alike. + +A raw ``ArrowTableStream`` lists the ``__`` columns of its table as data columns, with +a source-info column for each (``_source___x``); other streams hide them from +``as_table()`` while each row's ``Data`` carries them. Both forms are removed here. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import TYPE_CHECKING + +from orcapod.channels import WritableChannel +from orcapod.core.streams.arrow_table_stream import ArrowTableStream +from orcapod.protocols.core_protocols import DataProtocol, StreamProtocol, TagProtocol +from orcapod.system_constants import constants +from orcapod.types import Schema +from orcapod.utils.lazy_module import LazyModule + +if TYPE_CHECKING: + import pyarrow as pa +else: + pa = LazyModule("pyarrow") + +_META_SOURCE_PREFIX = f"{constants.SOURCE_PREFIX}{constants.META_PREFIX}" + + +def without_meta_columns(table: "pa.Table") -> "pa.Table": + """Drop the meta columns, and a raw stream's source-info columns for them. + + Args: + table: A stream's table. + + Returns: + The table without its ``__`` and ``_source___`` columns. + """ + meta = [ + name + for name in table.column_names + if name.startswith((constants.META_PREFIX, _META_SOURCE_PREFIX)) + ] + return table.drop_columns(meta) if meta else table + + +def without_meta_fields(schema: Schema) -> Schema: + """Drop the meta fields from a data schema (a raw stream lists them as data). + + Args: + schema: A data schema. + + Returns: + The schema without its ``__`` fields. + """ + if not any(name.startswith(constants.META_PREFIX) for name in schema): + return schema + return Schema( + { + name: data_type + for name, data_type in schema.items() + if not name.startswith(constants.META_PREFIX) + } + ) + + +def without_data_meta(data: DataProtocol) -> DataProtocol: + """Return ``data`` without its meta columns (``data`` itself when it has none).""" + if not data.meta_columns: + return data + return data.drop_meta_columns(*data.meta_columns) + + +def _carries_meta(stream: StreamProtocol, inputs: Sequence[StreamProtocol]) -> bool: + """Whether ``stream``, an operator's result, holds an input's meta columns. + + A table built by the operator holds them only as ``__`` data columns of a raw + stream. A result that is one of the inputs, returned unaltered, holds whatever + its rows' ``Data`` hold. + """ + if any(name.startswith(constants.META_PREFIX) for name in stream.keys()[1]): + return True + if not any(stream is input_stream for input_stream in inputs): + return False + first = next(iter(stream.iter_data()), None) + return first is not None and bool(first[1].meta_columns) + + +def without_input_meta( + stream: StreamProtocol, inputs: Sequence[StreamProtocol] +) -> StreamProtocol: + """Return an operator's sync result without its inputs' meta columns. + + Args: + stream: What the operator's ``static_process`` built. + inputs: The streams it was given. + + Returns: + ``stream`` itself when it holds no meta column; otherwise the same rows + rebuilt without them, with every other column (tags, system tags, data, + source info, data context) as it was. + """ + if not _carries_meta(stream, inputs): + return stream + table = stream.as_table( + columns={"source": True, "system_tags": True, "context": True} + ) + return ArrowTableStream( + without_meta_columns(table), + tag_columns=stream.keys()[0], + data_context=stream.data_context, + ) + + +class WithoutMetaWriter: + """An output channel writer that sends every row without its Data meta columns. + + The async counterpart of ``without_input_meta``: an operator whose + ``async_execute`` passes rows on wraps its output in it. + """ + + def __init__(self, output: WritableChannel[tuple[TagProtocol, DataProtocol]]): + self._output = output + + async def send(self, item: tuple[TagProtocol, DataProtocol]) -> None: + """Send ``item`` without its Data meta columns.""" + tag, data = item + await self._output.send((tag, without_data_meta(data))) + + async def close(self) -> None: + """Close the wrapped writer.""" + await self._output.close() diff --git a/src/orcapod/core/operators/pick.py b/src/orcapod/core/operators/pick.py index 84b698eb6..0da738473 100644 --- a/src/orcapod/core/operators/pick.py +++ b/src/orcapod/core/operators/pick.py @@ -7,6 +7,7 @@ from collections.abc import Sequence from typing import TYPE_CHECKING, Any, cast +from orcapod.core.operators.meta_columns import WithoutMetaWriter from orcapod.channels import ReadableChannel, WritableChannel from orcapod.core.operators.base import UnaryOperator from orcapod.core.streams.arrow_table_stream import ArrowTableStream @@ -415,6 +416,7 @@ async def async_execute( pipeline hash. """ try: + output = WithoutMetaWriter(output) async for tag, data in inputs[0]: result = self._process_one(tag, data) if result is not None: diff --git a/src/orcapod/core/operators/semijoin.py b/src/orcapod/core/operators/semijoin.py index fe1929cec..d0a1a3656 100644 --- a/src/orcapod/core/operators/semijoin.py +++ b/src/orcapod/core/operators/semijoin.py @@ -1,6 +1,7 @@ from collections.abc import Sequence from typing import TYPE_CHECKING, Any +from orcapod.core.operators.meta_columns import WithoutMetaWriter from orcapod.channels import ReadableChannel, WritableChannel from orcapod.core.operators.base import BinaryOperator, require_shared_arrow_type from orcapod.core.operators.match_keys import frozen_scalar @@ -194,6 +195,7 @@ async def async_execute( input's pipeline hash. """ try: + output = WithoutMetaWriter(output) left_ch, right_ch = inputs[0], inputs[1] # Phase 1: Build right-side lookup diff --git a/tests/test_core/operators/test_async_sync_equivalence.py b/tests/test_core/operators/test_async_sync_equivalence.py index cf883a9f0..135fce0a8 100644 --- a/tests/test_core/operators/test_async_sync_equivalence.py +++ b/tests/test_core/operators/test_async_sync_equivalence.py @@ -763,42 +763,14 @@ def make_sources() -> tuple[StreamProtocol, ...]: return Case(case.label, case.make_op, make_sources) -# Operators whose paths disagree on meta columns in ways that are not a local -# Join-style fix. Strict, so a fix flags them. -_MATERIALIZE_DROPS_META = ( - "ITL-846: barrier-mode async_execute rebuilds its input with " - "_materialize_to_stream, which drops Data meta columns" -) -_SYNC_DROPS_META = ( - "ITL-846: the sync path drops Data meta columns that the async path keeps (a raw " - "ArrowTableStream lists its __ columns as data columns, so the sync selection " - "or _materialize_to_stream discards them)" -) -META_XFAIL: dict[str, str] = { - "merge_join": _MATERIALIZE_DROPS_META, - "batch_2": _MATERIALIZE_DROPS_META, - "batch_all": _MATERIALIZE_DROPS_META, - "group_by": _MATERIALIZE_DROPS_META, - "polars_filter": _MATERIALIZE_DROPS_META, - "select_data_columns": _SYNC_DROPS_META, - "index": _SYNC_DROPS_META, - "pick": _SYNC_DROPS_META, -} -META_CASES = [ - pytest.param( - _meta_case(c), - id=c.label, - marks=[pytest.mark.xfail(strict=True, reason=META_XFAIL[c.label])] - if c.label in META_XFAIL - else [], - ) - for c in CASES -] +# Every operator, with each input carrying its own meta columns. No operator keeps +# them, on either path (ITL-1080; see ``test_no_operator_carries_raw_stream_meta``). +META_CASES = [pytest.param(_meta_case(c), id=c.label) for c in CASES] @pytest.mark.parametrize("case", META_CASES) def test_async_meta_equals_sync(case: Case) -> None: - """The async path's Data meta columns equal the sync path's (Join drops them on both).""" + """The async path's Data meta columns equal the sync path's: none, on both (ITL-1080).""" sources = case.make_sources() expected = sync_rows(case.make_op(), *sources) actual = async_rows(case.make_op(), *sources) diff --git a/tests/test_core/operators/test_drop_nulls.py b/tests/test_core/operators/test_drop_nulls.py index 654f4be06..c713368da 100644 --- a/tests/test_core/operators/test_drop_nulls.py +++ b/tests/test_core/operators/test_drop_nulls.py @@ -805,7 +805,8 @@ def test_no_argument_leaves_meta_columns_alone() -> None: """A meta (``__``) column is not a data column for "every column". A stream built straight from a table lists its meta columns among its data - keys (ITL-846); a null there still drops no row and the field stays nullable. + keys (ITL-846); a null there still drops no row. Like every operator, DropNulls + does not carry its input's meta columns (ITL-1080). """ meta = f"{constants.META_PREFIX}note" table = readings().as_table(columns=ALL) @@ -819,9 +820,8 @@ def test_no_argument_leaves_meta_columns_alone() -> None: assert [tag["id"] for tag, _ in rows] == [1] data = rows[0][1] assert data.as_dict() == {"x": 10, "y": 1.5, "note": "a"} - meta_field = data.arrow_schema(columns={"meta": True}).field(meta) - assert meta_field == pa.field(meta, pa.large_string(), nullable=True) - assert data.as_table(columns={"meta": True}).column(meta).to_pylist() == [None] + assert data.meta_columns == () + assert meta not in result.as_table(all_info=True).column_names # Non-public on purpose: a prediction is verified against the static result. computed = DropNulls().static_process(stream).output_schema() assert DropNulls().output_schema(stream) == computed From 65cbbc89d0c9df5162fc6bc27d612dcf49843774 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 16:51:23 -0800 Subject: [PATCH 04/12] fix(operators)!: no operator carries its inputs' Data meta columns, on either path (ITL-1080) Developer decision, November 13, 2026: an operator's output rows and its predicted output schema hold none of its inputs' meta (__) columns, on the sync and the async path, whether they came from a raw ArrowTableStream or from a cached pod's or function node's rows. One rule in core/operators/meta_columns.py: the operator base classes (and Join) pass their sync result and prediction through it, and every async_execute sends its rows through WithoutMetaWriter. A carry-forward mode is ITL-867. No row content hash, record id, pipeline hash or golden value moves. The eight strict xfails of test_async_meta_equals_sync are removed; new tests cover both meta origins for every operator. Docs, CLAUDE.md, .zed/rules, changelog fragment and design issue entry SA1. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 5 +++ CLAUDE.md | 6 ++++ changelog.d/ITL-1080.md | 21 ++++++++++++ design_issues/ITL-1080-SA1.md | 33 +++++++++++++++++++ design_issues/ITL-837-JA1.md | 2 +- docs/concepts/operators.md | 14 +++++--- src/orcapod/core/operators/meta_columns.py | 2 +- .../core/operators/static_output_pod.py | 6 +++- ...026-11-13-itl-1080-sync-async-same-rows.md | 14 ++++++-- .../test_operator_node_async_readback.py | 27 +++++++++++++++ 10 files changed, 120 insertions(+), 10 deletions(-) create mode 100644 changelog.d/ITL-1080.md create mode 100644 design_issues/ITL-1080-SA1.md diff --git a/.zed/rules b/.zed/rules index f14340ed6..2498f99f1 100644 --- a/.zed/rules +++ b/.zed/rules @@ -563,6 +563,11 @@ predicate on values, and DropNulls reads only whether a value is present (its va never the value. A filter decides which rows exist and never synthesizes a value; no other operator reads data content. +No operator carries its inputs' Data meta (`__`) columns (developer, November 13, 2026, +ITL-1080): on the sync and the async path, in rows and in the predicted output schema, for a +raw ArrowTableStream's `__` columns and for the meta a cached pod's or function node's rows +carry. One rule, core/operators/meta_columns.py. A carry-forward mode is ITL-867. + Operator or derived source? (developer, November 11, 2026) The test is: does any data element come from a tag value, or from anywhere other than the input's data elements? If yes, the step is not an operator: it is a derived source. So a tag's value is offered as data only diff --git a/CLAUDE.md b/CLAUDE.md index 95deee60a..833973f0b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -582,6 +582,12 @@ predicate on values, and `DropNulls` reads only whether a value is present (its never the value. A filter decides which rows exist and never synthesizes a value; no other operator reads data content. +No operator carries its inputs' `Data` meta (`__`) columns (developer, November 13, 2026, +ITL-1080): on the sync and the async path, in rows and in the predicted output schema, for a +raw `ArrowTableStream`'s `__` columns and for the meta a cached pod's or function node's rows +carry. The rule is in one place, `core/operators/meta_columns.py` (sync results through the +operator base classes, async rows through `WithoutMetaWriter`). A carry-forward mode is ITL-867. + **Operator or derived source?** (developer, November 11, 2026) The test is: does any data element come from a tag value, or from anywhere other than the input's data elements? If yes, the step is not an operator: it is a derived source. So a tag's value is offered as data only diff --git a/changelog.d/ITL-1080.md b/changelog.d/ITL-1080.md new file mode 100644 index 000000000..86a96145c --- /dev/null +++ b/changelog.d/ITL-1080.md @@ -0,0 +1,21 @@ +### Changed + +- No operator carries its inputs' `Data` meta (`__`-prefixed) columns any more, on the sync and + the async path alike, as `Join` already did (ITL-1080). An operator's output rows and its + predicted output schema hold none of them, whether they came from a raw `ArrowTableStream` + whose table has `__` columns or from the rows of a cached function pod or function node + (`__computed`, `__pf_*`, `__pod_ts`). Before, `MergeJoin`, `Batch`, `GroupBy`, `PolarsFilter`, + `SemiJoin`, `DropNulls`, `MapTags`, `MapData` and the column selections kept them on one path + or both. No row content hash, record id or stored table changes: meta columns were never + hashed, and job runs already stored operator rows without them. A mode that carries meta + forward is ITL-867. + +### Fixed + +- An operator node whose async run produces no row keeps the sync run's columns and Arrow types + (`int8` stays `int8`; system-tag and source-info columns are present), instead of a table + built from the Python output schema (ITL-1080). When an input itself delivered no row, that + input is rebuilt from its upstream's Python output schema on both paths alike (ITL-912), so a + `Join` of such an input with a non-empty one whose key is, say, `int8` now raises + `InputValidationError` on an async run as it already did on a sync run, instead of finishing + with a mistyped empty result. diff --git a/design_issues/ITL-1080-SA1.md b/design_issues/ITL-1080-SA1.md new file mode 100644 index 000000000..9b7358b53 --- /dev/null +++ b/design_issues/ITL-1080-SA1.md @@ -0,0 +1,33 @@ +## `src/orcapod/core/operators/` — Async execution + +### SA1 — Sync and async runs gave different rows: meta columns, and a zero-row async operator node +**Status:** resolved +**Severity:** high +**Issue:** ITL-1080 + +The same pipeline run with the sync and the async orchestrator must give the same rows +(developer, November 13, 2026). Nine strict expected failures pinned differences: + +* Eight operators disagreed on the inputs' `Data` meta (`__`) columns. Over a raw + `ArrowTableStream` that lists `__` columns as data, the sync path of `MergeJoin`, `Batch`, + `GroupBy` and `PolarsFilter` kept them while their barrier async path (`_rows_to_table`) + dropped them; `SelectDataColumns`, `Index` and `Pick` did the opposite. Over a cached function + pod's stream (rows carry `__computed`, `as_table()` hides it), `MapTags`, `MapData` and + `DropNulls` dropped it on sync and kept it on async. `Join` already dropped meta on both paths + (ITL-854, ITL-857). Row content hashes never differed: `Data.content_hash()` does not cover + meta, and both orchestrators already stored operator rows without meta. +* An async operator node whose run produced no row read back a table built from the Python + output schema (`int8` became `int64`; system-tag and source-info columns missing); the sync + run kept the operator's own result. + +**Fix:** the rule (developer, November 13, 2026): no operator carries its inputs' meta +columns, on either path, in rows and in the prediction. It lives in +`core/operators/meta_columns.py`: the unary and binary operator base classes (and `Join`) pass +their sync result through `without_input_meta` and their prediction through +`without_meta_fields`; every operator with its own `async_execute` sends its rows through +`WithoutMetaWriter`. A mode that carries meta forward across operators is ITL-867. +`OperatorJobNode.async_execute` taps its input channels (the first row, and the first row with +a `None` source-info token per column) and, when no row comes out, applies the operator to its +inputs rebuilt with no rows, exactly as the sync orchestrator rebuilds them +(`StaticOutputOperatorPod._materialize_upstream`, shared by both). No row content hash, record +id, pipeline hash or golden value moves. diff --git a/design_issues/ITL-837-JA1.md b/design_issues/ITL-837-JA1.md index e8cef7487..135a4cf6f 100644 --- a/design_issues/ITL-837-JA1.md +++ b/design_issues/ITL-837-JA1.md @@ -56,4 +56,4 @@ drop them. Nothing downstream of a `Join` reads or stores them, and meta is not two inputs that share only a meta name join. How meta columns should be handled across operators and pods is a separate design question (ITL-867). `MergeJoin` still carries input meta on its sync path (colliding names are merged into lists like data) and drops it on its -barrier async path (ITL-846). +barrier async path (ITL-846). Since ITL-1080 no operator carries input meta, on either path (SA1). diff --git a/docs/concepts/operators.md b/docs/concepts/operators.md index 60d6656e0..fbca438ab 100644 --- a/docs/concepts/operators.md +++ b/docs/concepts/operators.md @@ -37,6 +37,12 @@ The filters are the operators that read data columns: `PolarsFilter` evaluates i on values, and `DropNulls` reads only whether a value is present (its validity), never the value. Both decide which rows exist; neither synthesizes a value. +No operator carries its inputs' meta columns: the `__`-prefixed columns, such as a cached +function pod's `__computed`, that describe how an input row was produced. An operator's +output row is produced by the operator, so its `Data` holds none of them and its predicted +output schema lists none, on the sync and the async path alike. A way to carry them forward +across operators is an open design question (ITL-867). + ## Operator categories ### `UnaryOperator` -- single input @@ -58,9 +64,8 @@ Takes one or more streams. Used for `Join`, which performs an N-ary inner join. N-ary inner join on shared tag columns. Requires that input streams have non-overlapping data columns (raises `InputValidationError` on collision). Join is **commutative** -- the -order of input streams does not affect the result. Join drops its inputs' meta -(`__`-prefixed) columns, such as a cached function pod's `__computed`, so they never collide -and never appear in its output. Rows match on equality of every shared tag column, whatever +order of input streams does not affect the result. Like every operator, Join drops its +inputs' meta (`__`-prefixed) columns, so they never collide. Rows match on equality of every shared tag column, whatever its type (lists and structs compare element by element). A row with a null in any shared tag column matches nothing, a null list or struct included, and rows with null on both sides do not match each other (the SQL convention for nulls; `MergeJoin` and `SemiJoin` follow it too, on the sync and the async path, while `GroupBy` is the exception by design and puts null keys in one group); a null *inside* a list or struct is an ordinary element that equals @@ -491,7 +496,8 @@ print(by_method.pipeline_hash() == by_operator.pipeline_hash()) `DropNulls()` and `stream.drop_nulls()`, with no names, drop every row that has a null in **any** tag or data column, and make every tag and data column non-nullable. System-tag, -source-info, context and meta columns are never tested or changed. +source-info, context and meta columns are never tested (meta columns are not carried +at all, as by every operator). ```python visits = ArrowTableSource( diff --git a/src/orcapod/core/operators/meta_columns.py b/src/orcapod/core/operators/meta_columns.py index 652eaf96d..0e6a9002c 100644 --- a/src/orcapod/core/operators/meta_columns.py +++ b/src/orcapod/core/operators/meta_columns.py @@ -112,7 +112,7 @@ def without_input_meta( return ArrowTableStream( without_meta_columns(table), tag_columns=stream.keys()[0], - data_context=stream.data_context, + data_context=stream.data_context_key, ) diff --git a/src/orcapod/core/operators/static_output_pod.py b/src/orcapod/core/operators/static_output_pod.py index d2bc05dd1..bf8f862e1 100644 --- a/src/orcapod/core/operators/static_output_pod.py +++ b/src/orcapod/core/operators/static_output_pod.py @@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any, cast from orcapod.channels import ReadableChannel, WritableChannel +from orcapod import contexts from orcapod.config import OrcapodConfig from orcapod.contexts import DataContext from orcapod.core.base import TraceableBase @@ -318,8 +319,11 @@ def _materialize_upstream( tag_schema, data_schema = upstream.output_schema( columns={"system_tags": True, "source": True} ) + converter = contexts.resolve_context( + upstream.data_context_key + ).type_converter table = arrow_utils.make_empty_table( - {**tag_schema, **data_schema}, upstream.data_context.type_converter + {**tag_schema, **data_schema}, converter ) tag_keys = upstream.keys()[0] # No ``source_info`` argument: the table's own ``_source_*`` columns diff --git a/superpowers/plans/2026-11-13-itl-1080-sync-async-same-rows.md b/superpowers/plans/2026-11-13-itl-1080-sync-async-same-rows.md index c3b0beff8..db9e8c4af 100644 --- a/superpowers/plans/2026-11-13-itl-1080-sync-async-same-rows.md +++ b/superpowers/plans/2026-11-13-itl-1080-sync-async-same-rows.md @@ -23,8 +23,8 @@ input meta in every operator on both paths. - Diagnosis sent to coordinator 2026-11-13 (scratch notes in the worker transcript): - All 8 meta xfails: Data/Tag content hashes identical on both paths; only meta columns differ. - Main's Join DROPS meta (ITL-854/857), not keeps. Rule B (no operator carries input meta) - recommended; coordinator agrees, **developer decision pending: do not change operator meta - handling until answered.** + recommended; **developer decided B (Nov 13): no operator carries input meta; carry-forward + is ITL-867.** Implemented in `core/operators/meta_columns.py`. - Q2 (zero-row node) approved: implement now, test first, nothing stored for zero rows. - Step-4 matrix (scratchpad `matrix.py`): source → fn pod → 17 operators → fn pod, sync vs async job, in-memory store: no difference. MergeJoin over a dict column raises TypeError on BOTH @@ -52,7 +52,7 @@ input meta in every operator on both paths. zero_rows=True) ...])`. - [ ] Run targeted tests: readback, equivalence, sync orchestrator, golden. Commit. -## Task 2: Meta rule B (BLOCKED on developer decision) +## Task 2: Meta rule B (approved, done) - [ ] Prepare failing tests over both meta origins (raw `ArrowTableStream` with `__` columns; `CachedFunctionPod` stream rows with `__computed`) for every operator, sync and async: @@ -73,3 +73,11 @@ input meta in every operator on both paths. - [ ] `changelog.d/ITL-1080.md`, `design_issues/ITL-1080-SA1.md`. - [ ] Merge origin/main, push, PR "Fixes ITL-1080", CodeRabbit, CI on 3.11/3.12/3.13. + +## Progress (2026-11-13) + +- Task 1 done (dcf5c69e). Task 2 done. Targeted suites (core, pipeline, hashing, docs): 9356 + passed, golden unchanged. Step-4 matrix variants plain/log/cached/cached+log/none+log: no + sync/async difference; with an upstream that delivered no row, Join now raises on async as on + sync (ITL-912), pinned by a test. +- Next: PR, CodeRabbit, CI. diff --git a/tests/test_pipeline/test_operator_node_async_readback.py b/tests/test_pipeline/test_operator_node_async_readback.py index bf5b2d073..d4ee9f85f 100644 --- a/tests/test_pipeline/test_operator_node_async_readback.py +++ b/tests/test_pipeline/test_operator_node_async_readback.py @@ -39,6 +39,7 @@ from orcapod.core.nodes import OperatorJobNode from orcapod.core.sources import ArrowTableSource from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.errors import InputValidationError from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator from orcapod.pipeline.job import PipelineJob from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator @@ -694,3 +695,29 @@ def test_operator_node_reads_back_only_the_latest_run( assert sorted(repr(tag.as_dict()["id"]) for tag, _ in node.iter_data()) == [ repr(i) for i in expected_ids ] + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_join_with_an_input_that_delivered_no_row_fails_alike_on_both_paths( + orchestrator: str, tmp_path: Path +) -> None: + """An input with no row is rebuilt from its Python output schema on both paths. + + So an ``int8`` key comes back ``int64`` (ITL-912) and ``Join`` with a non-empty + ``int8`` input refuses the disagreement, on the async path as on the sync path + (ITL-1080); before, the async node finished with a mistyped zero-row result. + When ITL-912 keeps the Arrow types of an empty input, both paths complete. + """ + case = Case( + "join_after_no_rows", + lambda a, b, c: Join()(_no_match(a), b, label="op"), + 0, + ) + with pytest.raises((InputValidationError, ExceptionGroup)) as info: + _ = _run_job(case, orchestrator, "in_memory", tmp_path) + error = info.value + while isinstance(error, ExceptionGroup): + assert len(error.exceptions) == 1 + error = error.exceptions[0] + assert isinstance(error, InputValidationError) + assert "'id'" in str(error) From cb8ee9eaf84c900142f3071668cdb48e909b3f1d Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 17:05:44 -0800 Subject: [PATCH 05/12] fix(column_selection): a selection that leaves no data column raises alike on both paths (ITL-1080) DropDataColumns of every data column and SelectDataColumns(strict=False) matching none raise InputValidationError when applied (sync) and on the first row (async), with one message. Before, sync failed late with a ValueError and async emitted rows with empty Data. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1080.md | 5 +++ design_issues/ITL-1080-SA1.md | 5 +++ .../core/operators/column_selection.py | 27 ++++++++++++ .../operators/test_async_sync_equivalence.py | 41 +++++++++++++++++++ 4 files changed, 78 insertions(+) diff --git a/changelog.d/ITL-1080.md b/changelog.d/ITL-1080.md index 86a96145c..cabe50017 100644 --- a/changelog.d/ITL-1080.md +++ b/changelog.d/ITL-1080.md @@ -19,3 +19,8 @@ `Join` of such an input with a non-empty one whose key is, say, `int8` now raises `InputValidationError` on an async run as it already did on a sync run, instead of finishing with a mistyped empty result. +- `DropDataColumns` and `SelectDataColumns` raise `InputValidationError` when their result would + have no data column (every data column dropped, or `strict=False` matching none), when the + operator is applied on the sync path and on the first row on the async path, with one message + (ITL-1080). Before, the sync path failed only when the rows were read, with a `ValueError`, and + the async path emitted rows with empty data. diff --git a/design_issues/ITL-1080-SA1.md b/design_issues/ITL-1080-SA1.md index 9b7358b53..be60988cc 100644 --- a/design_issues/ITL-1080-SA1.md +++ b/design_issues/ITL-1080-SA1.md @@ -31,3 +31,8 @@ a `None` source-info token per column) and, when no row comes out, applies the o inputs rebuilt with no rows, exactly as the sync orchestrator rebuilds them (`StaticOutputOperatorPod._materialize_upstream`, shared by both). No row content hash, record id, pipeline hash or golden value moves. + +A further divergence (found by the ITL-1084 worker): `DropDataColumns` of every data column, or +`SelectDataColumns(strict=False)` matching none, failed on the sync path only when rows were read +and emitted rows with empty `Data` on the async path. Both now raise one `InputValidationError` +(`column_selection._require_a_data_column`): at `op(stream)` on sync, on the first row on async. diff --git a/src/orcapod/core/operators/column_selection.py b/src/orcapod/core/operators/column_selection.py index 97a4c7933..4e2cd5323 100644 --- a/src/orcapod/core/operators/column_selection.py +++ b/src/orcapod/core/operators/column_selection.py @@ -35,6 +35,27 @@ def _canonical_columns(columns: str | Collection[str]) -> tuple[str, ...]: return tuple(sorted(set(columns))) +def _require_a_data_column(remaining: Collection[str]) -> None: + """Refuse a selection whose result would have no data column. + + A stream needs at least one data column (``ArrowTableStream`` refuses a table + without one), so the sync path checks this when the operator is applied and + the async path on the first row, with this one error. Meta (``__``) columns + are not data: no operator carries them (ITL-1080). + + Args: + remaining: The data columns the result would have. + + Raises: + InputValidationError: If ``remaining`` has no data column. + """ + if not [c for c in remaining if not c.startswith(constants.META_PREFIX)]: + raise InputValidationError( + "The result would have no data column; a stream needs at least one. " + "Keep at least one data column." + ) + + class SelectTagColumns(UnaryOperator): """ Operator that selects specified columns from a stream. @@ -219,6 +240,7 @@ def validate_unary_input(self, stream: StreamProtocol) -> None: raise InputValidationError( f"Missing data columns: {missing_columns}. Make sure all specified columns to select are present or use strict=False to ignore missing columns" ) + _require_a_data_column([c for c in data_columns if c in columns_to_select]) def unary_output_schema( self, @@ -270,6 +292,7 @@ async def async_execute( f"specified columns to select are present or use " f"strict=False to ignore missing columns" ) + _require_a_data_column([c for c in pkt_keys if c in self.columns]) pkts_to_drop = [c for c in pkt_keys if c not in self.columns] if not pkts_to_drop: await output.send((tag, data)) @@ -477,6 +500,7 @@ def validate_unary_input(self, stream: StreamProtocol) -> None: raise InputValidationError( f"Missing data columns: {missing_columns}. Make sure all specified columns to drop are present or use strict=False to ignore missing columns" ) + _require_a_data_column([c for c in data_columns if c not in self.columns]) def unary_output_schema( self, @@ -525,6 +549,9 @@ async def async_execute( f"specified columns to drop are present or use " f"strict=False to ignore missing columns" ) + _require_a_data_column( + [c for c in pkt_keys if c not in self.columns] + ) effective_drops = ( list(self.columns) if self.strict diff --git a/tests/test_core/operators/test_async_sync_equivalence.py b/tests/test_core/operators/test_async_sync_equivalence.py index 135fce0a8..0f6dd9fdf 100644 --- a/tests/test_core/operators/test_async_sync_equivalence.py +++ b/tests/test_core/operators/test_async_sync_equivalence.py @@ -1438,3 +1438,44 @@ def test_no_operator_carries_cached_pod_meta(case: Case) -> None: _assert_no_input_meta(actual) assert_equivalent(expected, actual) assert_same_meta(expected, actual) + + +# ── a selection that would leave no data column (ITL-1080) ────────────────── + +NO_DATA_COLUMN_OPS = { + "drop_every_data_column": lambda: DropDataColumns(["i32", "u8", "f32", "ls"]), + "drop_every_data_column_not_strict": lambda: DropDataColumns( + ["i32", "u8", "f32", "ls", "absent"], strict=False + ), + "select_no_data_column_not_strict": lambda: SelectDataColumns( + ["absent"], strict=False + ), +} + + +@pytest.mark.parametrize("make_op", NO_DATA_COLUMN_OPS.values(), ids=NO_DATA_COLUMN_OPS) +def test_selection_leaving_no_data_column_raises_alike_on_both_paths( + make_op: Callable[[], StaticOutputOperatorPod], +) -> None: + """A stream needs a data column: both paths refuse a result without one. + + The sync path raises when the operator is applied, ``op(stream)``; the async + path on the first row, with the same error. Before, the sync path raised only + when the rows were read and the async path emitted rows with empty ``Data``. + """ + source = source_b(nested=False) + with pytest.raises(InputValidationError, match="no data column") as sync_error: + _ = make_op()(source) + with pytest.raises(InputValidationError, match="no data column") as async_error: + _ = async_rows(make_op(), source) + assert str(async_error.value) == str(sync_error.value) + + +def test_selection_leaving_only_meta_columns_raises() -> None: + """A raw stream's ``__`` columns are not data: they are not carried (ITL-1080).""" + source = _with_meta(source_b(nested=False), _meta_columns("m", 3)) + op = DropDataColumns(["i32", "u8", "f32", "ls"]) + with pytest.raises(InputValidationError, match="no data column"): + _ = op(source) + with pytest.raises(InputValidationError, match="no data column"): + _ = async_rows(DropDataColumns(["i32", "u8", "f32", "ls"]), source) From 489170b63955b9ff6b7c977f3de3b1fd89d9a89c Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 17:28:18 -0800 Subject: [PATCH 06/12] test(hashing): drop-spellings identity test keeps a data column (ITL-1080) DropDataColumns of every data column is now refused when applied; the test compares identities, so its source gets a third data column. Co-Authored-By: Claude Opus 5.5 --- tests/test_hashing/test_wrong_hash_findings.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/test_hashing/test_wrong_hash_findings.py b/tests/test_hashing/test_wrong_hash_findings.py index 093562627..e1d0afdd3 100644 --- a/tests/test_hashing/test_wrong_hash_findings.py +++ b/tests/test_hashing/test_wrong_hash_findings.py @@ -534,8 +534,16 @@ class TestColumnSelectionIdentity: @pytest.mark.parametrize("op_cls", [SelectDataColumns, DropDataColumns]) def test_data_column_spellings_share_identity(self, op_cls: Any) -> None: - """Every spelling of one set of data columns gives one identity.""" - source = _xy_source() + """Every spelling of one set of data columns gives one identity. + + The source has a third data column, ``z``, so dropping ``x`` and ``y`` leaves + one: a selection that leaves no data column is refused (ITL-1080). + """ + source = DictSource( + data=[{"k": 1, "j": 2, "x": 1, "y": 2, "z": 3}], + tag_columns=["k", "j"], + source_id="s", + ) spellings = [["x", "y"], ("x", "y"), {"x", "y"}, ["y", "x"], ["x", "y", "x"]] streams = [op_cls(columns=c)(source) for c in spellings] assert len({s.pipeline_hash() for s in streams}) == 1 From 6815013dbdfc7f83f24d3f91714d95a1aa96bfd2 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 18:15:26 -0800 Subject: [PATCH 07/12] fix(streams): a tag named with the meta prefix is refused where it would be made (ITL-1080) The Tag datagram treats a __ name as meta, so a raw stream's __ tag was lost on every path that rebuilds rows (async, Index/Pick, both orchestrators) while table-based sync paths kept it, and Join failed on it. ArrowTableStream, SourceStreamBuilder.build and MapTags now refuse such a name (schema_utils.require_tag_names_not_meta); the meta helpers strip only non-tag __ columns, which also repairs Join. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 2 + CLAUDE.md | 2 + changelog.d/ITL-1080.md | 6 +++ design_issues/ITL-1080-SA1.md | 6 +++ docs/concepts/operators.md | 3 +- src/orcapod/core/operators/join.py | 5 +- src/orcapod/core/operators/mappers.py | 4 ++ src/orcapod/core/operators/meta_columns.py | 18 ++++--- src/orcapod/core/sources/stream_builder.py | 6 ++- .../core/streams/arrow_table_stream.py | 4 +- src/orcapod/utils/schema_utils.py | 26 ++++++++++ .../operators/test_async_sync_equivalence.py | 52 +++++++++++++++++++ 12 files changed, 123 insertions(+), 11 deletions(-) diff --git a/.zed/rules b/.zed/rules index 2498f99f1..d761c80be 100644 --- a/.zed/rules +++ b/.zed/rules @@ -567,6 +567,8 @@ No operator carries its inputs' Data meta (`__`) columns (developer, November 13 ITL-1080): on the sync and the async path, in rows and in the predicted output schema, for a raw ArrowTableStream's `__` columns and for the meta a cached pod's or function node's rows carry. One rule, core/operators/meta_columns.py. A carry-forward mode is ITL-867. +A tag column never has a name starting with the meta prefix (`__`): ArrowTableStream, the +source builder and MapTags refuse one (schema_utils.require_tag_names_not_meta). Operator or derived source? (developer, November 11, 2026) The test is: does any data element come from a tag value, or from anywhere other than the input's data elements? If yes, diff --git a/CLAUDE.md b/CLAUDE.md index 833973f0b..cd9d0671a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -587,6 +587,8 @@ ITL-1080): on the sync and the async path, in rows and in the predicted output s raw `ArrowTableStream`'s `__` columns and for the meta a cached pod's or function node's rows carry. The rule is in one place, `core/operators/meta_columns.py` (sync results through the operator base classes, async rows through `WithoutMetaWriter`). A carry-forward mode is ITL-867. +A tag column never has a name starting with the meta prefix (`__`): `ArrowTableStream`, the +source builder and `MapTags` refuse one (`schema_utils.require_tag_names_not_meta`). **Operator or derived source?** (developer, November 11, 2026) The test is: does any data element come from a tag value, or from anywhere other than the input's data elements? If yes, diff --git a/changelog.d/ITL-1080.md b/changelog.d/ITL-1080.md index cabe50017..565e2defe 100644 --- a/changelog.d/ITL-1080.md +++ b/changelog.d/ITL-1080.md @@ -9,6 +9,12 @@ or both. No row content hash, record id or stored table changes: meta columns were never hashed, and job runs already stored operator rows without them. A mode that carries meta forward is ITL-867. +- A tag column whose name starts with `__`, the prefix reserved for meta columns, is refused with + `InputValidationError` where it would be made: building an `ArrowTableStream`, building a + source with such a name in `tag_columns` (before, a "tag_columns not found" error, since the + source drops `__` columns), and `MapTags` renaming a tag to such a name (ITL-1080). Before, a + raw stream could carry one, and every path that rebuilt rows silently lost it, so sync and + async runs differed. ### Fixed diff --git a/design_issues/ITL-1080-SA1.md b/design_issues/ITL-1080-SA1.md index be60988cc..da90605c9 100644 --- a/design_issues/ITL-1080-SA1.md +++ b/design_issues/ITL-1080-SA1.md @@ -36,3 +36,9 @@ A further divergence (found by the ITL-1084 worker): `DropDataColumns` of every `SelectDataColumns(strict=False)` matching none, failed on the sync path only when rows were read and emitted rows with empty `Data` on the async path. Both now raise one `InputValidationError` (`column_selection._require_a_data_column`): at `op(stream)` on sync, on the first row on async. + +A tag named with the meta prefix (`__g`) was a meta column to the `Tag` datagram, so every path +that rebuilds rows (every `async_execute`, `Index`/`Pick` on sync, both orchestrators) lost it +while table-based sync paths kept it, and `Join` failed on it outright. Such a tag is now +refused where it would be made (`schema_utils.require_tag_names_not_meta`: `ArrowTableStream`, +`SourceStreamBuilder.build`, `MapTags`), and the meta helpers strip only non-tag `__` columns. diff --git a/docs/concepts/operators.md b/docs/concepts/operators.md index fbca438ab..4732d1d4e 100644 --- a/docs/concepts/operators.md +++ b/docs/concepts/operators.md @@ -41,7 +41,8 @@ No operator carries its inputs' meta columns: the `__`-prefixed columns, such as function pod's `__computed`, that describe how an input row was produced. An operator's output row is produced by the operator, so its `Data` holds none of them and its predicted output schema lists none, on the sync and the async path alike. A way to carry them forward -across operators is an open design question (ITL-867). +across operators is an open design question (ITL-867). A tag column can never have a meta column's +name: a stream, a source and `MapTags` refuse a tag whose name starts with `__`. ## Operator categories diff --git a/src/orcapod/core/operators/join.py b/src/orcapod/core/operators/join.py index 520f5f40e..399ce4487 100644 --- a/src/orcapod/core/operators/join.py +++ b/src/orcapod/core/operators/join.py @@ -206,7 +206,7 @@ def static_process(self, *streams: StreamProtocol) -> StreamProtocol: tag_keys, _ = [set(k) for k in stream.keys()] table = without_meta_columns( - stream.as_table(columns={"source": True, "system_tags": True}) + stream.as_table(columns={"source": True, "system_tags": True}), tag_keys ) # trick to get cartesian product; use int8 explicitly so that zero-row # tables don't produce a null-typed column that Polars rejects on join @@ -221,7 +221,8 @@ def static_process(self, *streams: StreamProtocol) -> StreamProtocol: for idx, next_stream in enumerate(ordered_streams[1:], start=1): next_tag_keys, _ = next_stream.keys() next_table = without_meta_columns( - next_stream.as_table(columns={"source": True, "system_tags": True}) + next_stream.as_table(columns={"source": True, "system_tags": True}), + next_tag_keys, ) next_table = arrow_utils.append_to_system_tags( next_table, diff --git a/src/orcapod/core/operators/mappers.py b/src/orcapod/core/operators/mappers.py index 4060dc8ce..580083942 100644 --- a/src/orcapod/core/operators/mappers.py +++ b/src/orcapod/core/operators/mappers.py @@ -9,6 +9,7 @@ from orcapod.protocols.core_protocols import DataProtocol, StreamProtocol, TagProtocol from orcapod.system_constants import constants from orcapod.types import ColumnConfig, ContentHash, Schema +from orcapod.utils import schema_utils from orcapod.utils.lazy_module import LazyModule if TYPE_CHECKING: @@ -180,6 +181,9 @@ class MapTags(UnaryOperator): def __init__( self, name_map: Mapping[str, str], drop_unmapped: bool = False, **kwargs ): + # A tag never gets a meta column's name (ITL-1080): refused here, so no + # path (sync, async or the prediction) can produce one. + schema_utils.require_tag_names_not_meta(name_map.values()) self.name_map = dict(name_map) self.drop_unmapped = drop_unmapped super().__init__(**kwargs) diff --git a/src/orcapod/core/operators/meta_columns.py b/src/orcapod/core/operators/meta_columns.py index 0e6a9002c..a91a1f997 100644 --- a/src/orcapod/core/operators/meta_columns.py +++ b/src/orcapod/core/operators/meta_columns.py @@ -13,7 +13,7 @@ from __future__ import annotations -from collections.abc import Sequence +from collections.abc import Collection, Sequence from typing import TYPE_CHECKING from orcapod.channels import WritableChannel @@ -31,19 +31,24 @@ _META_SOURCE_PREFIX = f"{constants.SOURCE_PREFIX}{constants.META_PREFIX}" -def without_meta_columns(table: "pa.Table") -> "pa.Table": - """Drop the meta columns, and a raw stream's source-info columns for them. +def without_meta_columns(table: "pa.Table", tag_columns: Collection[str]) -> "pa.Table": + """Drop the data meta columns, and a raw stream's source-info columns for them. + + A tag column is never a meta column, whatever its name: a raw stream may name + a tag with the ``__`` prefix, and it stays. Args: table: A stream's table. + tag_columns: The stream's tag columns. Returns: - The table without its ``__`` and ``_source___`` columns. + The table without its ``__`` data columns and ``_source___`` columns. """ meta = [ name for name in table.column_names if name.startswith((constants.META_PREFIX, _META_SOURCE_PREFIX)) + and name not in tag_columns ] return table.drop_columns(meta) if meta else table @@ -109,9 +114,10 @@ def without_input_meta( table = stream.as_table( columns={"source": True, "system_tags": True, "context": True} ) + tag_columns = stream.keys()[0] return ArrowTableStream( - without_meta_columns(table), - tag_columns=stream.keys()[0], + without_meta_columns(table, tag_columns), + tag_columns=tag_columns, data_context=stream.data_context_key, ) diff --git a/src/orcapod/core/sources/stream_builder.py b/src/orcapod/core/sources/stream_builder.py index 8018e73fa..165ff6cab 100644 --- a/src/orcapod/core/sources/stream_builder.py +++ b/src/orcapod/core/sources/stream_builder.py @@ -17,7 +17,7 @@ from orcapod.core.streams.arrow_table_stream import ArrowTableStream from orcapod.system_constants import constants from orcapod.types import ContentHash -from orcapod.utils import arrow_utils +from orcapod.utils import arrow_utils, schema_utils from orcapod.utils.schema_utils import compute_source_schema_hash from orcapod.utils.lazy_module import LazyModule @@ -186,6 +186,10 @@ def build( tag_columns_tuple = tuple(tag_columns) system_tag_columns_tuple = tuple(system_tag_columns) + # A tag never has a meta column's name; the meta columns are dropped next, + # so say so here rather than report the tag as missing (ITL-1080). + schema_utils.require_tag_names_not_meta(tag_columns_tuple) + # 1. Drop system columns from raw input. table = arrow_utils.drop_system_columns(table) diff --git a/src/orcapod/core/streams/arrow_table_stream.py b/src/orcapod/core/streams/arrow_table_stream.py index e881a88c2..ea2b09f25 100644 --- a/src/orcapod/core/streams/arrow_table_stream.py +++ b/src/orcapod/core/streams/arrow_table_stream.py @@ -16,7 +16,7 @@ from orcapod.protocols.hashing_protocols import PipelineElementProtocol from orcapod.system_constants import constants from orcapod.types import ColumnConfig, Schema, SourceInfoValue -from orcapod.utils import arrow_utils +from orcapod.utils import arrow_utils, schema_utils from orcapod.utils.lazy_module import LazyModule if TYPE_CHECKING: @@ -80,6 +80,8 @@ def __init__( prefix_info = {constants.SOURCE_PREFIX: source_info} + # A tag never has a meta column's name (ITL-1080). + schema_utils.require_tag_names_not_meta(tag_columns) # determine tag columns first and then exclude any source info self._tag_columns = tuple(c for c in tag_columns if c in table.column_names) self._system_tag_columns = tuple( diff --git a/src/orcapod/utils/schema_utils.py b/src/orcapod/utils/schema_utils.py index 6b69aa11c..b3c6c0035 100644 --- a/src/orcapod/utils/schema_utils.py +++ b/src/orcapod/utils/schema_utils.py @@ -577,3 +577,29 @@ def require_output_keys_not_tags( + "name cannot be both a tag and a data column. Rename the output " + "(the function's output key), or rename the tag with map_tags." ) + + +def require_tag_names_not_meta(tag_names: Collection[str]) -> None: + """Raise if a tag column's name starts with the reserved meta prefix. + + Names starting with the meta prefix (``__``) are meta columns: a datagram + keeps them apart from its tags and data, and no operator carries them + (ITL-1080). A tag with such a name would silently stop being a tag on every + path that rebuilds rows, so no stream, source or operator may produce one. + + Args: + tag_names: The tag column names. + + Raises: + InputValidationError: If a name starts with the meta prefix. The message + names the column. + """ + from orcapod.system_constants import constants + + for name in sorted(tag_names): + if name.startswith(constants.META_PREFIX): + raise InputValidationError( + f"tag column {name!r} starts with {constants.META_PREFIX!r}, which is " + "reserved for meta columns; a tag needs another name. Rename the " + "column before building the stream or source." + ) diff --git a/tests/test_core/operators/test_async_sync_equivalence.py b/tests/test_core/operators/test_async_sync_equivalence.py index 0f6dd9fdf..0a0faac85 100644 --- a/tests/test_core/operators/test_async_sync_equivalence.py +++ b/tests/test_core/operators/test_async_sync_equivalence.py @@ -1479,3 +1479,55 @@ def test_selection_leaving_only_meta_columns_raises() -> None: _ = op(source) with pytest.raises(InputValidationError, match="no data column"): _ = async_rows(DropDataColumns(["i32", "u8", "f32", "ls"]), source) + + +# ── a tag named with the meta prefix (ITL-1080) ────────────────────────────── +# +# A ``__`` name is a meta column: a datagram keeps it apart from its tags, so a tag with +# such a name would stop being a tag on every path that rebuilds rows. No route may +# produce one. + + +def _table_with_meta_named_column() -> pa.Table: + return pa.table( + { + "k": pa.array([1, 2], pa.int16()), + "__g": pa.array([1, 1], pa.int8()), + "x": pa.array([3, 4], pa.int8()), + } + ) + + +def test_a_stream_refuses_a_tag_named_with_the_meta_prefix() -> None: + with pytest.raises(InputValidationError, match="'__g'.*reserved for meta"): + _ = ArrowTableStream(_table_with_meta_named_column(), tag_columns=["k", "__g"]) + + +def test_a_source_refuses_a_tag_named_with_the_meta_prefix() -> None: + """Not "tag column not found": the source would drop the meta column first.""" + with pytest.raises(InputValidationError, match="'__g'.*reserved for meta"): + _ = ArrowTableSource( + _table_with_meta_named_column(), tag_columns=["k", "__g"], source_id="s" + ) + + +def test_map_tags_refuses_a_meta_named_target() -> None: + """Refused when built, so neither path nor the prediction can produce one.""" + with pytest.raises(InputValidationError, match="'__t'.*reserved for meta"): + _ = MapTags({"ta": "__t"}) + + +def test_join_keeps_a_tag_and_drops_the_data_meta_columns() -> None: + """Join drops only data meta columns; a tag is never one (it once dropped by prefix). + + A stream with a ``__`` tag cannot be built, so the one case left is data meta + next to ordinary tags: the tags all survive, the meta columns do not. + """ + left = _with_meta(source_a(), _meta_columns("m", 3)) + stream = Join()(left, source_b(nested=False)) + tag_keys, data_keys = stream.keys() + assert set(tag_keys) == {"k", "ta"} + assert not [name for name in data_keys if name.startswith("__")] + assert not [ + name for name in stream.as_table(all_info=True).column_names if "__m" in name + ] From 1f630e4b5d8df17d663cebae3186a767a7977627 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 18:24:17 -0800 Subject: [PATCH 08/12] fix(operators): address CodeRabbit review of #369 (ITL-1080) - MapData's sync path no longer renames a raw stream's meta column into a data column, and its prediction agrees with the async path. - SemiJoin never matches on a meta column, sync and validation alike. - _materialize_upstream keeps the upstream's data context (constructor and context column), on both branches. - without_meta_fields keeps the optional fields it retains. - MapData refuses a meta-prefixed new name (schema_utils.require_names_not_meta, now shared by tags and data names). - changelog wording. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 4 +- CLAUDE.md | 4 +- changelog.d/ITL-1080.md | 8 +-- design_issues/ITL-1080-SA1.md | 4 +- docs/concepts/operators.md | 5 +- src/orcapod/core/operators/mappers.py | 15 ++++-- src/orcapod/core/operators/meta_columns.py | 7 ++- src/orcapod/core/operators/semijoin.py | 8 ++- .../core/operators/static_output_pod.py | 20 ++++++-- src/orcapod/core/sources/stream_builder.py | 2 +- .../core/streams/arrow_table_stream.py | 2 +- src/orcapod/utils/schema_utils.py | 18 +++---- .../operators/test_async_sync_equivalence.py | 50 +++++++++++++++++++ .../test_operator_node_async_readback.py | 30 +++++++++++ .../test_system_tag_run_stability.py | 9 +++- 15 files changed, 153 insertions(+), 33 deletions(-) diff --git a/.zed/rules b/.zed/rules index d761c80be..11a35a034 100644 --- a/.zed/rules +++ b/.zed/rules @@ -567,8 +567,8 @@ No operator carries its inputs' Data meta (`__`) columns (developer, November 13 ITL-1080): on the sync and the async path, in rows and in the predicted output schema, for a raw ArrowTableStream's `__` columns and for the meta a cached pod's or function node's rows carry. One rule, core/operators/meta_columns.py. A carry-forward mode is ITL-867. -A tag column never has a name starting with the meta prefix (`__`): ArrowTableStream, the -source builder and MapTags refuse one (schema_utils.require_tag_names_not_meta). +The meta prefix (`__`) is reserved: ArrowTableStream and the source builder refuse a tag named +with it, and MapTags / MapData refuse it as a new name (schema_utils.require_names_not_meta). Operator or derived source? (developer, November 11, 2026) The test is: does any data element come from a tag value, or from anywhere other than the input's data elements? If yes, diff --git a/CLAUDE.md b/CLAUDE.md index cd9d0671a..f09b82bb1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -587,8 +587,8 @@ ITL-1080): on the sync and the async path, in rows and in the predicted output s raw `ArrowTableStream`'s `__` columns and for the meta a cached pod's or function node's rows carry. The rule is in one place, `core/operators/meta_columns.py` (sync results through the operator base classes, async rows through `WithoutMetaWriter`). A carry-forward mode is ITL-867. -A tag column never has a name starting with the meta prefix (`__`): `ArrowTableStream`, the -source builder and `MapTags` refuse one (`schema_utils.require_tag_names_not_meta`). +The meta prefix (`__`) is reserved: `ArrowTableStream` and the source builder refuse a tag named +with it, and `MapTags` / `MapData` refuse it as a new name (`schema_utils.require_names_not_meta`). **Operator or derived source?** (developer, November 11, 2026) The test is: does any data element come from a tag value, or from anywhere other than the input's data elements? If yes, diff --git a/changelog.d/ITL-1080.md b/changelog.d/ITL-1080.md index 565e2defe..baa3a73f4 100644 --- a/changelog.d/ITL-1080.md +++ b/changelog.d/ITL-1080.md @@ -1,6 +1,6 @@ ### Changed -- No operator carries its inputs' `Data` meta (`__`-prefixed) columns any more, on the sync and +- No operator carries its inputs' `Data` meta (`__`-prefixed) columns anymore, on the sync and the async path alike, as `Join` already did (ITL-1080). An operator's output rows and its predicted output schema hold none of them, whether they came from a raw `ArrowTableStream` whose table has `__` columns or from the rows of a cached function pod or function node @@ -12,9 +12,9 @@ - A tag column whose name starts with `__`, the prefix reserved for meta columns, is refused with `InputValidationError` where it would be made: building an `ArrowTableStream`, building a source with such a name in `tag_columns` (before, a "tag_columns not found" error, since the - source drops `__` columns), and `MapTags` renaming a tag to such a name (ITL-1080). Before, a - raw stream could carry one, and every path that rebuilt rows silently lost it, so sync and - async runs differed. + source drops `__` columns), and `MapTags` renaming a tag to such a name (ITL-1080). `MapData` + likewise refuses to rename a data column to such a name. Before, a raw stream could carry such + a tag, and every path that rebuilt rows silently lost it, so sync and async runs differed. ### Fixed diff --git a/design_issues/ITL-1080-SA1.md b/design_issues/ITL-1080-SA1.md index da90605c9..366fc30d4 100644 --- a/design_issues/ITL-1080-SA1.md +++ b/design_issues/ITL-1080-SA1.md @@ -40,5 +40,5 @@ and emitted rows with empty `Data` on the async path. Both now raise one `InputV A tag named with the meta prefix (`__g`) was a meta column to the `Tag` datagram, so every path that rebuilds rows (every `async_execute`, `Index`/`Pick` on sync, both orchestrators) lost it while table-based sync paths kept it, and `Join` failed on it outright. Such a tag is now -refused where it would be made (`schema_utils.require_tag_names_not_meta`: `ArrowTableStream`, -`SourceStreamBuilder.build`, `MapTags`), and the meta helpers strip only non-tag `__` columns. +refused where it would be made (`schema_utils.require_names_not_meta`: `ArrowTableStream`, +`SourceStreamBuilder.build`, `MapTags`; `MapData` refuses it as a new data name), and the meta helpers strip only non-tag `__` columns. diff --git a/docs/concepts/operators.md b/docs/concepts/operators.md index 4732d1d4e..649469f57 100644 --- a/docs/concepts/operators.md +++ b/docs/concepts/operators.md @@ -41,8 +41,9 @@ No operator carries its inputs' meta columns: the `__`-prefixed columns, such as function pod's `__computed`, that describe how an input row was produced. An operator's output row is produced by the operator, so its `Data` holds none of them and its predicted output schema lists none, on the sync and the async path alike. A way to carry them forward -across operators is an open design question (ITL-867). A tag column can never have a meta column's -name: a stream, a source and `MapTags` refuse a tag whose name starts with `__`. +across operators is an open design question (ITL-867). The `__` prefix is reserved for meta +columns: a stream and a source refuse a tag column named with it, and `MapTags` and `MapData` +refuse it as the new name of a tag or data column. ## Operator categories diff --git a/src/orcapod/core/operators/mappers.py b/src/orcapod/core/operators/mappers.py index 580083942..4c74ee764 100644 --- a/src/orcapod/core/operators/mappers.py +++ b/src/orcapod/core/operators/mappers.py @@ -28,6 +28,9 @@ class MapData(UnaryOperator): def __init__( self, name_map: Mapping[str, str], drop_unmapped: bool = False, **kwargs ): + # A data column never gets a meta column's name (ITL-1080): refused here, + # so no path (sync, async or the prediction) can produce one. + schema_utils.require_names_not_meta(name_map.values()) self.name_map = dict(name_map) self.drop_unmapped = drop_unmapped super().__init__(**kwargs) @@ -47,7 +50,12 @@ def to_config(self) -> dict[str, Any]: return config def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: - tag_columns, data_columns = stream.keys() + tag_columns, listed_columns = stream.keys() + # A raw stream lists its meta (``__``) columns as data; they are not data, + # are never renamed, and no operator carries them (ITL-1080). + data_columns = tuple( + c for c in listed_columns if not c.startswith(constants.META_PREFIX) + ) unmapped_columns = set(data_columns) - set(self.name_map.keys()) if not any(n in data_columns for n in self.name_map): @@ -120,7 +128,8 @@ def unary_output_schema( new_data_schema = { self.name_map.get(k, k): v for k, v in data_schema.items() - if k in self.name_map or not self.drop_unmapped + if not k.startswith(constants.META_PREFIX) + and (k in self.name_map or not self.drop_unmapped) } return tag_schema, Schema(new_data_schema) @@ -183,7 +192,7 @@ def __init__( ): # A tag never gets a meta column's name (ITL-1080): refused here, so no # path (sync, async or the prediction) can produce one. - schema_utils.require_tag_names_not_meta(name_map.values()) + schema_utils.require_names_not_meta(name_map.values()) self.name_map = dict(name_map) self.drop_unmapped = drop_unmapped super().__init__(**kwargs) diff --git a/src/orcapod/core/operators/meta_columns.py b/src/orcapod/core/operators/meta_columns.py index a91a1f997..2eebd9117 100644 --- a/src/orcapod/core/operators/meta_columns.py +++ b/src/orcapod/core/operators/meta_columns.py @@ -69,7 +69,12 @@ def without_meta_fields(schema: Schema) -> Schema: name: data_type for name, data_type in schema.items() if not name.startswith(constants.META_PREFIX) - } + }, + optional_fields={ + name + for name in schema.optional_fields + if not name.startswith(constants.META_PREFIX) + }, ) diff --git a/src/orcapod/core/operators/semijoin.py b/src/orcapod/core/operators/semijoin.py index d0a1a3656..ca0a57fb8 100644 --- a/src/orcapod/core/operators/semijoin.py +++ b/src/orcapod/core/operators/semijoin.py @@ -1,7 +1,7 @@ from collections.abc import Sequence from typing import TYPE_CHECKING, Any -from orcapod.core.operators.meta_columns import WithoutMetaWriter +from orcapod.core.operators.meta_columns import WithoutMetaWriter, without_meta_fields from orcapod.channels import ReadableChannel, WritableChannel from orcapod.core.operators.base import BinaryOperator, require_shared_arrow_type from orcapod.core.operators.match_keys import frozen_scalar @@ -45,6 +45,9 @@ def binary_static_process( """ left_tag_schema, left_data_schema = left_stream.output_schema() right_tag_schema, right_data_schema = right_stream.output_schema() + # A raw stream's meta (``__``) columns are not data and never match (ITL-1080). + left_data_schema = without_meta_fields(left_data_schema) + right_data_schema = without_meta_fields(right_data_schema) # Find overlapping columns across all columns (tags + data) left_all_schema = schema_utils.union_schemas(left_tag_schema, left_data_schema) @@ -134,6 +137,9 @@ def validate_binary_inputs( try: left_tag_schema, left_data_schema = left_stream.output_schema() right_tag_schema, right_data_schema = right_stream.output_schema() + # Meta (``__``) columns never match, as in ``binary_static_process``. + left_data_schema = without_meta_fields(left_data_schema) + right_data_schema = without_meta_fields(right_data_schema) # Check that overlapping columns have compatible types across all columns left_all_schema = schema_utils.union_schemas( diff --git a/src/orcapod/core/operators/static_output_pod.py b/src/orcapod/core/operators/static_output_pod.py index bf8f862e1..4b8126eea 100644 --- a/src/orcapod/core/operators/static_output_pod.py +++ b/src/orcapod/core/operators/static_output_pod.py @@ -308,8 +308,16 @@ def _materialize_upstream( Returns: A stream over the rows (none when ``zero_rows``), in order. """ + from orcapod.core.base import DataContextMixin + from orcapod.system_constants import constants from orcapod.utils import arrow_utils + # The rebuilt stream is in the upstream's data context, as its rows are. + context = ( + upstream.data_context + if isinstance(upstream, DataContextMixin) + else contexts.resolve_context(upstream.data_context_key) + ) if rows: table, tag_keys = StaticOutputOperatorPod._rows_to_table(rows) if zero_rows: @@ -319,13 +327,16 @@ def _materialize_upstream( tag_schema, data_schema = upstream.output_schema( columns={"system_tags": True, "source": True} ) - converter = contexts.resolve_context( - upstream.data_context_key - ).type_converter table = arrow_utils.make_empty_table( - {**tag_schema, **data_schema}, converter + {**tag_schema, **data_schema}, context.type_converter ) tag_keys = upstream.keys()[0] + # The context column names the upstream's context on every row; without + # it, the stream would fill in the default context's key. + table = table.append_column( + pa.field(constants.CONTEXT_KEY, pa.large_string(), nullable=False), + pa.array([context.context_key] * table.num_rows, pa.large_string()), + ) # No ``source_info`` argument: the table's own ``_source_*`` columns # pass through, one value per row. return ArrowTableStream( @@ -333,6 +344,7 @@ def _materialize_upstream( tag_columns=tag_keys, producer=upstream.producer, upstreams=upstream.upstreams, + data_context=context, ) @staticmethod diff --git a/src/orcapod/core/sources/stream_builder.py b/src/orcapod/core/sources/stream_builder.py index 165ff6cab..d736317d8 100644 --- a/src/orcapod/core/sources/stream_builder.py +++ b/src/orcapod/core/sources/stream_builder.py @@ -188,7 +188,7 @@ def build( # A tag never has a meta column's name; the meta columns are dropped next, # so say so here rather than report the tag as missing (ITL-1080). - schema_utils.require_tag_names_not_meta(tag_columns_tuple) + schema_utils.require_names_not_meta(tag_columns_tuple) # 1. Drop system columns from raw input. table = arrow_utils.drop_system_columns(table) diff --git a/src/orcapod/core/streams/arrow_table_stream.py b/src/orcapod/core/streams/arrow_table_stream.py index ea2b09f25..bd265c23d 100644 --- a/src/orcapod/core/streams/arrow_table_stream.py +++ b/src/orcapod/core/streams/arrow_table_stream.py @@ -81,7 +81,7 @@ def __init__( prefix_info = {constants.SOURCE_PREFIX: source_info} # A tag never has a meta column's name (ITL-1080). - schema_utils.require_tag_names_not_meta(tag_columns) + schema_utils.require_names_not_meta(tag_columns) # determine tag columns first and then exclude any source info self._tag_columns = tuple(c for c in tag_columns if c in table.column_names) self._system_tag_columns = tuple( diff --git a/src/orcapod/utils/schema_utils.py b/src/orcapod/utils/schema_utils.py index b3c6c0035..890522ef0 100644 --- a/src/orcapod/utils/schema_utils.py +++ b/src/orcapod/utils/schema_utils.py @@ -579,16 +579,17 @@ def require_output_keys_not_tags( ) -def require_tag_names_not_meta(tag_names: Collection[str]) -> None: - """Raise if a tag column's name starts with the reserved meta prefix. +def require_names_not_meta(names: Collection[str]) -> None: + """Raise if a tag or data column name starts with the reserved meta prefix. Names starting with the meta prefix (``__``) are meta columns: a datagram keeps them apart from its tags and data, and no operator carries them - (ITL-1080). A tag with such a name would silently stop being a tag on every - path that rebuilds rows, so no stream, source or operator may produce one. + (ITL-1080). A tag or data column given such a name would silently stop being + one on every path that rebuilds rows, so no stream, source or operator may + produce one. Args: - tag_names: The tag column names. + names: Tag or data column names. Raises: InputValidationError: If a name starts with the meta prefix. The message @@ -596,10 +597,9 @@ def require_tag_names_not_meta(tag_names: Collection[str]) -> None: """ from orcapod.system_constants import constants - for name in sorted(tag_names): + for name in sorted(names): if name.startswith(constants.META_PREFIX): raise InputValidationError( - f"tag column {name!r} starts with {constants.META_PREFIX!r}, which is " - "reserved for meta columns; a tag needs another name. Rename the " - "column before building the stream or source." + f"column name {name!r} starts with {constants.META_PREFIX!r}, which is " + "reserved for meta columns; a tag or data column needs another name." ) diff --git a/tests/test_core/operators/test_async_sync_equivalence.py b/tests/test_core/operators/test_async_sync_equivalence.py index 0a0faac85..ef149917f 100644 --- a/tests/test_core/operators/test_async_sync_equivalence.py +++ b/tests/test_core/operators/test_async_sync_equivalence.py @@ -1531,3 +1531,53 @@ def test_join_keeps_a_tag_and_drops_the_data_meta_columns() -> None: assert not [ name for name in stream.as_table(all_info=True).column_names if "__m" in name ] + + +def test_map_data_does_not_rename_a_meta_column_into_data() -> None: + """A raw stream's ``__`` column is meta: ``MapData`` renames it on neither path. + + The sync path used to build its renames from ``keys()``, which on a raw stream + lists ``__`` columns, so the meta value reached a data column (CodeRabbit, #369). + """ + source = _with_meta(source_a(), _meta_columns("m", 3)) + expected = sync_rows(MapData({"__m_i8": "renamed"}), source) + actual = async_rows(MapData({"__m_i8": "renamed"}), source) + for rows in (expected, actual): + for _, data in rows: + assert "renamed" not in data.keys() + assert_equivalent(expected, actual) + _, predicted = MapData({"__m_i8": "renamed"}).output_schema(source) + assert "renamed" not in predicted + + +def test_map_data_refuses_a_meta_named_target() -> None: + """A data column never gets a meta column's name: refused when built.""" + with pytest.raises(InputValidationError, match="'__x'.*reserved for meta"): + _ = MapData({"i8": "__x"}) + + +def test_semijoin_does_not_match_on_a_meta_column() -> None: + """A ``__`` column both raw inputs carry is not a match key, on either path.""" + left = _with_meta(source_c(), {"__shared": pa.array([1, 2, 3], pa.int8())}) + right = _with_meta( + source_b(nested=False), {"__shared": pa.array([9, 9, 9], pa.int8())} + ) + expected = sync_rows(SemiJoin(), left, right) + op = SemiJoin() + _ = op(left, right) # records the common keys, as an operator node does + actual = async_rows(op, left, right) + assert len(expected) == len(actual) == 3 + assert_equivalent(expected, actual) + + +def test_without_meta_fields_keeps_optional_fields() -> None: + """Dropping the meta fields leaves the optionality of the others (CodeRabbit, #369).""" + from orcapod.core.operators.meta_columns import without_meta_fields + from orcapod.types import Schema + + schema = Schema( + {"x": int, "y": int, "__note": str}, optional_fields={"x", "__note"} + ) + kept = without_meta_fields(schema) + assert dict(kept) == {"x": int, "y": int} + assert kept.optional_fields == frozenset({"x"}) diff --git a/tests/test_pipeline/test_operator_node_async_readback.py b/tests/test_pipeline/test_operator_node_async_readback.py index d4ee9f85f..e0f067fa3 100644 --- a/tests/test_pipeline/test_operator_node_async_readback.py +++ b/tests/test_pipeline/test_operator_node_async_readback.py @@ -721,3 +721,33 @@ def test_join_with_an_input_that_delivered_no_row_fails_alike_on_both_paths( error = error.exceptions[0] assert isinstance(error, InputValidationError) assert "'id'" in str(error) + + +def test_rebuilt_upstream_keeps_the_upstream_data_context() -> None: + """Rows rebuilt as an upstream's stream keep that stream's data context. + + Non-public on purpose: ``_materialize_upstream`` is how both orchestrators rebuild + an operator node's inputs, and a second data context is not reachable from a job. + """ + import dataclasses + + from orcapod.contexts import get_default_context + from orcapod.core.operators.static_output_pod import StaticOutputOperatorPod + from orcapod.core.streams import ArrowTableStream + + other = dataclasses.replace(get_default_context(), context_key="std:v0.9:other") + upstream = ArrowTableStream( + pa.table({"id": pa.array([1, 2], pa.int8()), "x": pa.array([3, 4], pa.int8())}), + tag_columns=["id"], + data_context=other, + ) + rows = list(upstream.iter_data()) + for buffer, zero_rows in ((rows, False), (rows, True), ([], False)): + stream = StaticOutputOperatorPod._materialize_upstream( + buffer, upstream, zero_rows=zero_rows + ) + assert stream.data_context_key == "std:v0.9:other" + contexts_column = stream.as_table(columns={"context": True}).column( + "_context_key" + ) + assert set(contexts_column.to_pylist()) <= {"std:v0.9:other"} diff --git a/tests/test_pipeline/test_system_tag_run_stability.py b/tests/test_pipeline/test_system_tag_run_stability.py index e4b67a735..df746a9f3 100644 --- a/tests/test_pipeline/test_system_tag_run_stability.py +++ b/tests/test_pipeline/test_system_tag_run_stability.py @@ -17,6 +17,8 @@ import pyarrow as pa import pytest +from orcapod import contexts + import orcapod as op from orcapod import side_effects from orcapod.core.datagrams import Data, Tag @@ -146,7 +148,12 @@ def test_operator_materialize_to_stream(self) -> None: def test_orchestrator_materialize_as_stream(self) -> None: """The orchestrator materializer emits rows in the first row's column order.""" - upstream = SimpleNamespace(producer=None, upstreams=()) + # The rebuilt stream takes the upstream's data context (ITL-1080). + upstream = SimpleNamespace( + producer=None, + upstreams=(), + data_context_key=contexts.get_default_context_key(), + ) stream = SyncPipelineOrchestrator._materialize_as_stream( _rows_with_swapped_tag_order(), upstream ) From 7430fc8e87dd867883325159ea7b65db600a91d4 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 18:38:11 -0800 Subject: [PATCH 09/12] fix(mappers): MapData and MapTags agree on both paths when nothing is renamed (ITL-1080) CodeRabbit round 2 of #369: - MapData's collision check skips meta (__) sources. - MapData(drop_unmapped=True) that would leave no data column raises the shared InputValidationError on both paths (require_a_data_column). - MapTags(drop_unmapped=True) with no applicable source drops every tag on both paths, as its prediction says. - The MapData meta test compares its prediction with the static result. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1080.md | 6 ++ .../core/operators/column_selection.py | 10 ++-- src/orcapod/core/operators/mappers.py | 20 +++++-- .../operators/test_async_sync_equivalence.py | 58 ++++++++++++++++++- 4 files changed, 82 insertions(+), 12 deletions(-) diff --git a/changelog.d/ITL-1080.md b/changelog.d/ITL-1080.md index baa3a73f4..03afd4e8d 100644 --- a/changelog.d/ITL-1080.md +++ b/changelog.d/ITL-1080.md @@ -30,3 +30,9 @@ operator is applied on the sync path and on the first row on the async path, with one message (ITL-1080). Before, the sync path failed only when the rows were read, with a `ValueError`, and the async path emitted rows with empty data. +- `MapData(..., drop_unmapped=True)` whose sources are not in the stream would leave no data + column: it now raises the same `InputValidationError` on both paths, where both paths used to + keep every data column while the prediction listed none. `MapTags(..., drop_unmapped=True)` in + the same case drops every tag on both paths, as its prediction says; the async path used to + keep them. A meta (`__`) source of `MapData` is never renamed and no longer reports a false + collision (ITL-1080). diff --git a/src/orcapod/core/operators/column_selection.py b/src/orcapod/core/operators/column_selection.py index 4e2cd5323..b8db82268 100644 --- a/src/orcapod/core/operators/column_selection.py +++ b/src/orcapod/core/operators/column_selection.py @@ -35,7 +35,7 @@ def _canonical_columns(columns: str | Collection[str]) -> tuple[str, ...]: return tuple(sorted(set(columns))) -def _require_a_data_column(remaining: Collection[str]) -> None: +def require_a_data_column(remaining: Collection[str]) -> None: """Refuse a selection whose result would have no data column. A stream needs at least one data column (``ArrowTableStream`` refuses a table @@ -240,7 +240,7 @@ def validate_unary_input(self, stream: StreamProtocol) -> None: raise InputValidationError( f"Missing data columns: {missing_columns}. Make sure all specified columns to select are present or use strict=False to ignore missing columns" ) - _require_a_data_column([c for c in data_columns if c in columns_to_select]) + require_a_data_column([c for c in data_columns if c in columns_to_select]) def unary_output_schema( self, @@ -292,7 +292,7 @@ async def async_execute( f"specified columns to select are present or use " f"strict=False to ignore missing columns" ) - _require_a_data_column([c for c in pkt_keys if c in self.columns]) + require_a_data_column([c for c in pkt_keys if c in self.columns]) pkts_to_drop = [c for c in pkt_keys if c not in self.columns] if not pkts_to_drop: await output.send((tag, data)) @@ -500,7 +500,7 @@ def validate_unary_input(self, stream: StreamProtocol) -> None: raise InputValidationError( f"Missing data columns: {missing_columns}. Make sure all specified columns to drop are present or use strict=False to ignore missing columns" ) - _require_a_data_column([c for c in data_columns if c not in self.columns]) + require_a_data_column([c for c in data_columns if c not in self.columns]) def unary_output_schema( self, @@ -549,7 +549,7 @@ async def async_execute( f"specified columns to drop are present or use " f"strict=False to ignore missing columns" ) - _require_a_data_column( + require_a_data_column( [c for c in pkt_keys if c not in self.columns] ) effective_drops = ( diff --git a/src/orcapod/core/operators/mappers.py b/src/orcapod/core/operators/mappers.py index 4c74ee764..2175d6ff3 100644 --- a/src/orcapod/core/operators/mappers.py +++ b/src/orcapod/core/operators/mappers.py @@ -4,6 +4,7 @@ from orcapod.core.operators.meta_columns import WithoutMetaWriter from orcapod.channels import ReadableChannel, WritableChannel from orcapod.core.operators.base import UnaryOperator +from orcapod.core.operators.column_selection import require_a_data_column from orcapod.core.streams import ArrowTableStream from orcapod.errors import InputValidationError from orcapod.protocols.core_protocols import DataProtocol, StreamProtocol, TagProtocol @@ -98,9 +99,13 @@ def validate_unary_input(self, stream: StreamProtocol) -> None: relevant_source = [] relevant_target = [] for source, target in self.name_map.items(): - if source in data_columns: + # A meta (``__``) column a raw stream lists as data is never renamed. + if source in data_columns and not source.startswith(constants.META_PREFIX): relevant_source.append(source) relevant_target.append(target) + if self.drop_unmapped: + # Only the renamed columns remain, and a stream needs a data column. + require_a_data_column(relevant_target) remaining_data_columns = set(data_columns) - set(relevant_source) overlapping_data_columns = remaining_data_columns.intersection(relevant_target) overlapping_tag_columns = set(tag_columns).intersection(relevant_target) @@ -161,6 +166,7 @@ async def async_execute( k: self.name_map[k] for k in pkt_keys if k in self.name_map } if self.drop_unmapped: + require_a_data_column(list(rename_map.values())) unmapped = [k for k in pkt_keys if k not in self.name_map] if not rename_map: await output.send((tag, data)) @@ -215,8 +221,10 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: tag_columns, data_columns = stream.keys() missing_tags = set(tag_columns) - set(self.name_map.keys()) - if not any(n in tag_columns for n in self.name_map): - # nothing to rename in the tags, return stream as is + if not any(n in tag_columns for n in self.name_map) and not ( + self.drop_unmapped and missing_tags + ): + # nothing to rename or drop in the tags, return stream as is return stream table = stream.as_table( @@ -315,10 +323,12 @@ async def async_execute( } if self.drop_unmapped: unmapped = [k for k in tag_keys if k not in self.name_map] - if not rename_map: + if not rename_map and not unmapped: await output.send((tag, data)) else: - new_tag = tag.rename(rename_map) + # With ``drop_unmapped`` the unmapped tags go even when no + # tag is renamed, as on the sync path and in the prediction. + new_tag = tag.rename(rename_map) if rename_map else tag if unmapped: new_tag = new_tag.drop(*unmapped) await output.send((new_tag, data)) diff --git a/tests/test_core/operators/test_async_sync_equivalence.py b/tests/test_core/operators/test_async_sync_equivalence.py index ef149917f..56ade9214 100644 --- a/tests/test_core/operators/test_async_sync_equivalence.py +++ b/tests/test_core/operators/test_async_sync_equivalence.py @@ -1546,8 +1546,62 @@ def test_map_data_does_not_rename_a_meta_column_into_data() -> None: for _, data in rows: assert "renamed" not in data.keys() assert_equivalent(expected, actual) - _, predicted = MapData({"__m_i8": "renamed"}).output_schema(source) - assert "renamed" not in predicted + op = MapData({"__m_i8": "renamed"}) + predicted = op.output_schema(source) + assert "renamed" not in predicted[1] + # Non-public on purpose: a prediction is verified against the static result. + assert predicted == op.static_process(source).output_schema() + + +def test_map_data_meta_source_does_not_collide() -> None: + """A meta source is not a rename source, so it cannot collide with a data column.""" + source = _with_meta(source_b(nested=False), _meta_columns("m", 3)) + op = MapData({"__m_i8": "i32", "u8": "v8"}) + expected = sync_rows(op, source) + actual = async_rows(MapData({"__m_i8": "i32", "u8": "v8"}), source) + for _, data in expected: + assert set(data.keys()) == {"i32", "v8", "f32", "ls"} + assert_equivalent(expected, actual) + + +@pytest.mark.parametrize( + "name_map", + [{"absent": "a"}, {"__m_i8": "renamed"}], + ids=["absent_source", "meta_source"], +) +def test_map_data_dropping_every_data_column_raises_alike( + name_map: dict[str, str], +) -> None: + """``drop_unmapped`` with no applicable source would leave no data column. + + Before, both paths kept every data column while the prediction listed none. + """ + source = _with_meta(source_b(nested=False), _meta_columns("m", 3)) + with pytest.raises(InputValidationError, match="no data column") as sync_error: + _ = MapData(name_map, drop_unmapped=True)(source) + with pytest.raises(InputValidationError, match="no data column") as async_error: + _ = async_rows(MapData(name_map, drop_unmapped=True), source) + assert str(async_error.value) == str(sync_error.value) + + +def test_map_tags_dropping_every_tag_agrees_on_both_paths() -> None: + """``MapTags`` with ``drop_unmapped`` and no applicable source drops every tag. + + A stream may have no tags, so this is not an error. Before, the async path kept + every tag while the sync path and the prediction dropped them. + """ + source = source_a() + op = MapTags({"absent": "a"}, drop_unmapped=True) + expected = sync_rows(op, source) + actual = async_rows(MapTags({"absent": "a"}, drop_unmapped=True), source) + assert op.output_schema(source)[0] == {} + for rows in (expected, actual): + assert len(rows) == 3 + for tag, _ in rows: + assert tag.keys() == () + for (tag, data), (sync_tag, sync_data) in zip(actual, expected): + assert data.content_hash() == sync_data.content_hash() + assert list(tag.system_tags().items()) == list(sync_tag.system_tags().items()) def test_map_data_refuses_a_meta_named_target() -> None: From d18ba9b19e4ffcfffeaa4a3f0ab7fe923df93264 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 18:56:10 -0800 Subject: [PATCH 10/12] fix(column_selection): a strict selection naming a meta column fails alike on both paths (ITL-1080) CodeRabbit full review of #369: a raw stream lists its __ columns in keys(), so a strict SelectDataColumns / DropDataColumns naming one passed on sync and reported it missing on async. The sync validators now skip meta names too. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1080.md | 3 +- .../core/operators/column_selection.py | 14 +++++++-- .../operators/test_async_sync_equivalence.py | 30 +++++++++++++++++++ 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/changelog.d/ITL-1080.md b/changelog.d/ITL-1080.md index 03afd4e8d..c23b065f7 100644 --- a/changelog.d/ITL-1080.md +++ b/changelog.d/ITL-1080.md @@ -35,4 +35,5 @@ keep every data column while the prediction listed none. `MapTags(..., drop_unmapped=True)` in the same case drops every tag on both paths, as its prediction says; the async path used to keep them. A meta (`__`) source of `MapData` is never renamed and no longer reports a false - collision (ITL-1080). + collision, and a strict `SelectDataColumns` / `DropDataColumns` naming a meta column reports it + missing on both paths (ITL-1080). diff --git a/src/orcapod/core/operators/column_selection.py b/src/orcapod/core/operators/column_selection.py index addd1a8fb..4cc280fc9 100644 --- a/src/orcapod/core/operators/column_selection.py +++ b/src/orcapod/core/operators/column_selection.py @@ -234,7 +234,12 @@ def validate_unary_input(self, stream: StreamProtocol) -> None: It takes two streams as input and raises an error if the inputs are not valid. """ # TODO: remove redundant logic - tag_columns, data_columns = stream.keys() + tag_columns, listed_columns = stream.keys() + # A raw stream lists its meta (``__``) columns as data; they are not data + # columns to select, as on the async path (ITL-1080). + data_columns = [ + c for c in listed_columns if not c.startswith(constants.META_PREFIX) + ] columns_to_select = self.columns missing_columns = set(columns_to_select) - set(data_columns) if missing_columns and self.strict: @@ -495,7 +500,12 @@ def validate_unary_input(self, stream: StreamProtocol) -> None: It takes two streams as input and raises an error if the inputs are not valid. """ # TODO: remove redundant logic - _, data_columns = stream.keys() + _, listed_columns = stream.keys() + # A raw stream lists its meta (``__``) columns as data; they are not data + # columns to drop, as on the async path (ITL-1080). + data_columns = [ + c for c in listed_columns if not c.startswith(constants.META_PREFIX) + ] missing_columns = set(self.columns) - set(data_columns) if missing_columns and self.strict: raise InputValidationError( diff --git a/tests/test_core/operators/test_async_sync_equivalence.py b/tests/test_core/operators/test_async_sync_equivalence.py index 4df659121..6301a7a01 100644 --- a/tests/test_core/operators/test_async_sync_equivalence.py +++ b/tests/test_core/operators/test_async_sync_equivalence.py @@ -1648,3 +1648,33 @@ def test_without_meta_fields_keeps_optional_fields() -> None: kept = without_meta_fields(schema) assert dict(kept) == {"x": int, "y": int} assert kept.optional_fields == frozenset({"x"}) + + +@pytest.mark.parametrize( + "make_op", + [ + lambda: SelectDataColumns(["i32", "__note"]), + lambda: DropDataColumns(["__note"]), + ], + ids=["select", "drop"], +) +def test_strict_selection_naming_a_meta_column_raises_alike( + make_op: Callable[[], StaticOutputOperatorPod], +) -> None: + """A raw stream's ``__`` column is not a data column for a strict selection. + + The sync path found it in ``keys()``; the async path, whose ``Data`` keeps it as + meta, reported it missing (CodeRabbit, #369). Both now report it missing. + """ + source = _with_meta( + source_b(nested=False), {"__note": pa.array(["a", "b", "c"], pa.string())} + ) + with pytest.raises( + InputValidationError, match="Missing data columns" + ) as sync_error: + _ = make_op()(source) + with pytest.raises( + InputValidationError, match="Missing data columns" + ) as async_error: + _ = async_rows(make_op(), source) + assert str(async_error.value) == str(sync_error.value) From 44d82ae887dee791d9ea85ba2e174d5bcd0381b7 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 19:03:06 -0800 Subject: [PATCH 11/12] refactor(column_selection): one require_a_data_column and one meta writer for every column operator (ITL-1080) ITL-1084's SelectColumns/DropColumns (merged from main) had their own no-data-column helper and dropped meta inline; they now share require_a_data_column (keeping ITL-1084's message) and WithoutMetaWriter. Co-Authored-By: Claude Opus 5.5 --- .../core/operators/column_selection.py | 33 ++++--------------- 1 file changed, 7 insertions(+), 26 deletions(-) diff --git a/src/orcapod/core/operators/column_selection.py b/src/orcapod/core/operators/column_selection.py index 4cc280fc9..64554270e 100644 --- a/src/orcapod/core/operators/column_selection.py +++ b/src/orcapod/core/operators/column_selection.py @@ -37,11 +37,13 @@ def _canonical_columns(columns: str | Collection[str]) -> tuple[str, ...]: def require_a_data_column(remaining: Collection[str]) -> None: - """Refuse a selection whose result would have no data column. + """Refuse a column operation whose result would have no data column. A stream needs at least one data column (``ArrowTableStream`` refuses a table without one), so the sync path checks this when the operator is applied and - the async path on the first row, with this one error. Meta (``__``) columns + the async path on the first row, with this one error. Every column operator + (``SelectDataColumns``, ``DropDataColumns``, ``SelectColumns``, + ``DropColumns``, ``MapData``) uses it. Meta (``__``) columns are not data: no operator carries them (ITL-1080). Args: @@ -52,8 +54,7 @@ def require_a_data_column(remaining: Collection[str]) -> None: """ if not [c for c in remaining if not c.startswith(constants.META_PREFIX)]: raise InputValidationError( - "The result would have no data column; a stream needs at least one. " - "Keep at least one data column." + "no data column would remain: a stream needs at least one data column" ) @@ -583,25 +584,6 @@ def identity_structure(self) -> Any: ) -def _require_a_data_column(remaining_data_columns: Collection[str]) -> None: - """Raise unless at least one data column remains. - - A stream needs at least one data column (``ArrowTableStream`` refuses a table - without one), so a column operator whose result would have none is refused - when it is called, before any row is built. - - Args: - remaining_data_columns: The data columns the result would keep. - - Raises: - InputValidationError: When ``remaining_data_columns`` is empty. - """ - if not remaining_data_columns: - raise InputValidationError( - "no data column would remain: a stream needs at least one data column" - ) - - def _is_meta(name: str) -> bool: """Whether ``name`` is a meta column or a raw stream's source-info column for one.""" return name.startswith( @@ -688,7 +670,7 @@ def _validate_columns( ) _, data_removed = self._removals(tag_columns, data_columns) try: - _require_a_data_column([c for c in data_columns if c not in data_removed]) + require_a_data_column([c for c in data_columns if c not in data_removed]) except InputValidationError as error: raise InputValidationError( f"{self.__class__.__name__}({list(self.columns)}) would leave no data " @@ -780,6 +762,7 @@ async def async_execute( pipeline hash. """ try: + output = WithoutMetaWriter(output) removals: tuple[list[str], list[str]] | None = None async for tag, data in inputs[0]: if removals is None: @@ -791,8 +774,6 @@ async def async_execute( tag = tag.drop(*tags_removed) if data_removed: data = data.drop(*data_removed) - if data.meta_columns: - data = data.drop_meta_columns(*data.meta_columns) await output.send((tag, data)) finally: await output.close() From a236bc4456ee23f51e306d4f4b5d4526d7c86272 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 21:54:20 -0800 Subject: [PATCH 12/12] fix(pick, index): refuse an out name with the meta prefix (ITL-1080) Pick(out="__x") and Index(out="__x") made a data column with a meta column's name, which every path then dropped: the run completed without the requested column and no error. Both now refuse such an out name at construction through schema_utils.require_names_not_meta, as MapData and MapTags refuse such a target. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 3 ++- CLAUDE.md | 3 ++- changelog.d/ITL-1080.md | 3 ++- src/orcapod/core/operators/index.py | 6 +++++- src/orcapod/core/operators/pick.py | 6 +++++- tests/test_core/operators/test_pick_index.py | 14 ++++++++++++++ 6 files changed, 30 insertions(+), 5 deletions(-) diff --git a/.zed/rules b/.zed/rules index 38ed25352..a469e37bb 100644 --- a/.zed/rules +++ b/.zed/rules @@ -572,7 +572,8 @@ ITL-1080): on the sync and the async path, in rows and in the predicted output s raw ArrowTableStream's `__` columns and for the meta a cached pod's or function node's rows carry. One rule, core/operators/meta_columns.py. A carry-forward mode is ITL-867. The meta prefix (`__`) is reserved: ArrowTableStream and the source builder refuse a tag named -with it, and MapTags / MapData refuse it as a new name (schema_utils.require_names_not_meta). +with it, and MapTags / MapData (targets) and Pick / Index (out) refuse it as a new name +(schema_utils.require_names_not_meta, at construction). Operator or derived source? (developer, November 11, 2026) The test is: does any data element come from a tag value, or from anywhere other than the input's data elements? If yes, diff --git a/CLAUDE.md b/CLAUDE.md index bcb742523..dbc401cbc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -592,7 +592,8 @@ raw `ArrowTableStream`'s `__` columns and for the meta a cached pod's or functio carry. The rule is in one place, `core/operators/meta_columns.py` (sync results through the operator base classes, async rows through `WithoutMetaWriter`). A carry-forward mode is ITL-867. The meta prefix (`__`) is reserved: `ArrowTableStream` and the source builder refuse a tag named -with it, and `MapTags` / `MapData` refuse it as a new name (`schema_utils.require_names_not_meta`). +with it, and `MapTags` / `MapData` (targets) and `Pick` / `Index` (`out`) refuse it as a new name +(`schema_utils.require_names_not_meta`, at construction). **Operator or derived source?** (developer, November 11, 2026) The test is: does any data element come from a tag value, or from anywhere other than the input's data elements? If yes, diff --git a/changelog.d/ITL-1080.md b/changelog.d/ITL-1080.md index c23b065f7..92b1d7e93 100644 --- a/changelog.d/ITL-1080.md +++ b/changelog.d/ITL-1080.md @@ -13,7 +13,8 @@ `InputValidationError` where it would be made: building an `ArrowTableStream`, building a source with such a name in `tag_columns` (before, a "tag_columns not found" error, since the source drops `__` columns), and `MapTags` renaming a tag to such a name (ITL-1080). `MapData` - likewise refuses to rename a data column to such a name. Before, a raw stream could carry such + likewise refuses to rename a data column to such a name, and `Pick` and `Index` refuse such an + `out` name (before, the picked column was silently dropped). Before, a raw stream could carry such a tag, and every path that rebuilt rows silently lost it, so sync and async runs differed. ### Fixed diff --git a/src/orcapod/core/operators/index.py b/src/orcapod/core/operators/index.py index ed8e5a477..06956138e 100644 --- a/src/orcapod/core/operators/index.py +++ b/src/orcapod/core/operators/index.py @@ -15,7 +15,7 @@ from orcapod.errors import InputValidationError from orcapod.protocols.core_protocols import DataProtocol, StreamProtocol, TagProtocol from orcapod.types import ColumnConfig, ContentHash, Schema, SourceInfoValue -from orcapod.utils import arrow_utils +from orcapod.utils import arrow_utils, schema_utils from orcapod.utils.lazy_module import LazyModule if TYPE_CHECKING: @@ -60,6 +60,10 @@ def __init__( ) -> None: self.column = column self.i = i + # A data column never gets a meta column's name (ITL-1080): refused here, + # as ``MapData`` refuses it, so no path can produce one and drop it. + if out is not None: + schema_utils.require_names_not_meta([out]) self.out = out self.fail_on_miss = fail_on_miss self._output_type: type = None # type: ignore[assignment] diff --git a/src/orcapod/core/operators/pick.py b/src/orcapod/core/operators/pick.py index 0da738473..ace2c3622 100644 --- a/src/orcapod/core/operators/pick.py +++ b/src/orcapod/core/operators/pick.py @@ -14,7 +14,7 @@ from orcapod.errors import InputValidationError from orcapod.protocols.core_protocols import DataProtocol, StreamProtocol, TagProtocol from orcapod.types import ColumnConfig, ContentHash, Schema -from orcapod.utils import arrow_utils +from orcapod.utils import arrow_utils, schema_utils from orcapod.utils.lazy_module import LazyModule if TYPE_CHECKING: @@ -61,6 +61,10 @@ def __init__( ) -> None: self.column = column self.key = key + # A data column never gets a meta column's name (ITL-1080): refused here, + # as ``MapData`` refuses it, so no path can produce one and drop it. + if out is not None: + schema_utils.require_names_not_meta([out]) self.out = out self.fail_on_miss = fail_on_miss self._output_type: type = None # type: ignore[assignment] diff --git a/tests/test_core/operators/test_pick_index.py b/tests/test_core/operators/test_pick_index.py index 43f072a87..5306674bc 100644 --- a/tests/test_core/operators/test_pick_index.py +++ b/tests/test_core/operators/test_pick_index.py @@ -67,6 +67,13 @@ def test_pick_out_collision_raises(dict_stream): Pick("scores", "speed", out="scores")(dict_stream) +def test_pick_out_with_the_meta_prefix_raises(): + """``__`` is reserved for meta columns, which no operator carries (ITL-1080): an + ``out`` named with it would be dropped on every path, so it is refused at once.""" + with pytest.raises(InputValidationError, match="reserved for meta columns"): + Pick("scores", "speed", out="__speed") + + # ── functional tests: Pick ──────────────────────────────────────────────────── @@ -232,6 +239,13 @@ def test_index_out_collision_raises(): Index("items", 0, out="items")(stream) +def test_index_out_with_the_meta_prefix_raises(): + """``__`` is reserved for meta columns, which no operator carries (ITL-1080): an + ``out`` named with it would be dropped on every path, so it is refused at once.""" + with pytest.raises(InputValidationError, match="reserved for meta columns"): + Index("items", 0, out="__first") + + # ── functional tests: Index ───────────────────────────────────────────────────