From df7e526ce070d8cf35899ef4df0bf1f8978847dd Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 10:50:26 -0800 Subject: [PATCH 01/13] test(operators): failing zero-row column order sweep (ITL-1131) Co-Authored-By: Claude Opus 5.5 --- ...26-11-14-itl-1131-zero-row-column-order.md | 86 +++++++ .../test_zero_row_column_order.py | 228 ++++++++++++++++++ 2 files changed, 314 insertions(+) create mode 100644 superpowers/plans/2026-11-14-itl-1131-zero-row-column-order.md create mode 100644 tests/test_pipeline/test_zero_row_column_order.py diff --git a/superpowers/plans/2026-11-14-itl-1131-zero-row-column-order.md b/superpowers/plans/2026-11-14-itl-1131-zero-row-column-order.md new file mode 100644 index 000000000..a7b1aafd9 --- /dev/null +++ b/superpowers/plans/2026-11-14-itl-1131-zero-row-column-order.md @@ -0,0 +1,86 @@ +# ITL-1131: a zero-row table has its populated table's column order — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Every operator's zero-row table (plain stream; job node before a run and after a run +whose input had no rows, sync and async) has exactly the populated table's schema: column +names, order, Arrow types, nullability. + +**Architecture:** The row order of a node's rows (`_rows_to_table`: user tags, system tags, +data, source info) gets one owner, `StaticOutputOperatorPod._in_row_order`. The places that +restate it (`_zero_row_input`, `OperatorJobNode._typed_zero_row_table`) call it, and so do the +two places that keep a run's zero-row result (sync `execute()`, async `async_execute()` empty +branch), which today keep the operator's own order. `Index` and `Pick` build their empty result +with the input's system tags and source info, in row order, with the source-info flags a +row-built table has. The populated order is never changed (stored tables depend on it). + +**Tech Stack:** Python, PyArrow, pytest, `uv run`. + +--- + +## Findings (sweep on main 52d67858, before the fix) + +Scripts: the worker's scratchpad `sweep.py`, `sweep_direct.py`, `latent.py` (copied into the +coordinator status message `msg_86fd0618208e`). + +| Path | Differs | Where the order is decided | +|---|---|---| +| plain stream `op(stream)`, zero input, `as_table` / `head()` | `Index`, `Pick`: system tags missing; `Pick`: `_source_*` nullable | `Index`/`Pick.unary_static_process` empty branch (`stream.as_table(columns={"source": True})` / `stream.as_table()`) | +| job node before run | `Index`, `Pick` as above | `OperatorJobNode._typed_zero_row_table` (row order) over `zero_row_result` | +| job node after a zero-input run, sync and async | `Join`, `MergeJoin`: ORDER (data before system tags); `Index`, `Pick` as above | `OperatorJobNode.execute()` keeps `result_stream`; `async_execute` empty branch keeps `operator.process(...)`: the operator's own order. Populated keeps `_materialize_to_stream` (row order) | +| direct `OperatorJobNode(...).run()` (no job) | before-run table is row order, populated is operator order: every operator | OUT OF SCOPE (coordinator, option A): two populated orders per node; reported | +| function pod (stream and node) | none | — | + +Coordinator decision (2026-11-14): approve a–d; item 4 (direct node) out of scope, reported. + +## Files + +- Modify: `src/orcapod/core/operators/static_output_pod.py` (`_in_row_order`, `_zero_row_input`, `_zero_rows_in_row_order`) +- Modify: `src/orcapod/core/nodes/operator_node.py` (`_typed_zero_row_table`, `execute`, `async_execute`) +- Modify: `src/orcapod/core/operators/index.py`, `src/orcapod/core/operators/pick.py` (empty branch) +- Create: `tests/test_pipeline/test_zero_row_column_order.py` +- Create: `changelog.d/ITL-1131.md`, `design_issues/ITL-1131-ZR2.md` + +### Task 1: Failing sweep test + +- [ ] Write `tests/test_pipeline/test_zero_row_column_order.py`: for every operator in + `core/operators` and a function pod, inputs with 1 and 2 system-tag pairs, compare + `as_table(all_info=True).schema` with `schema.equals`: + plain stream over a zero-row input and its `head(all_info=True)` against the plain stream + populated; job node before a run, and after a sync / async run whose input had no rows, + against the job node populated by the same orchestrator. Also `Index`/`Pick` with `out=`. +- [ ] Run `uv run pytest tests/test_pipeline/test_zero_row_column_order.py -q`: expect the Join, + MergeJoin, Index and Pick cases to fail, every other to pass. +- [ ] Commit `test(operators): failing zero-row column order sweep (ITL-1131)`. + +### Task 2: One owner of the row order; node keeps a zero-row run in it + +- [ ] `StaticOutputOperatorPod._in_row_order(table, tag_keys)`: select + `[*tag_keys, *system tags, *data (remaining, table order), *source info]`. +- [ ] `_zero_row_input` and `OperatorJobNode._typed_zero_row_table` call it. +- [ ] `StaticOutputOperatorPod._zero_rows_in_row_order(result)`: `ArrowTableStream` of + `_in_row_order(result.as_table(columns=ZERO_ROW_INPUT_COLUMNS), tag_keys)`, the result's data + context. `execute()` (no rows) and the async empty branch keep it instead of the result stream. +- [ ] Run the sweep: Join/MergeJoin pass. Run `tests/test_pipeline tests/test_core/operators tests/test_channels`. +- [ ] Commit `fix(operator_node): a run with no rows keeps the row order of a run with rows (ITL-1131)`. + +### Task 3: Index and Pick empty result + +- [ ] Empty branch: schema from `stream.as_table(columns={"source": True, "system_tags": True})` + (large-normalized), projected column (and token) replaced or appended as today, then + `_in_row_order` and `_flag_source_info_nullability` on the empty table. +- [ ] Run the sweep and `tests/test_core/operators -k "index or pick"`. +- [ ] Commit `fix(index, pick): an empty result keeps the system tags and source flags (ITL-1131)`. + +### Task 4: Docs, changelog, design issue, full suite, PR + +- [ ] `changelog.d/ITL-1131.md` (`### Fixed`), `design_issues/ITL-1131-ZR2.md` (resolved, Fix note). +- [ ] `uv run --with ruff ruff check .`, `ruff format --check .`, basedpyright (exit code), + `uv run pytest tests/`. +- [ ] Merge origin/main; after PR #380 merges, turn its strict xfail + `test_a_join_with_no_input_rows_keeps_its_column_order` into a passing test. +- [ ] Draft PR against main, `Fixes ITL-1131`. + +## Progress notes + +- 2026-11-14: sweep done, coordinator approved a–d, item 4 out of scope. diff --git a/tests/test_pipeline/test_zero_row_column_order.py b/tests/test_pipeline/test_zero_row_column_order.py new file mode 100644 index 000000000..1c782748e --- /dev/null +++ b/tests/test_pipeline/test_zero_row_column_order.py @@ -0,0 +1,228 @@ +"""A zero-row table has exactly its populated table's schema, column order included (ITL-1131). + +ITL-912 made a table with no rows carry the Arrow types and nullability of the rows. This +sweep checks the rest of that rule for every operator and for a function pod: the zero-row +table has the populated table's columns in the populated table's order. It compares +``as_table(all_info=True)`` with ``pa.Schema.equals`` (names, order, types, nullability): + +- a plain stream, ``op(stream)``, over an input with no rows, and its ``head()`` before it + computes, against the same stream over an input with rows; +- a job node before a run, and after a sync or async run whose input had no rows, against the + same job node after a run with rows by the same orchestrator. + +Inputs carry one or two system-tag pairs (a source, or a join of two sources). +""" + +from __future__ import annotations + +from collections.abc import Callable + +import pyarrow as pa +import pytest + +from orcapod import contexts, function_pod +from orcapod.core.operators import ( + Batch, + DropColumns, + DropDataColumns, + DropNulls, + DropTagColumns, + GroupBy, + Index, + Join, + MapData, + MapTags, + MergeJoin, + Pick, + PolarsFilter, + SelectColumns, + SelectDataColumns, + SelectTagColumns, + SemiJoin, +) +from orcapod.core.sources import ArrowTableSource +from orcapod.databases import InMemoryArrowDatabase +from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator +from orcapod.pipeline.job import PipelineJob +from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator +from orcapod.protocols.core_protocols import StreamProtocol + +ORCHESTRATORS = { + "sync": SyncPipelineOrchestrator, + "async": AsyncPipelineOrchestrator, +} + +WITH_ROWS = ("g1", "g1", "g2") +"""Groups of the sources' rows: the filter on ``g1`` keeps two rows.""" +NO_ROWS = ("g2", "g2", "g3") +"""Groups of the sources' rows: the filter on ``g1`` keeps none.""" + + +def _dict_column() -> pa.Array: + """A ``dict[str, int]`` column in the type the data context stores it with.""" + converter = contexts.resolve_context(None).type_converter + table = converter.python_dicts_to_arrow_table( + [{"m": {"k": i}} for i in (1, 2, 3)], python_schema={"m": dict[str, int]} + ) + return table.column("m").combine_chunks() + + +def _table(groups: tuple[str, ...], data: dict[str, pa.Array]) -> pa.Table: + """Tags ``g`` and ``id`` and ``data``; every column non-nullable except ``n``.""" + table = pa.table( + { + "g": pa.array(list(groups), pa.large_string()), + "id": pa.array([1, 2, 3], pa.int8()), + **data, + } + ) + return table.cast( + pa.schema( + [pa.field(f.name, f.type, nullable=f.name == "n") for f in table.schema] + ) + ) + + +def _source( + name: str, groups: tuple[str, ...], data: dict[str, pa.Array] +) -> StreamProtocol: + """A source of ``data`` behind a filter keeping group ``g1``.""" + source = ArrowTableSource( + _table(groups, data), tag_columns=["g", "id"], source_id=name + ) + return PolarsFilter(constraints={"g": "g1"})(source) + + +def _main(groups: tuple[str, ...]) -> StreamProtocol: + item = pa.field("item", pa.int8(), nullable=False) + return _source( + "a", + groups, + { + "x": pa.array([10, 20, 30], pa.int8()), + "l": pa.array([[1], [2, 3], [4]], pa.large_list(item)), + "n": pa.array([1, None, 3], pa.int8()), + "m": _dict_column(), + }, + ) + + +def _other(groups: tuple[str, ...]) -> StreamProtocol: + return _source("b", groups, {"z": pa.array([7, 8, 9], pa.int16())}) + + +def _colliding(groups: tuple[str, ...]) -> StreamProtocol: + return _source("c", groups, {"x": pa.array([1, 2, 3], pa.int8())}) + + +def _input(groups: tuple[str, ...], pairs: int) -> StreamProtocol: + """The operator's input: one source, or a join of two (two system-tag pairs).""" + stream = _main(groups) + if pairs == 2: + extra = _source("d", groups, {"w": pa.array([5, 6, 7], pa.int32())}) + stream = Join()(stream, extra) + return stream + + +@function_pod(output_keys="y") +def _plus_one(x: int) -> int: + return x + 1 + + +Step = Callable[[StreamProtocol, tuple[str, ...]], StreamProtocol] + +STEPS: dict[str, Step] = { + "join": lambda a, g: Join()(a, _other(g), label="op"), + "merge_join": lambda a, g: MergeJoin()( + SelectDataColumns(["x"])(a), _colliding(g), label="op" + ), + "semijoin": lambda a, g: SemiJoin()(a, _other(g), label="op"), + "batch": lambda a, g: Batch(batch_size=2)(a, label="op"), + "group_by": lambda a, g: GroupBy(by=["g"])(a, label="op"), + "select_data": lambda a, g: SelectDataColumns(["x", "l"])(a, label="op"), + "drop_data": lambda a, g: DropDataColumns(["l"])(a, label="op"), + "select_tags": lambda a, g: SelectTagColumns(["g"])(a, label="op"), + "drop_tags": lambda a, g: DropTagColumns(["id"])(a, label="op"), + "select_columns": lambda a, g: SelectColumns(["g", "x"])(a, label="op"), + "drop_columns": lambda a, g: DropColumns(["id", "l"])(a, label="op"), + "map_tags": lambda a, g: MapTags({"id": "ident"})(a, label="op"), + "map_data": lambda a, g: MapData({"x": "xx"})(a, label="op"), + "filter": lambda a, g: PolarsFilter(constraints={"id": 1})(a, label="op"), + "drop_nulls": lambda a, g: DropNulls(["n"])(a, label="op"), + "drop_nulls_all": lambda a, g: DropNulls()(a, label="op"), + "index": lambda a, g: Index("l", 0)(a, label="op"), + "index_out": lambda a, g: Index("l", 0, out="first")(a, label="op"), + "pick": lambda a, g: Pick("m", "k")(a, label="op"), + "pick_out": lambda a, g: Pick("m", "k", out="k")(a, label="op"), + "function_pod": lambda a, g: _plus_one.pod(a, label="op"), +} + + +def _stream(step: str, pairs: int, groups: tuple[str, ...]) -> StreamProtocol: + return STEPS[step](_input(groups, pairs), groups) + + +def _job(step: str, pairs: int, groups: tuple[str, ...]) -> PipelineJob: + job = PipelineJob(name="itl1131", database=InMemoryArrowDatabase()) + with job: + _ = STEPS[step](_input(groups, pairs), groups) + return job + + +def _populated(table: pa.Table) -> pa.Schema: + """The reference schema: a table that has rows.""" + assert table.num_rows > 0 + return table.schema + + +def _assert_same_schema(table: pa.Table, expected: pa.Schema) -> None: + assert table.num_rows == 0 + assert table.schema.equals(expected), ( + f"zero-row columns {table.schema.names}\n" + f"populated columns {expected.names}\n" + f"zero-row schema:\n{table.schema}\npopulated schema:\n{expected}" + ) + + +PAIRS = [1, 2] + + +@pytest.mark.parametrize("pairs", PAIRS) +@pytest.mark.parametrize("step", list(STEPS)) +class TestPlainStream: + """``op(stream)`` with no rows has the schema of ``op(stream)`` with rows.""" + + def test_over_an_input_with_no_rows(self, step: str, pairs: int) -> None: + expected = _populated(_stream(step, pairs, WITH_ROWS).as_table(all_info=True)) + table = _stream(step, pairs, NO_ROWS).as_table(all_info=True) + _assert_same_schema(table, expected) + + def test_head_before_it_computes(self, step: str, pairs: int) -> None: + expected = _populated(_stream(step, pairs, WITH_ROWS).as_table(all_info=True)) + view = _stream(step, pairs, WITH_ROWS).head(all_info=True) + _assert_same_schema(view.as_table(), expected) + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +@pytest.mark.parametrize("pairs", PAIRS) +@pytest.mark.parametrize("step", list(STEPS)) +class TestJobNode: + """A job node with no rows has the schema of the same node after a run with rows.""" + + def _expected(self, step: str, pairs: int, orchestrator: str) -> pa.Schema: + job = _job(step, pairs, WITH_ROWS) + _ = job.run(orchestrator=ORCHESTRATORS[orchestrator]()) + return _populated(job.nodes["op"].as_table(all_info=True)) + + def test_before_a_run(self, step: str, pairs: int, orchestrator: str) -> None: + expected = self._expected(step, pairs, orchestrator) + job = _job(step, pairs, WITH_ROWS) + _assert_same_schema(job.nodes["op"].as_table(all_info=True), expected) + + def test_after_a_run_whose_input_had_no_rows( + self, step: str, pairs: int, orchestrator: str + ) -> None: + expected = self._expected(step, pairs, orchestrator) + job = _job(step, pairs, NO_ROWS) + _ = job.run(orchestrator=ORCHESTRATORS[orchestrator]()) + _assert_same_schema(job.nodes["op"].as_table(all_info=True), expected) From bcb299f01f32807f371ea9380fa31c4d095642b1 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 10:57:46 -0800 Subject: [PATCH 02/13] fix(operator_node): a run with no rows keeps the row order of a run with rows (ITL-1131) The row order of a node's rows (user tags, system tags, data, source info) has one owner, StaticOutputOperatorPod._in_row_order, used by _zero_row_input, the node's typed zero-row table and the result a sync or async run with no rows keeps (_zero_rows_in_row_order). A Join or MergeJoin node after such a run listed its data before its system tags. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/operator_node.py | 29 +++++---- .../core/operators/static_output_pod.py | 64 +++++++++++++++---- 2 files changed, 69 insertions(+), 24 deletions(-) diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index 2b98053e5..de8be5526 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -1255,16 +1255,11 @@ def _typed_zero_row_table(self) -> "pa.Table | None": # table is read. operator = cast(StaticOutputOperatorPod, self._operator) result = operator.zero_row_result(*inputs) - table = result.as_table(columns=ZERO_ROW_INPUT_COLUMNS) - # The column order of the node's rows (``_rows_to_table``): user tags, system - # tags, data, source info; an operator's own table may order them otherwise - # (a join lists its data before its system tags). - tag_keys, data_keys = result.keys() - names = table.column_names - system_tags = [n for n in names if n.startswith(constants.SYSTEM_TAG_PREFIX)] - source_info = [n for n in names if n.startswith(constants.SOURCE_PREFIX)] - data = [n for n in data_keys if n in names] - return table.select([*tag_keys, *system_tags, *data, *source_info]) + # In the column order of the node's rows; an operator's own table may order + # them otherwise (a join lists its data before its system tags). + return StaticOutputOperatorPod._in_row_order( + result.as_table(columns=ZERO_ROW_INPUT_COLUMNS), result.keys()[0] + ) def _make_empty_table(self) -> "pa.Table": """Build the zero-row table this node reads as when it holds no rows. @@ -1430,7 +1425,11 @@ def execute( output ) else: - self._cached_output_stream = result_stream + # The result with no rows, in the column order of a run with rows + # (ITL-1131). + self._cached_output_stream = ( + StaticOutputOperatorPod._zero_rows_in_row_order(result_stream) + ) self._update_modified_time() @@ -1908,9 +1907,13 @@ async def forward() -> None: for tap, upstream in zip(taps, self._input_streams, strict=True) ) ) - # Computed, as execute() computes it, so the node reads as run. + # Computed, as execute() computes it, so the node reads as run, and + # kept in the column order of a run with rows, as execute() keeps it + # (ITL-1131). _ = list(empty_result.iter_data()) - self._cached_output_stream = empty_result + self._cached_output_stream = ( + StaticOutputOperatorPod._zero_rows_in_row_order(empty_result) + ) self._update_modified_time() diff --git a/src/orcapod/core/operators/static_output_pod.py b/src/orcapod/core/operators/static_output_pod.py index 467ad80c6..6c16fbc24 100644 --- a/src/orcapod/core/operators/static_output_pod.py +++ b/src/orcapod/core/operators/static_output_pod.py @@ -259,6 +259,57 @@ def _rows_to_table( full_table = arrow_utils.hstack_tables(combined_tags, combined_data) return full_table, tuple(rows[0][0].keys()) + @staticmethod + def _in_row_order(table: pa.Table, tag_keys: Sequence[str]) -> pa.Table: + """Return ``table`` with its columns in the order of a table built from rows. + + The one statement of that order (ITL-1131): user tags, system tags, data, + source info, as ``_rows_to_table`` builds it (each row's tag table with its + system tags, then its data table with its source info). Data columns keep their + order in ``table``. Every table a node or an operator input holds with no rows + takes this order, so it equals the table a run with rows gives. + + Args: + table: A table of user tags, system tags, data and source info. + tag_keys: The user tag columns, in order. + + Returns: + The same columns, reordered; no column is added, dropped or changed. + """ + from orcapod.system_constants import constants + + names = table.column_names + system_tags = [n for n in names if n.startswith(constants.SYSTEM_TAG_PREFIX)] + source_info = [n for n in names if n.startswith(constants.SOURCE_PREFIX)] + placed = {*tag_keys, *system_tags, *source_info} + data = [n for n in names if n not in placed] + return table.select([*tag_keys, *system_tags, *data, *source_info]) + + @staticmethod + def _zero_rows_in_row_order(result: StreamProtocol) -> StreamProtocol: + """Return an operator's result with no rows as a run with rows would keep it. + + The zero-row counterpart of ``_materialize_to_stream``: ``result``'s zero-row + table (user tags, system tags, data and source info, with its Arrow types and + nullability) in the order of a table built from rows (``_in_row_order``). An + operator's own table may order its columns otherwise (a join lists its data + before its system tags), so a node that keeps the result of a run with no rows + keeps this, and reads with the columns of a run with rows (ITL-1131). + + Args: + result: An operator's result that has no rows. + + Returns: + A stream over the reordered zero-row table, in ``result``'s data context. + """ + tag_keys = result.keys()[0] + table = StaticOutputOperatorPod._in_row_order( + result.as_table(columns=ZERO_ROW_INPUT_COLUMNS), tag_keys + ) + return ArrowTableStream( + table, tag_columns=tag_keys, data_context=result.data_context_key + ) + @staticmethod def _flag_source_info_nullability(table: pa.Table) -> pa.Table: """Declare each ``_source_*`` field nullable only if some row's token is ``None``. @@ -302,20 +353,11 @@ def _zero_row_input(upstream: StreamProtocol) -> StreamProtocol | None: types without rows; the caller then builds its input from the Python output schema. """ - from orcapod.system_constants import constants - table = typed_zero_row_input(upstream) if table is None: return None - # The column order of a row-built table (``_rows_to_table``): user tags, - # system tags, data, source info. - tag_keys, data_keys = upstream.keys() - names = table.column_names - system_tags = [n for n in names if n.startswith(constants.SYSTEM_TAG_PREFIX)] - source_info = [n for n in names if n.startswith(constants.SOURCE_PREFIX)] - # A raw stream lists its meta columns as data; the zero-row table has none. - data = [n for n in data_keys if not n.startswith(constants.META_PREFIX)] - table = table.select([*tag_keys, *system_tags, *data, *source_info]) + tag_keys = upstream.keys()[0] + table = StaticOutputOperatorPod._in_row_order(table, tag_keys) return StaticOutputOperatorPod._as_upstream_stream( StaticOutputOperatorPod._flag_source_info_nullability(table), tag_keys, From 8ffb1a35e3f8e975af66813bd45c4cb279c33e89 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 11:04:17 -0800 Subject: [PATCH 03/13] fix(index, pick): an empty result keeps the system tags and source-info flags (ITL-1131) Index and Pick built the result of a run that kept no row from the input's table without its system-tag columns (Pick also without its source info, which then read as nullable, and without the token of an out= column). They now build it from the input with system tags and source info, as a row-built table (StaticOutputOperatorPod._as_row_built_table: row order, source-info flags). Adds the ITL-1133 reproduction script (one node table, two column orders). Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/operators/index.py | 6 +- src/orcapod/core/operators/pick.py | 17 ++++- .../core/operators/static_output_pod.py | 23 ++++++- .../scripts/itl_1133_store_column_order.py | 68 +++++++++++++++++++ .../test_zero_row_column_order.py | 7 +- 5 files changed, 112 insertions(+), 9 deletions(-) create mode 100644 superpowers/scripts/itl_1133_store_column_order.py diff --git a/src/orcapod/core/operators/index.py b/src/orcapod/core/operators/index.py index 06956138e..8ec72009c 100644 --- a/src/orcapod/core/operators/index.py +++ b/src/orcapod/core/operators/index.py @@ -371,8 +371,10 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: # The source-info columns are part of it: the projected column's token # type is the input's with one list level removed, as for a non-empty # result (ITL-1046), and every other token column keeps its input type. + # The system tags are part of it too: a result with rows keeps each row's + # (ITL-1131). input_schema = arrow_utils.normalize_schema_to_large_types( - stream.as_table(columns={"source": True}).schema + stream.as_table(columns={"source": True, "system_tags": True}).schema ) element_field = pa.field( self.out or self.column, @@ -404,7 +406,7 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: output_schema = input_schema.append(element_field) # pyright: ignore[reportUnknownMemberType] # stubs leave Field unparametrized output_schema = output_schema.append(token_field) # pyright: ignore[reportUnknownMemberType] # stubs leave Field unparametrized return ArrowTableStream( - output_schema.empty_table(), + self._as_row_built_table(output_schema.empty_table(), tag_columns), tag_columns=tag_columns, data_context=stream.data_context, ) diff --git a/src/orcapod/core/operators/pick.py b/src/orcapod/core/operators/pick.py index ace2c3622..a0a43ecac 100644 --- a/src/orcapod/core/operators/pick.py +++ b/src/orcapod/core/operators/pick.py @@ -12,6 +12,7 @@ from orcapod.core.operators.base import UnaryOperator from orcapod.core.streams.arrow_table_stream import ArrowTableStream from orcapod.errors import InputValidationError +from orcapod.system_constants import constants from orcapod.protocols.core_protocols import DataProtocol, StreamProtocol, TagProtocol from orcapod.types import ColumnConfig, ContentHash, Schema from orcapod.utils import arrow_utils, schema_utils @@ -373,8 +374,10 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: # The schema is the input's Arrow schema (large-normalized, as for # a non-empty result) with only the projected column changed. tag_columns, _ = stream.keys() + # The system tags and source info are part of it: a result with rows keeps + # each row's (ITL-1131). input_schema = arrow_utils.normalize_schema_to_large_types( - stream.as_table().schema + stream.as_table(columns={"source": True, "system_tags": True}).schema ) value_field = pa.field( self.out or self.column, @@ -383,13 +386,23 @@ def unary_static_process(self, stream: StreamProtocol) -> StreamProtocol: ), nullable=self._output_nullable(), ) + # A picked value's token is a string (``_process_one``), typed by the + # source-info type rule. + token_name = f"{constants.SOURCE_PREFIX}{self.out or self.column}" + token_field = pa.field( + token_name, arrow_utils.source_info_arrow_type(token_name) + ) if self.out is None: position = input_schema.get_field_index(self.column) output_schema = input_schema.set(position, value_field) # pyright: ignore[reportUnknownMemberType] # stubs leave Field unparametrized + output_schema = output_schema.set( + output_schema.get_field_index(token_name), token_field + ) # pyright: ignore[reportUnknownMemberType] # stubs leave Field unparametrized else: output_schema = input_schema.append(value_field) # pyright: ignore[reportUnknownMemberType] # stubs leave Field unparametrized + output_schema = output_schema.append(token_field) # pyright: ignore[reportUnknownMemberType] # stubs leave Field unparametrized return ArrowTableStream( - output_schema.empty_table(), + self._as_row_built_table(output_schema.empty_table(), tag_columns), tag_columns=tag_columns, data_context=stream.data_context, ) diff --git a/src/orcapod/core/operators/static_output_pod.py b/src/orcapod/core/operators/static_output_pod.py index 6c16fbc24..65be4a630 100644 --- a/src/orcapod/core/operators/static_output_pod.py +++ b/src/orcapod/core/operators/static_output_pod.py @@ -285,6 +285,26 @@ def _in_row_order(table: pa.Table, tag_keys: Sequence[str]) -> pa.Table: data = [n for n in names if n not in placed] return table.select([*tag_keys, *system_tags, *data, *source_info]) + @staticmethod + def _as_row_built_table(table: pa.Table, tag_keys: Sequence[str]) -> pa.Table: + """Return ``table`` as ``_rows_to_table`` would build it from the same rows. + + Its columns in row order (``_in_row_order``) and each source-info field flagged + nullable only when some row's token is ``None`` + (``_flag_source_info_nullability``). For a table with no rows: what a result + built from rows has, with no row (ITL-1131). + + Args: + table: A table of user tags, system tags, data and source info. + tag_keys: The user tag columns, in order. + + Returns: + The same arrays, reordered, under the new source-info field flags. + """ + return StaticOutputOperatorPod._flag_source_info_nullability( + StaticOutputOperatorPod._in_row_order(table, tag_keys) + ) + @staticmethod def _zero_rows_in_row_order(result: StreamProtocol) -> StreamProtocol: """Return an operator's result with no rows as a run with rows would keep it. @@ -357,9 +377,8 @@ def _zero_row_input(upstream: StreamProtocol) -> StreamProtocol | None: if table is None: return None tag_keys = upstream.keys()[0] - table = StaticOutputOperatorPod._in_row_order(table, tag_keys) return StaticOutputOperatorPod._as_upstream_stream( - StaticOutputOperatorPod._flag_source_info_nullability(table), + StaticOutputOperatorPod._as_row_built_table(table, tag_keys), tag_keys, upstream, ) diff --git a/superpowers/scripts/itl_1133_store_column_order.py b/superpowers/scripts/itl_1133_store_column_order.py new file mode 100644 index 000000000..13d07d515 --- /dev/null +++ b/superpowers/scripts/itl_1133_store_column_order.py @@ -0,0 +1,68 @@ +"""ITL-1133 reproduction (found during ITL-1131): one operator node table, two column orders. + +A job run keeps and stores an operator node's rows in row order (user tags, system tags, +data, source info: ``_rows_to_table``); a direct ``node.run()`` (``_compute_and_store``) +in ``CacheMode.LOG`` stores them in the operator's own order (a join: user tags, data, +system tags). Appending the rows of one to a table holding the other fails with +``StoreWriteError`` (schema mismatch) in the in-memory store, and in a Delta store while the +first write is a pending batch (when the store is not read in between, the in-memory store +fails on the next read instead). A Delta store flushed between the two writes appends by +name and does not fail. With identical rows nothing is appended +(``skip_duplicates``), so each step here has new rows under the same source ids (same +node path and content hash). + +Run: ``uv run python superpowers/scripts/itl_1133_store_column_order.py`` +""" + +from __future__ import annotations + +import tempfile + +import pyarrow as pa + +from orcapod.core.nodes.operator_node import CacheMode +from orcapod.core.operators import Join +from orcapod.core.sources import ArrowTableSource +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.pipeline.job import PipelineJob + + +def _source(name: str, column: str, ids: list[int]) -> ArrowTableSource: + table = pa.table( + {"id": pa.array(ids, pa.int64()), column: pa.array(ids, pa.int64())} + ) + return ArrowTableSource(table, tag_columns=["id"], source_id=name) + + +def _job(ids: list[int]) -> PipelineJob: + job = PipelineJob(name="itl1133", database=InMemoryArrowDatabase()) + with job: + _ = Join()(_source("a", "x", ids), _source("b", "z", ids), label="joined") + return job + + +def _attempt(store_kind: str, first: str, second: str) -> str: + store = ( + InMemoryArrowDatabase() + if store_kind == "in-memory" + else DeltaTableDatabase(tempfile.mkdtemp()) + ) + try: + for step, ids in ((first, [1, 2]), (second, [3, 4])): + job = _job(ids) + node = job.nodes["joined"] + node.attach_databases(pipeline_database=store, cache_mode=CacheMode.LOG) + if step == "job": + _ = job.run() # sync orchestrator -> execute(): row order + else: + _ = node.run() # _compute_and_store: the operator's order + _ = node.get_all_records() + return "OK" + except Exception as exc: # noqa: BLE001 # a reproduction reports any failure + return f"{type(exc).__name__}: {str(exc).splitlines()[0][:120]}" + + +if __name__ == "__main__": + for kind in ("in-memory", "delta (no flush)"): + for first, second in (("job", "node"), ("node", "job")): + print(f"{kind:17s} {first} then {second}: {_attempt(kind, first, second)}") diff --git a/tests/test_pipeline/test_zero_row_column_order.py b/tests/test_pipeline/test_zero_row_column_order.py index 1c782748e..cd5516b29 100644 --- a/tests/test_pipeline/test_zero_row_column_order.py +++ b/tests/test_pipeline/test_zero_row_column_order.py @@ -16,6 +16,7 @@ from __future__ import annotations from collections.abc import Callable +from typing import Any import pyarrow as pa import pytest @@ -58,7 +59,7 @@ """Groups of the sources' rows: the filter on ``g1`` keeps none.""" -def _dict_column() -> pa.Array: +def _dict_column() -> "pa.Array[Any]": """A ``dict[str, int]`` column in the type the data context stores it with.""" converter = contexts.resolve_context(None).type_converter table = converter.python_dicts_to_arrow_table( @@ -67,7 +68,7 @@ def _dict_column() -> pa.Array: return table.column("m").combine_chunks() -def _table(groups: tuple[str, ...], data: dict[str, pa.Array]) -> pa.Table: +def _table(groups: tuple[str, ...], data: dict[str, "pa.Array[Any]"]) -> pa.Table: """Tags ``g`` and ``id`` and ``data``; every column non-nullable except ``n``.""" table = pa.table( { @@ -84,7 +85,7 @@ def _table(groups: tuple[str, ...], data: dict[str, pa.Array]) -> pa.Table: def _source( - name: str, groups: tuple[str, ...], data: dict[str, pa.Array] + name: str, groups: tuple[str, ...], data: dict[str, "pa.Array[Any]"] ) -> StreamProtocol: """A source of ``data`` behind a filter keeping group ``g1``.""" source = ArrowTableSource( From 91882e8fbac067b016c20e07b520d0051d5e5f93 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 11:04:40 -0800 Subject: [PATCH 04/13] docs: zero-row column order rule, changelog fragment and design issue ZR2 (ITL-1131) Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 7 +++++++ CLAUDE.md | 7 +++++++ changelog.d/ITL-1131.md | 17 +++++++++++++++++ design_issues/ITL-1131-ZR2.md | 30 ++++++++++++++++++++++++++++++ 4 files changed, 61 insertions(+) create mode 100644 changelog.d/ITL-1131.md create mode 100644 design_issues/ITL-1131-ZR2.md diff --git a/.zed/rules b/.zed/rules index b801c0552..fa1c05b6b 100644 --- a/.zed/rules +++ b/.zed/rules @@ -735,6 +735,13 @@ data. The rules: rows is built in one place, StreamBase._head_without_rows, from the stream's _zero_row_table(config), which equals as_table(columns=config) with no rows; a kind whose as_table shapes its table its own way overrides _zero_row_table with that shaping. + The zero-row table has the populated table's columns in the populated order too (ITL-1131). + A table built from rows (and a node's or operator input's table with no rows) is in row + order, user tags, system tags, data, source info, stated once in + StaticOutputOperatorPod._in_row_order; a node that keeps a run with no rows keeps + _zero_rows_in_row_order, and an operator's own empty result built from its input + goes through _as_row_built_table. Never change the order of a populated table: + stored tables depend on it. - Type strictness (developer, November 10, 2026): Orcapod is type strict. It never expands or converts a type implicitly; the only exception is normalization to large variants. Inputs that disagree on a column's type are an error naming the column and both types, on diff --git a/CLAUDE.md b/CLAUDE.md index c2c43bbf3..c0a70c400 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -777,6 +777,13 @@ data. The rules: rows is built in one place, `StreamBase._head_without_rows`, from the stream's `_zero_row_table(config)`, which equals `as_table(columns=config)` with no rows; a kind whose `as_table` shapes its table its own way overrides `_zero_row_table` with that shaping. + The zero-row table has the populated table's columns in the populated order too (ITL-1131). + A table built from rows (and a node's or operator input's table with no rows) is in row + order, user tags, system tags, data, source info, stated once in + `StaticOutputOperatorPod._in_row_order`; a node that keeps a run with no rows keeps + `_zero_rows_in_row_order`, and an operator's own empty result built from its input + goes through `_as_row_built_table`. Never change the order of a populated table: + stored tables depend on it. - **Type strictness** (developer, November 10, 2026): Orcapod is type strict. It never expands or converts a type implicitly; the only exception is normalization to large variants. Inputs that disagree on a column's type are an error naming the column and both types, on diff --git a/changelog.d/ITL-1131.md b/changelog.d/ITL-1131.md new file mode 100644 index 000000000..210d240a6 --- /dev/null +++ b/changelog.d/ITL-1131.md @@ -0,0 +1,17 @@ +### Fixed + +- A table with no rows has its populated table's columns in the same order (ITL-1131). + A `Join` or `MergeJoin` job node after a run whose input had no rows (sync and async) + listed its data columns before its system tags, where the same node after a run with rows + lists them after. `Index` and `Pick` over an input with no rows, or whose rows were all + skipped, dropped the system-tag columns, and `Pick` also gave its source-info columns as + nullable and lost the token column of an `out=` column. Every operator's zero-row table now + equals its populated table in column names, order, Arrow types and nullability. No + populated table, stored table or hash changes. + +### Known limitations + +- An operator node run on its own (`node.run()` on a node built over streams outside a job) + keeps and stores its rows in the operator's own column order (a join lists its data before + its system tags), while a job run keeps and stores them in row order, and the node's table + before a run is in row order. A store that holds both fails to append the second (ITL-1133). diff --git a/design_issues/ITL-1131-ZR2.md b/design_issues/ITL-1131-ZR2.md new file mode 100644 index 000000000..0c42c3722 --- /dev/null +++ b/design_issues/ITL-1131-ZR2.md @@ -0,0 +1,30 @@ +## Cross-cutting + +### ZR2 — A zero-row table had a different column order from its populated table +**Status:** resolved +**Severity:** medium +**Issue:** ITL-1131 + +ITL-912 (ZR1) gave every table with no rows the Arrow types and nullability of the rows, but +not always their column order or all their columns. A `Join` or `MergeJoin` job node after a +run whose input had no rows (sync `OperatorJobNode.execute()` and the async empty branch) +kept the operator's own result, which lists data before system tags, while a run with rows +keeps `_materialize_to_stream` (row order: user tags, system tags, data, source info). +`Index` and `Pick` built their empty result from the input's table without system tags +(`Pick` also without source info, so its source-info columns came out nullable, and without +the token of an `out=` column). The row order was restated in two places +(`StaticOutputOperatorPod._zero_row_input`, `OperatorJobNode._typed_zero_row_table`). + +**Fix:** the row order has one owner, `StaticOutputOperatorPod._in_row_order`, used by +`_zero_row_input`, `OperatorJobNode._typed_zero_row_table`, `_zero_rows_in_row_order` (what a +node keeps after a sync or async run with no rows) and `_as_row_built_table` (row order plus +source-info flags; `Index` and `Pick` build their empty result from the input with system +tags and source info through it). `tests/test_pipeline/test_zero_row_column_order.py` checks +every operator and a function pod, plain stream and job node, sync and async, before a run +and after a zero-input run, with one and two system-tag pairs. No populated or stored order +changed. + +Out of scope (ITL-1133): an operator node run on its own (`_compute_and_store`) keeps and +stores the operator's own order while a job run uses row order, so one node has two populated +orders; a store holding both fails to append. Reproduction: +`superpowers/scripts/itl_1133_store_column_order.py`. From a5bcae085cf9dae7543ed93f86a59a82b86e2057 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 11:11:11 -0800 Subject: [PATCH 05/13] test(operator_node): failing tests for one column order on every node path (ITL-1133) Co-Authored-By: Claude Opus 5.5 --- .../test_zero_row_column_order.py | 113 +++++++++++++++++- 1 file changed, 112 insertions(+), 1 deletion(-) diff --git a/tests/test_pipeline/test_zero_row_column_order.py b/tests/test_pipeline/test_zero_row_column_order.py index cd5516b29..e3c15f9cc 100644 --- a/tests/test_pipeline/test_zero_row_column_order.py +++ b/tests/test_pipeline/test_zero_row_column_order.py @@ -11,6 +11,12 @@ same job node after a run with rows by the same orchestrator. Inputs carry one or two system-tag pairs (a source, or a join of two sources). + +An operator node also has one populated order on every path (ITL-1133): the row order of a job +run (user tags, system tags, data, source info). A node run on its own (``node.run()`` over +streams outside a job, without a store, in ``LOG`` and read back in ``REPLAY``) keeps and +stores that order, so a store written by a job run and by a node run on its own holds one +order and accepts both. """ from __future__ import annotations @@ -42,11 +48,13 @@ SemiJoin, ) from orcapod.core.sources import ArrowTableSource -from orcapod.databases import InMemoryArrowDatabase +from orcapod.core.nodes import OperatorJobNode +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator from orcapod.pipeline.job import PipelineJob from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator from orcapod.protocols.core_protocols import StreamProtocol +from orcapod.types import CacheMode ORCHESTRATORS = { "sync": SyncPipelineOrchestrator, @@ -227,3 +235,106 @@ def test_after_a_run_whose_input_had_no_rows( job = _job(step, pairs, NO_ROWS) _ = job.run(orchestrator=ORCHESTRATORS[orchestrator]()) _assert_same_schema(job.nodes["op"].as_table(all_info=True), expected) + + +OPERATOR_STEPS = [step for step in STEPS if step != "function_pod"] + + +def _node( + step: str, pairs: int, groups: tuple[str, ...], **kwargs: Any +) -> OperatorJobNode: + """An operator node of ``step`` over the plain streams the step reads, outside a job.""" + stream = _stream(step, pairs, groups) + return OperatorJobNode(stream.producer, stream.upstreams, **kwargs) + + +@pytest.mark.parametrize("pairs", PAIRS) +@pytest.mark.parametrize("step", OPERATOR_STEPS) +class TestNodeRunOnItsOwn: + """A node run on its own has the schema a job run gives it, on every path (ITL-1133).""" + + def _expected(self, step: str, pairs: int) -> pa.Schema: + job = _job(step, pairs, WITH_ROWS) + _ = job.run(orchestrator=SyncPipelineOrchestrator()) + return _populated(job.nodes["op"].as_table(all_info=True)) + + def test_run_without_a_store(self, step: str, pairs: int) -> None: + node = _node(step, pairs, WITH_ROWS) + _ = node.run() + table = node.as_table(all_info=True) + assert table.num_rows > 0 + assert table.schema.equals(self._expected(step, pairs)) + + def test_before_a_run(self, step: str, pairs: int) -> None: + node = _node(step, pairs, WITH_ROWS) + _assert_same_schema(node.as_table(all_info=True), self._expected(step, pairs)) + + def test_run_whose_input_had_no_rows(self, step: str, pairs: int) -> None: + node = _node(step, pairs, NO_ROWS) + _ = node.run() + _assert_same_schema(node.as_table(all_info=True), self._expected(step, pairs)) + + def test_log_run_and_replay(self, step: str, pairs: int) -> None: + expected = self._expected(step, pairs) + store = InMemoryArrowDatabase() + node = _node( + step, pairs, WITH_ROWS, pipeline_database=store, cache_mode=CacheMode.LOG + ) + _ = node.run() + assert node.as_table(all_info=True).schema.equals(expected) + replay = _node( + step, pairs, WITH_ROWS, pipeline_database=store, cache_mode=CacheMode.REPLAY + ) + table = replay.as_table(all_info=True) + assert table.num_rows > 0 + assert table.schema.equals(expected) + + +def _join_job(ids: list[int]) -> PipelineJob: + """A job joining two sources whose rows have ``ids``; same source ids every time.""" + + def source(name: str, column: str) -> ArrowTableSource: + table = pa.table( + {"id": pa.array(ids, pa.int64()), column: pa.array(ids, pa.int64())} + ) + return ArrowTableSource(table, tag_columns=["id"], source_id=name) + + job = PipelineJob(name="itl1133", database=InMemoryArrowDatabase()) + with job: + _ = Join()(source("a", "x"), source("b", "z"), label="joined") + return job + + +STORES = ["in_memory", "delta", "delta_flushed"] + + +@pytest.mark.parametrize("store_kind", STORES) +@pytest.mark.parametrize("order", [("job", "node"), ("node", "job")]) +def test_a_job_run_and_a_node_run_share_one_stored_table( + store_kind: str, order: tuple[str, str], tmp_path: Any +) -> None: + """A job run and a node run on its own store one column order in one table (ITL-1133). + + Each run has new rows under the same source ids, so both write the same node table. + """ + store = ( + InMemoryArrowDatabase() + if store_kind == "in_memory" + else DeltaTableDatabase(tmp_path) + ) + schemas = [] + for how, ids in zip(order, ([1, 2], [3, 4]), strict=True): + job = _join_job(ids) + node = job.nodes["joined"] + node.attach_databases(pipeline_database=store, cache_mode=CacheMode.LOG) + if how == "job": + _ = job.run(orchestrator=SyncPipelineOrchestrator()) + else: + _ = node.run() + schemas.append(node.as_table(all_info=True).schema) + if store_kind == "delta_flushed": + store.flush() + assert schemas[0].equals(schemas[1]) + stored = node.get_all_records(all_info=True) + assert stored is not None + assert stored.num_rows == 4 From 3960c889c83c715e28de64bbaaacf49784893845 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 11:20:15 -0800 Subject: [PATCH 06/13] fix(operator_node): a node run on its own keeps and stores the row order of a job run (ITL-1133) OperatorJobNode.run() and _compute_and_store kept the operator's own result, whose columns follow the operator (a join lists its data before its system tags), and stored it in that order, while a job run keeps and stores the row order. A store holding both refused the second append. Every way a node runs now keeps StaticOutputOperatorPod._kept_output: the rows as _materialize_to_stream builds them, or the zero-row result in the same order. Record ids do not depend on column order, so no hash moves. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/operator_node.py | 33 +++++++++---------- .../core/operators/static_output_pod.py | 24 ++++++++++++++ 2 files changed, 40 insertions(+), 17 deletions(-) diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index de8be5526..3a90a7619 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -1419,17 +1419,10 @@ def execute( # Materialize output = list(result_stream.iter_data()) - # Cache - if output: - self._cached_output_stream = StaticOutputOperatorPod._materialize_to_stream( - output - ) - else: - # The result with no rows, in the column order of a run with rows - # (ITL-1131). - self._cached_output_stream = ( - StaticOutputOperatorPod._zero_rows_in_row_order(result_stream) - ) + # Cache, in row order with or without rows (ITL-1131, ITL-1133) + self._cached_output_stream = StaticOutputOperatorPod._kept_output( + result_stream, output + ) self._update_modified_time() @@ -1444,11 +1437,19 @@ def execute( ctx_obs.on_node_end(node_label, node_hash) return output + def _compute_output(self) -> StreamProtocol: + """Run the operator over the node's inputs and return what the node keeps. + + The rows in row order, as a job run keeps them (``_kept_output``), so a node + run on its own has the columns, in memory and stored, of the same node in a + job (ITL-1133). The rows are computed here, not when first read. + """ + result = self._operator.process(*self._input_streams) + return StaticOutputOperatorPod._kept_output(result, list(result.iter_data())) + def _compute_and_store(self) -> None: """Compute operator output, optionally store in DB.""" - self._cached_output_stream = self._operator.process( - *self._input_streams, - ) + self._cached_output_stream = self._compute_output() if self._cache_mode == CacheMode.OFF: self._update_modified_time() @@ -1524,9 +1525,7 @@ def run(self) -> StreamView: self._compute_and_store() computed = True else: - self._cached_output_stream = self._operator.process( - *self._input_streams, - ) + self._cached_output_stream = self._compute_output() self._update_modified_time() computed = True table = self.as_table() diff --git a/src/orcapod/core/operators/static_output_pod.py b/src/orcapod/core/operators/static_output_pod.py index 65be4a630..4ddb6de4d 100644 --- a/src/orcapod/core/operators/static_output_pod.py +++ b/src/orcapod/core/operators/static_output_pod.py @@ -330,6 +330,30 @@ def _zero_rows_in_row_order(result: StreamProtocol) -> StreamProtocol: table, tag_columns=tag_keys, data_context=result.data_context_key ) + @staticmethod + def _kept_output( + result: StreamProtocol, + rows: Sequence[tuple[TagProtocol, DataProtocol]], + ) -> StreamProtocol: + """Return what an operator node keeps, and stores, of a run of its operator. + + The rows of the run as ``_materialize_to_stream`` builds them, or, with no row, + the result with no rows in the same order (``_zero_rows_in_row_order``). Every + way a node runs its operator (a job run, sync or async, and ``node.run()`` on + its own) keeps this, so a node has one column order, the row order, in memory + and in its stored table (ITL-1133). + + Args: + result: The operator's result of the run. + rows: ``result``'s rows, in order. + + Returns: + A stream over the rows, in row order. + """ + if rows: + return StaticOutputOperatorPod._materialize_to_stream(rows) + return StaticOutputOperatorPod._zero_rows_in_row_order(result) + @staticmethod def _flag_source_info_nullability(table: pa.Table) -> pa.Table: """Declare each ``_source_*`` field nullable only if some row's token is ``None``. From ef9d6354405d6af8d6523ff883e5e4f88575a48a Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 11:20:54 -0800 Subject: [PATCH 07/13] test(operator_node): failing tests: an operator cache stored in operator order reads in row order (ITL-1133) Co-Authored-By: Claude Opus 5.5 --- .../test_zero_row_column_order.py | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/tests/test_pipeline/test_zero_row_column_order.py b/tests/test_pipeline/test_zero_row_column_order.py index e3c15f9cc..11b9ee37f 100644 --- a/tests/test_pipeline/test_zero_row_column_order.py +++ b/tests/test_pipeline/test_zero_row_column_order.py @@ -338,3 +338,82 @@ def test_a_job_run_and_a_node_run_share_one_stored_table( stored = node.get_all_records(all_info=True) assert stored is not None assert stored.num_rows == 4 + + +def _write_in_operator_order(source_dir: Any, target_dir: Any) -> tuple[str, ...]: + """Copy the one operator cache table of ``source_dir`` as the old writer wrote it. + + Test-only stand-in for an operator cache written before ITL-1133 by a node run on its + own: the same stored rows, with the user columns in a join's own order (user tags, + data, system tags, source info) and the bookkeeping (``__``) columns where they were. + """ + source = DeltaTableDatabase(source_dir) + (path,) = [p for p in source.list_table_paths() if p[-1] == "odb_v1"] + rows = source.get_all_records( + path, record_id_column=OperatorJobNode.HASH_COLUMN_NAME + ) + assert rows is not None + names = rows.column_names + user = [n for n in names if not n.startswith("__")] + system_tags = [n for n in user if n.startswith("_tag")] + source_info = [n for n in user if n.startswith("_source_")] + tags = [n for n in user if n == "id"] + data = [n for n in user if n not in {*tags, *system_tags, *source_info}] + old_order = iter([*tags, *data, *system_tags, *source_info]) + reordered = [n if n.startswith("__") else next(old_order) for n in names] + target = DeltaTableDatabase(target_dir) + target.add_records( + path, + rows.select(reordered), + record_id_column=OperatorJobNode.HASH_COLUMN_NAME, + ) + target.flush() + return path + + +class TestOperatorCacheWrittenInOperatorOrder: + """A stored operator cache in a join's own order reads in row order (ITL-1133, B).""" + + def _expected(self) -> pa.Schema: + job = _join_job([1, 2]) + _ = job.run(orchestrator=SyncPipelineOrchestrator()) + return _populated(job.nodes["joined"].as_table(all_info=True)) + + def _old_table(self, tmp_path: Any) -> Any: + written = tmp_path / "written" + store = DeltaTableDatabase(written) + job = _join_job([1, 2]) + node = job.nodes["joined"] + node.attach_databases(pipeline_database=store, cache_mode=CacheMode.LOG) + _ = job.run(orchestrator=SyncPipelineOrchestrator()) + store.flush() + old = tmp_path / "old" + _ = _write_in_operator_order(written, old) + return old + + def _replay(self, store: DeltaTableDatabase, ids: list[int]) -> OperatorJobNode: + node = _join_job(ids).nodes["joined"] + node.attach_databases(pipeline_database=store, cache_mode=CacheMode.REPLAY) + return node + + def test_reads_in_row_order(self, tmp_path: Any) -> None: + old = self._old_table(tmp_path) + table = self._replay(DeltaTableDatabase(old), [1, 2]).as_table(all_info=True) + assert table.num_rows == 2 + assert table.schema.equals(self._expected()) + + def test_accepts_a_row_order_append(self, tmp_path: Any) -> None: + old = self._old_table(tmp_path) + store = DeltaTableDatabase(old) + job = _join_job([3, 4]) + node = job.nodes["joined"] + node.attach_databases(pipeline_database=store, cache_mode=CacheMode.LOG) + _ = node.run() + store.flush() + stored = node.get_all_records(all_info=True) + assert stored is not None + assert stored.num_rows == 4 + # Both runs bind the same source ids, so the node's stored rows are all four. + table = self._replay(DeltaTableDatabase(old), [3, 4]).as_table(all_info=True) + assert table.num_rows == 4 + assert table.schema.equals(self._expected()) From 214c1f70f928adef9dd0627fcbd5b18d2dce20d1 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 11:21:30 -0800 Subject: [PATCH 08/13] fix(operator_node): read a stored operator cache in row order (ITL-1133, option B) An operator cache written by a node run on its own before ITL-1133 holds its operator's column order. The one read path (_read_current_context_records) puts the user columns in row order, keeping the bookkeeping columns in place, so such a table reads with the columns of every other path and accepts row-order appends; a table in row order is returned unchanged. No layout bump. Its own commit, so the developer's choice between options A, B and C can drop or replace it. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/operator_node.py | 31 ++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index 3a90a7619..9940a067f 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -1225,7 +1225,36 @@ def _read_current_context_records(self) -> "pa.Table | None": self._pipeline_database.base_path + self._stored_table_path, stored_rows - records.num_rows, ) - return records.drop([constants.DATA_CONTEXT_COL]) + return self._in_row_order_on_read(records.drop([constants.DATA_CONTEXT_COL])) + + def _in_row_order_on_read(self, records: "pa.Table") -> "pa.Table": + """Put a stored table's user columns in row order; bookkeeping stays in place. + + A node run on its own stored its rows in its operator's column order before + ITL-1133 (a join lists its data before its system tags); every writer now + stores the row order (``StaticOutputOperatorPod._in_row_order``). Read in row + order, such a table gives the columns of every other path, and a table in row + order is returned unchanged. Only the order changes: no column is added, + dropped, renamed or retyped, and the bookkeeping columns (``__``, and + ``HASH_COLUMN_NAME``) keep their positions. + """ + names = records.column_names + + def bookkeeping(name: str) -> bool: + return ( + name.startswith(constants.META_PREFIX) or name == self.HASH_COLUMN_NAME + ) + + user = [n for n in names if not bookkeeping(n)] + ordered = iter( + StaticOutputOperatorPod._in_row_order( + records.select(user), self.keys()[0] + ).column_names + ) + order = [n if bookkeeping(n) else next(ordered) for n in names] + if order == names: + return records + return records.select(order) def _typed_zero_row_table(self) -> "pa.Table | None": """This node's zero-row table: its operator applied to zero-row inputs. From b01a6d6620b4609814ed7a86962eea1904bfd2e0 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 11:25:07 -0800 Subject: [PATCH 09/13] docs: one operator-node column order (ITL-1133): changelog, design issue ZR3, rule Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 6 +++-- CLAUDE.md | 6 +++-- changelog.d/ITL-1131.md | 18 ++++++------- design_issues/ITL-1131-ZR2.md | 6 ++--- design_issues/ITL-1133-ZR3.md | 27 +++++++++++++++++++ ...26-11-14-itl-1131-zero-row-column-order.md | 7 +++++ .../scripts/itl_1133_store_column_order.py | 2 ++ .../test_zero_row_column_order.py | 10 ++++--- 8 files changed, 62 insertions(+), 20 deletions(-) create mode 100644 design_issues/ITL-1133-ZR3.md diff --git a/.zed/rules b/.zed/rules index fa1c05b6b..8ab930d0a 100644 --- a/.zed/rules +++ b/.zed/rules @@ -740,8 +740,10 @@ data. The rules: order, user tags, system tags, data, source info, stated once in StaticOutputOperatorPod._in_row_order; a node that keeps a run with no rows keeps _zero_rows_in_row_order, and an operator's own empty result built from its input - goes through _as_row_built_table. Never change the order of a populated table: - stored tables depend on it. + goes through _as_row_built_table. An operator node keeps and stores the row order on every + path (_kept_output: a job run, sync or async, and node.run() on its own; ITL-1133), + and reads a stored operator cache in row order (_in_row_order_on_read). Never change + the order of any other populated table: stored tables depend on it. - Type strictness (developer, November 10, 2026): Orcapod is type strict. It never expands or converts a type implicitly; the only exception is normalization to large variants. Inputs that disagree on a column's type are an error naming the column and both types, on diff --git a/CLAUDE.md b/CLAUDE.md index c0a70c400..f5fa42766 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -782,8 +782,10 @@ data. The rules: order, user tags, system tags, data, source info, stated once in `StaticOutputOperatorPod._in_row_order`; a node that keeps a run with no rows keeps `_zero_rows_in_row_order`, and an operator's own empty result built from its input - goes through `_as_row_built_table`. Never change the order of a populated table: - stored tables depend on it. + goes through `_as_row_built_table`. An operator node keeps and stores the row order on every + path (`_kept_output`: a job run, sync or async, and `node.run()` on its own; ITL-1133), + and reads a stored operator cache in row order (`_in_row_order_on_read`). Never change + the order of any other populated table: stored tables depend on it. - **Type strictness** (developer, November 10, 2026): Orcapod is type strict. It never expands or converts a type implicitly; the only exception is normalization to large variants. Inputs that disagree on a column's type are an error naming the column and both types, on diff --git a/changelog.d/ITL-1131.md b/changelog.d/ITL-1131.md index 210d240a6..09bbe7168 100644 --- a/changelog.d/ITL-1131.md +++ b/changelog.d/ITL-1131.md @@ -6,12 +6,12 @@ lists them after. `Index` and `Pick` over an input with no rows, or whose rows were all skipped, dropped the system-tag columns, and `Pick` also gave its source-info columns as nullable and lost the token column of an `out=` column. Every operator's zero-row table now - equals its populated table in column names, order, Arrow types and nullability. No - populated table, stored table or hash changes. - -### Known limitations - -- An operator node run on its own (`node.run()` on a node built over streams outside a job) - keeps and stores its rows in the operator's own column order (a join lists its data before - its system tags), while a job run keeps and stores them in row order, and the node's table - before a run is in row order. A store that holds both fails to append the second (ITL-1133). + equals its populated table in column names, order, Arrow types and nullability. +- An operator node has one column order on every path, in memory and stored (ITL-1133): the + row order of a job run (user tags, system tags, data, source info). A node run on its own + (`node.run()`, without a store or in `LOG`) kept and stored its operator's own order (a join + lists its data before its system tags), so its table differed from the same node's in a job + and from its own table before a run, and a store written by both refused the second append. + An operator cache stored in the operator's order by an earlier build reads in row order and + accepts new rows; it is not rewritten and its path does not change. No hash or record id + changes: a record id does not depend on column order. diff --git a/design_issues/ITL-1131-ZR2.md b/design_issues/ITL-1131-ZR2.md index 0c42c3722..8a8b6cca3 100644 --- a/design_issues/ITL-1131-ZR2.md +++ b/design_issues/ITL-1131-ZR2.md @@ -24,7 +24,5 @@ every operator and a function pod, plain stream and job node, sync and async, be and after a zero-input run, with one and two system-tag pairs. No populated or stored order changed. -Out of scope (ITL-1133): an operator node run on its own (`_compute_and_store`) keeps and -stores the operator's own order while a job run uses row order, so one node has two populated -orders; a store holding both fails to append. Reproduction: -`superpowers/scripts/itl_1133_store_column_order.py`. +Found alongside it and fixed in the same change: an operator node run on its own kept and +stored its operator's order (ZR3, ITL-1133). diff --git a/design_issues/ITL-1133-ZR3.md b/design_issues/ITL-1133-ZR3.md new file mode 100644 index 000000000..9bde42acb --- /dev/null +++ b/design_issues/ITL-1133-ZR3.md @@ -0,0 +1,27 @@ +## Cross-cutting + +### ZR3 — An operator node kept and stored two column orders +**Status:** resolved +**Severity:** medium +**Issue:** ITL-1133 + +Found during ITL-1131 (ZR2). A job run keeps and stores an operator node's rows in row order +(`_materialize_to_stream`: user tags, system tags, data, source info), but a node run on its +own (`OperatorJobNode.run()`, `_compute_and_store`) kept its operator's result and stored it in +the operator's own order (a join lists its data before its system tags; every operator that +keeps its input's order did the same over a source, which lists system tags after data). So one +node had two populated orders, its table before a run (row order) matched only one of them, and +a store holding rows of both refused the second append (`StoreWriteError`, schema mismatch, in +the in-memory store and in a Delta store with a pending batch). Only the operator cache +(`odb_v1`) was affected: function pipeline and result tables, side-effect logs and cached +sources store one order on every path. `odb_v1` dates from 2026-11-12 (ITL-967) and was never +released. + +**Fix:** every way a node runs its operator keeps `StaticOutputOperatorPod._kept_output` (the +rows as `_materialize_to_stream` builds them, or the zero-row result in row order), so a node +run on its own keeps and stores the row order. The one operator-cache read path +(`OperatorJobNode._read_current_context_records`) puts the user columns of a stored table in +row order (`_in_row_order_on_read`; bookkeeping columns stay in place), so a table an earlier +build stored in the operator's order reads like any other and accepts row-order appends +(developer, November 14, 2026: option B, no layout bump). Record ids do not depend on column +order, so no hash moves. Reproduction: `superpowers/scripts/itl_1133_store_column_order.py`. diff --git a/superpowers/plans/2026-11-14-itl-1131-zero-row-column-order.md b/superpowers/plans/2026-11-14-itl-1131-zero-row-column-order.md index a7b1aafd9..452f2f631 100644 --- a/superpowers/plans/2026-11-14-itl-1131-zero-row-column-order.md +++ b/superpowers/plans/2026-11-14-itl-1131-zero-row-column-order.md @@ -84,3 +84,10 @@ Coordinator decision (2026-11-14): approve a–d; item 4 (direct node) out of sc ## Progress notes - 2026-11-14: sweep done, coordinator approved a–d, item 4 out of scope. +- 2026-11-14: a-d done (commits on branch). Coordinator then moved ITL-1133 into this PR + (developer: one stored column order): writer change (`_kept_output`, `_compute_output`) + and option B (reorder on read, `_in_row_order_on_read`, own commit; developer approved B, + no layout bump). Migration assessment: only `odb_v1` held two orders; `odb_v1` unreleased + (introduced 2026-11-12, ITL-967). Merged main 0208dca4 (#387): predicted schema order + equals the row order on every job node. +- Remaining: full suite, draft PR, CI, CodeRabbit; after PR #380 merges, flip its strict xfail. diff --git a/superpowers/scripts/itl_1133_store_column_order.py b/superpowers/scripts/itl_1133_store_column_order.py index 13d07d515..5e3471d10 100644 --- a/superpowers/scripts/itl_1133_store_column_order.py +++ b/superpowers/scripts/itl_1133_store_column_order.py @@ -1,5 +1,7 @@ """ITL-1133 reproduction (found during ITL-1131): one operator node table, two column orders. +Fails on main before the ITL-1133 fix (52d67858); prints OK for every case after it. + A job run keeps and stores an operator node's rows in row order (user tags, system tags, data, source info: ``_rows_to_table``); a direct ``node.run()`` (``_compute_and_store``) in ``CacheMode.LOG`` stores them in the operator's own order (a join: user tags, data, diff --git a/tests/test_pipeline/test_zero_row_column_order.py b/tests/test_pipeline/test_zero_row_column_order.py index 11b9ee37f..6dbce689a 100644 --- a/tests/test_pipeline/test_zero_row_column_order.py +++ b/tests/test_pipeline/test_zero_row_column_order.py @@ -22,7 +22,7 @@ from __future__ import annotations from collections.abc import Callable -from typing import Any +from typing import Any, cast import pyarrow as pa import pytest @@ -54,6 +54,7 @@ from orcapod.pipeline.job import PipelineJob from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator from orcapod.protocols.core_protocols import StreamProtocol +from orcapod.protocols.core_protocols.operator_pod import OperatorPodProtocol from orcapod.types import CacheMode ORCHESTRATORS = { @@ -245,7 +246,8 @@ def _node( ) -> OperatorJobNode: """An operator node of ``step`` over the plain streams the step reads, outside a job.""" stream = _stream(step, pairs, groups) - return OperatorJobNode(stream.producer, stream.upstreams, **kwargs) + operator = cast(OperatorPodProtocol, stream.producer) + return OperatorJobNode(operator, stream.upstreams, **kwargs) @pytest.mark.parametrize("pairs", PAIRS) @@ -323,9 +325,11 @@ def test_a_job_run_and_a_node_run_share_one_stored_table( else DeltaTableDatabase(tmp_path) ) schemas = [] + nodes = [] for how, ids in zip(order, ([1, 2], [3, 4]), strict=True): job = _join_job(ids) node = job.nodes["joined"] + nodes.append(node) node.attach_databases(pipeline_database=store, cache_mode=CacheMode.LOG) if how == "job": _ = job.run(orchestrator=SyncPipelineOrchestrator()) @@ -335,7 +339,7 @@ def test_a_job_run_and_a_node_run_share_one_stored_table( if store_kind == "delta_flushed": store.flush() assert schemas[0].equals(schemas[1]) - stored = node.get_all_records(all_info=True) + stored = nodes[-1].get_all_records(all_info=True) assert stored is not None assert stored.num_rows == 4 From afe7e06dae77f9647f1e0e357fe4776e893e8102 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 11:33:08 -0800 Subject: [PATCH 10/13] fix(column_selection): DropTagColumns predicts the system-tag columns its result keeps (ITL-1135) Its output_schema kept only the remaining user tags, so a prediction asking for system tags left them out. Adds a prediction sweep: every operator, every column config, plain stream and job node, 1 and 2 system-tag pairs, against static_process(...).output_schema(...). Batch and GroupBy disagree on a nullable member column (list[T | None] predicted, list[T] computed): strict xfail, reported. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1131.md | 3 + design_issues/ITL-1135-SP1.md | 19 ++++++ .../core/operators/column_selection.py | 7 +- .../test_zero_row_column_order.py | 66 +++++++++++++++++++ 4 files changed, 91 insertions(+), 4 deletions(-) create mode 100644 design_issues/ITL-1135-SP1.md diff --git a/changelog.d/ITL-1131.md b/changelog.d/ITL-1131.md index 09bbe7168..cfd0da8e0 100644 --- a/changelog.d/ITL-1131.md +++ b/changelog.d/ITL-1131.md @@ -15,3 +15,6 @@ An operator cache stored in the operator's order by an earlier build reads in row order and accepts new rows; it is not rewritten and its path does not change. No hash or record id changes: a record id does not depend on column order. +- `DropTagColumns` predicts the system-tag columns its result has (ITL-1135): + `output_schema(columns={"system_tags": True})` and `output_schema(all_info=True)` left them + out, though the computed table keeps them. No hash, record id or stored path changes. diff --git a/design_issues/ITL-1135-SP1.md b/design_issues/ITL-1135-SP1.md new file mode 100644 index 000000000..8966f4e9f --- /dev/null +++ b/design_issues/ITL-1135-SP1.md @@ -0,0 +1,19 @@ +## Cross-cutting + +### SP1 — DropTagColumns predicted no system-tag columns +**Status:** resolved +**Severity:** low +**Issue:** ITL-1135 + +Found during ITL-1131. `DropTagColumns.unary_output_schema` kept only the tag-schema keys +that were remaining user tags, so a prediction that asked for system tags +(`columns={"system_tags": True}`, `all_info=True`) left them out, while the computed result +keeps them; `SelectTagColumns` was right. The golden stream summary of `DropTagColumns` +pinned that wrong prediction (`system_tag_columns: []`). + +**Fix:** drop only the named user tags from the predicted tag schema. A sweep checks every +operator's prediction against `static_process(...).output_schema(...)` for every column +config, plain stream and job node, with one and two system-tag pairs. It found one more +disagreement, not fixed here: `Batch` and `GroupBy` predict `list[T | None]` for a nullable +data column whose computed table reads as `list[T]` (strict xfail in the sweep): the +Arrow-to-Python reading drops list item nullability (ITL-1136). diff --git a/src/orcapod/core/operators/column_selection.py b/src/orcapod/core/operators/column_selection.py index 64554270e..ae6f1db48 100644 --- a/src/orcapod/core/operators/column_selection.py +++ b/src/orcapod/core/operators/column_selection.py @@ -387,10 +387,9 @@ def unary_output_schema( tag_schema, data_schema = stream.output_schema( columns=columns, all_info=all_info ) - tag_columns, _ = stream.keys() - new_tag_columns = [c for c in tag_columns if c not in self.columns] - - new_tag_schema = {k: v for k, v in tag_schema.items() if k in new_tag_columns} + # Only the named user tags go: the system-tag columns stay, as in the + # computed result (ITL-1135). + new_tag_schema = {k: v for k, v in tag_schema.items() if k not in self.columns} return Schema(new_tag_schema), data_schema diff --git a/tests/test_pipeline/test_zero_row_column_order.py b/tests/test_pipeline/test_zero_row_column_order.py index 6dbce689a..3aec1767c 100644 --- a/tests/test_pipeline/test_zero_row_column_order.py +++ b/tests/test_pipeline/test_zero_row_column_order.py @@ -421,3 +421,69 @@ def test_accepts_a_row_order_append(self, tmp_path: Any) -> None: table = self._replay(DeltaTableDatabase(old), [3, 4]).as_table(all_info=True) assert table.num_rows == 4 assert table.schema.equals(self._expected()) + + +COLUMN_CONFIGS: dict[str, dict[str, Any]] = { + "default": {}, + "system_tags": {"columns": {"system_tags": True}}, + "source": {"columns": {"source": True}}, + "context": {"columns": {"context": True}}, + "content_hash": {"columns": {"content_hash": True}}, + "meta": {"columns": {"meta": True}}, + "all_info": {"all_info": True}, +} + + +def _schema_items( + schemas: tuple[Any, Any], +) -> tuple[list[tuple[str, Any]], list[tuple[str, Any]]]: + """The tag and data schemas as ordered (name, Python type) lists.""" + tag_schema, data_schema = schemas + return list(tag_schema.items()), list(data_schema.items()) + + +def _computed(step: str, pairs: int) -> Any: + """The operator's computed result over the step's inputs. + + A schema prediction is verified against ``static_process(...).output_schema(...)`` + (CLAUDE.md's standing exception to the public-interface rule). + """ + stream = _stream(step, pairs, WITH_ROWS) + return stream.producer.static_process(*stream.upstreams) # pyright: ignore[reportOptionalMemberAccess, reportAttributeAccessIssue] # an operator's stream has a producer with static_process + + +_NULLABLE_MEMBER = pytest.mark.xfail( + strict=True, + reason=( + "Batch and GroupBy predict list[int | None] for the nullable column n; their " + "computed table's large_list (nullable items) reads as list[int] " + "(ITL-1136: the Arrow-to-Python reading drops list item nullability)" + ), +) +PREDICTION_STEPS = [ + pytest.param(step, marks=_NULLABLE_MEMBER) + if step in ("batch", "group_by") + else step + for step in OPERATOR_STEPS +] + + +@pytest.mark.parametrize("config", list(COLUMN_CONFIGS)) +@pytest.mark.parametrize("pairs", PAIRS) +@pytest.mark.parametrize("step", PREDICTION_STEPS) +class TestOutputSchemaPrediction: + """Every operator predicts the schema of its computed result (ITL-1135).""" + + def test_plain_stream(self, step: str, pairs: int, config: str) -> None: + kwargs = COLUMN_CONFIGS[config] + predicted = _stream(step, pairs, WITH_ROWS).output_schema(**kwargs) + computed = _computed(step, pairs).output_schema(**kwargs) + assert _schema_items(predicted) == _schema_items(computed) + + def test_job_node(self, step: str, pairs: int, config: str) -> None: + kwargs = COLUMN_CONFIGS[config] + job = _job(step, pairs, WITH_ROWS) + _ = job.run(orchestrator=SyncPipelineOrchestrator()) + predicted = job.nodes["op"].output_schema(**kwargs) + computed = _computed(step, pairs).output_schema(**kwargs) + assert _schema_items(predicted) == _schema_items(computed) From 83ddb2c70d8c1b80cd467e74adca34194aa924a1 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 11:33:32 -0800 Subject: [PATCH 11/13] test(golden): pin DropTagColumns' system-tag columns (ITL-1135) The stream summary of DropTagColumns pinned its wrong prediction (system_tag_columns: []). Changes that one entry to the two system-tag column names and adds the four per-row system-tag values they bring. No content hash, pipeline hash, data hash or stored path moves. Pending developer approval. Co-Authored-By: Claude Opus 5.5 --- tests/test_hashing/golden/data/pipeline.json | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_hashing/golden/data/pipeline.json b/tests/test_hashing/golden/data/pipeline.json index 973021542..f1ebcc509 100644 --- a/tests/test_hashing/golden/data/pipeline.json +++ b/tests/test_hashing/golden/data/pipeline.json @@ -28,9 +28,16 @@ ], "column_selection.drop_tag.content_hash": "semantic_v1:918de396eb2329433b82d3ca7f28c8268ad236aad9853191f04218d78125afd4", "column_selection.drop_tag.pipeline_hash": "semantic_v1:76f153e7528e3e6f21bc27b2fcab2401f817c1d4595323a8e4409a515da18340", + "column_selection.drop_tag.row[{\"cohort\":\"a\"}]._tag_record_id::7d650c4855ec007531f6a110836de7a0ff2ac68b4bfff32e58b2c41d02f44f7c": "37d45a37e6255cd78bb62c4c9d047ca5", + "column_selection.drop_tag.row[{\"cohort\":\"a\"}]._tag_source_id::7d650c4855ec007531f6a110836de7a0ff2ac68b4bfff32e58b2c41d02f44f7c": "cohorts", "column_selection.drop_tag.row[{\"cohort\":\"a\"}].data_hash": "arrow_v1:00000252e4f32a133cd6a8813f816e08b6bde1cdb467baa95d026fc41c5672b6b98110", + "column_selection.drop_tag.row[{\"cohort\":\"b\"}]._tag_record_id::7d650c4855ec007531f6a110836de7a0ff2ac68b4bfff32e58b2c41d02f44f7c": "d0cc03b6594d51788075066f5c81deb0", + "column_selection.drop_tag.row[{\"cohort\":\"b\"}]._tag_source_id::7d650c4855ec007531f6a110836de7a0ff2ac68b4bfff32e58b2c41d02f44f7c": "cohorts", "column_selection.drop_tag.row[{\"cohort\":\"b\"}].data_hash": "arrow_v1:0000025f33a2f5bf3e956edfb7ca628b6871f52b7bbf788d5a3774ddffe300d7d4aff5", - "column_selection.drop_tag.system_tag_columns": [], + "column_selection.drop_tag.system_tag_columns": [ + "_tag_source_id::7d650c4855ec007531f6a110836de7a0ff2ac68b4bfff32e58b2c41d02f44f7c", + "_tag_record_id::7d650c4855ec007531f6a110836de7a0ff2ac68b4bfff32e58b2c41d02f44f7c" + ], "column_selection.select_data.content_hash": "semantic_v1:5296eb99a253bd93c79f8e162af14447fabc9fd02c42fdb19061794fb7ace639", "column_selection.select_data.pipeline_hash": "semantic_v1:a64b3ddcd8c16aa17dc18e516e3a49cccf97216d6b7869b32b5bcde06c3b4f4e", "column_selection.select_data.row[{\"subject\":\"s1\"}]._tag_record_id::d22029fcc2b0458566a83389c23c1a850dc69499f2456ae83bc6345284f861a5": "e8e02084627f5e1aad83839e421538e8", From 52696d95ccc608d1188573c6e8dc96f84ad3d12b Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 12:13:04 -0800 Subject: [PATCH 12/13] docs(plan): ITL-1131 progress notes Co-Authored-By: Claude Opus 5.5 --- .../plans/2026-11-14-itl-1131-zero-row-column-order.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/superpowers/plans/2026-11-14-itl-1131-zero-row-column-order.md b/superpowers/plans/2026-11-14-itl-1131-zero-row-column-order.md index 452f2f631..426316f98 100644 --- a/superpowers/plans/2026-11-14-itl-1131-zero-row-column-order.md +++ b/superpowers/plans/2026-11-14-itl-1131-zero-row-column-order.md @@ -91,3 +91,9 @@ Coordinator decision (2026-11-14): approve a–d; item 4 (direct node) out of sc (introduced 2026-11-12, ITL-967). Merged main 0208dca4 (#387): predicted schema order equals the row order on every job node. - Remaining: full suite, draft PR, CI, CodeRabbit; after PR #380 merges, flip its strict xfail. +- 2026-11-14: ITL-1135 (DropTagColumns prediction) added as its own commits (fix + golden, + developer approved the golden move). PR #390 ready (developer). CodeRabbit round 1 (review + 5906873847 on 83ddb2c7): no actionable findings; signed reply posted. CI green. +- Remaining: when #380 merges (coordinator says when), merge main, turn its strict xfail + `test_a_join_with_no_input_rows_keeps_its_column_order` into a passing test, push, CI, one + signed '@coderabbitai review', worker_done. From 760e8892b144b823a3dc650729671aa6078b65f6 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 12:44:46 -0800 Subject: [PATCH 13/13] test(operator_node): REPLAY reads run the node first, as a node shows its current run (ITL-1095) After #380 a node shows only its current run, so the REPLAY reads in the sweep run the node first; a REPLAY run over lazy inputs it does not compute then reads as stale once they compute (as on main e4b4319c), so those inputs are computed first. Co-Authored-By: Claude Opus 5.5 --- tests/test_pipeline/test_zero_row_column_order.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_pipeline/test_zero_row_column_order.py b/tests/test_pipeline/test_zero_row_column_order.py index c9f5fe890..416620cf5 100644 --- a/tests/test_pipeline/test_zero_row_column_order.py +++ b/tests/test_pipeline/test_zero_row_column_order.py @@ -307,6 +307,12 @@ def test_log_run_and_replay(self, step: str, pairs: int) -> None: replay = _node( step, pairs, WITH_ROWS, pipeline_database=store, cache_mode=CacheMode.REPLAY ) + # A REPLAY run over lazy inputs that it does not compute reads as stale once + # they compute (seen on main e4b4319c, reported); computed first, they do not. + for upstream in replay.upstreams: + _ = upstream.as_table() + # A node shows its current run (ITL-1095): a REPLAY run loads the stored rows. + _ = replay.run() table = replay.as_table(all_info=True) assert table.num_rows > 0 assert table.schema.equals(expected) @@ -416,8 +422,10 @@ def _old_table(self, tmp_path: Any) -> Any: return old def _replay(self, store: DeltaTableDatabase, ids: list[int]) -> OperatorJobNode: + """A REPLAY node over ``store`` after its run, which loads the stored rows.""" node = _join_job(ids).nodes["joined"] node.attach_databases(pipeline_database=store, cache_mode=CacheMode.REPLAY) + _ = node.run() # a node shows its current run (ITL-1095) return node def test_reads_in_row_order(self, tmp_path: Any) -> None: