From 2d058609665b0c0b06de3e1babd72d17e878318b Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 22:27:09 -0800 Subject: [PATCH 1/4] fix(datagrams): a Data's source info follows its data columns (ITL-1040) Data.drop and Data.select now remove the source info of the columns they remove, and the source-info table holds only the entries of current columns. Async select and drop paths, which drop columns datagram by datagram, no longer store an extra _source_ that a sync run does not store. WIP: not pushed until the developer decides on existing tables. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/datagrams/tag_data.py | 27 +++- ...-12-itl-1040-dropped-column-source-info.md | 28 ++++ .../test_data_source_info_follows_columns.py | 117 +++++++++++++++ .../test_sync_async_stored_records.py | 133 ++++++++++++++++++ 4 files changed, 302 insertions(+), 3 deletions(-) create mode 100644 superpowers/plans/2026-11-12-itl-1040-dropped-column-source-info.md create mode 100644 tests/test_core/datagrams/test_data_source_info_follows_columns.py create mode 100644 tests/test_pipeline/test_sync_async_stored_records.py diff --git a/src/orcapod/core/datagrams/tag_data.py b/src/orcapod/core/datagrams/tag_data.py index 383570844..49aebd51d 100644 --- a/src/orcapod/core/datagrams/tag_data.py +++ b/src/orcapod/core/datagrams/tag_data.py @@ -367,10 +367,13 @@ def _ensure_source_info_table(self) -> "pa.Table": if self._source_info_table is None: import pyarrow as _pa - if self._source_info: + # Only the source info of the data columns this Data holds: a column it + # no longer has has no source info in any view (ITL-1040). + data_keys = set(super().keys()) + current = {k: v for k, v in self._source_info.items() if k in data_keys} + if current: prefixed = { - f"{constants.SOURCE_PREFIX}{k}": v - for k, v in self._source_info.items() + f"{constants.SOURCE_PREFIX}{k}": v for k, v in current.items() } schema = _pa.schema( [ @@ -476,6 +479,24 @@ def as_table( table = arrow_utils.hstack_tables(table, si_table) return table + def select(self, *column_names: str) -> Self: + """Return a copy holding only the given data columns and their source info.""" + new_p = super().select(*column_names) + new_p._source_info = { + k: v for k, v in self._source_info.items() if k in column_names + } + new_p._source_info_table = None + return new_p + + def drop(self, *column_names: str, ignore_missing: bool = False) -> Self: + """Return a copy without the given data columns and their source info.""" + new_p = super().drop(*column_names, ignore_missing=ignore_missing) + new_p._source_info = { + k: v for k, v in self._source_info.items() if k not in column_names + } + new_p._source_info_table = None + return new_p + def rename(self, column_mapping: "Mapping[str, str]") -> Self: new_p = super().rename(column_mapping) new_p._source_info = { diff --git a/superpowers/plans/2026-11-12-itl-1040-dropped-column-source-info.md b/superpowers/plans/2026-11-12-itl-1040-dropped-column-source-info.md new file mode 100644 index 000000000..693b4dac5 --- /dev/null +++ b/superpowers/plans/2026-11-12-itl-1040-dropped-column-source-info.md @@ -0,0 +1,28 @@ +# ITL-1040 Dropped data column keeps its source info: plan + +**Goal:** a `Data` without a column carries no source info for it in any view, so sync and +async runs store identical pipeline records. + +**Cause:** `Datagram.drop` (and `select`) are not overridden by `Data`; they leave the dropped +column's entry in `Data._source_info`, and `_ensure_source_info_table` builds the source-info +table (used by `as_table(source=True)` and `arrow_schema(source=True)`) from all of +`_source_info`, while `as_dict` / `source_info()` filter to `keys()`. The async +`SelectDataColumns` / `DropDataColumns` call `data.drop`; their sync paths drop `_source_*` +explicitly. + +**Steps:** +1. MEASURE FIRST (send to coordinator before any push): with `main`, an async run of a job + whose pod input is auto-selected writes a pipeline table with `_source_s`; then the fixed + build appends new rows to that table (Delta and in memory). Record: succeeds / fails / + writes nulls; what a read returns. Also state with evidence whether any hash, record id or + stored path depends on the extra column. +2. Failing-first tests: `Data.drop` / `Data.select` + `as_table(source=True)`, + `arrow_schema(source=True)`; sync == async stored records (auto select, explicit + select, explicit drop; in-memory and Delta); other Datagram operations (`rename`, + `update`, `with_columns`, `copy`) and `Tag` system tags checked for the same drift. +3. Fix: `Data` keeps `_source_info` to its own keys: override `drop` and `select` to prune, + and build the source-info table from the entries of current keys only. +4. design_issues entry, changelog fragment, full suite, PR "Fixes ITL-1040". + +No hash, golden, API change expected; stored schema of async-written tables changes (the +extra column disappears from new rows) - the developer decides handling from step 1. diff --git a/tests/test_core/datagrams/test_data_source_info_follows_columns.py b/tests/test_core/datagrams/test_data_source_info_follows_columns.py new file mode 100644 index 000000000..52040547d --- /dev/null +++ b/tests/test_core/datagrams/test_data_source_info_follows_columns.py @@ -0,0 +1,117 @@ +"""A ``Data``'s source info follows its data columns in every view (ITL-1040). + +``Data.drop`` and ``Data.select`` used to remove a data column but keep its source info, +which ``as_dict`` filtered out and ``as_table`` / ``arrow_schema`` did not. The async paths +of ``SelectDataColumns`` and ``DropDataColumns`` call them, so an async run stored an extra +``_source_`` that a sync run did not. +""" + +from __future__ import annotations + +from typing import Any + +import pyarrow as pa +import pytest + +from orcapod.core.datagrams import Data, Tag +from orcapod.system_constants import constants + +SOURCE = {"x": "src::row_0::x", "s": "src::row_0::s", "m": "src::row_0::m"} + + +def data(backing: str) -> Data: + """A ``Data`` with columns ``x``, ``s``, ``m`` and a source token for each.""" + values: dict[str, Any] = {"x": 1, "s": "p", "m": 2.5} + if backing == "dict": + return Data(values, source_info=SOURCE) + table = pa.table( + { + "x": pa.array([1], pa.int64()), + "s": pa.array(["p"], pa.large_string()), + "m": pa.array([2.5], pa.float64()), + **{ + f"{constants.SOURCE_PREFIX}{k}": pa.array([v], pa.large_string()) + for k, v in SOURCE.items() + }, + } + ) + return Data(table) + + +BACKINGS = ["dict", "arrow"] + + +def source_columns(names: list[str]) -> list[str]: + """The source-info column names among ``names``, in order.""" + return [n for n in names if n.startswith(constants.SOURCE_PREFIX)] + + +def assert_source_follows(d: Data, expected_columns: list[str]) -> None: + """Every view of ``d`` with source info names exactly ``expected_columns``.""" + expected = [f"{constants.SOURCE_PREFIX}{c}" for c in expected_columns] + assert d.keys() == tuple(expected_columns) + config = {"source": True} + assert source_columns(d.as_table(columns=config).column_names) == expected + assert source_columns(d.arrow_schema(columns=config).names) == expected + assert source_columns(list(d.as_dict(columns=config))) == expected + assert source_columns(list(d.schema(columns=config))) == expected + assert source_columns(list(d.keys(columns=config))) == expected + assert list(d.source_info()) == expected_columns + + +@pytest.mark.parametrize("backing", BACKINGS) +def test_drop_removes_the_source_info_of_the_dropped_column(backing: str) -> None: + dropped = data(backing).drop("s") + assert_source_follows(dropped, ["x", "m"]) + assert dropped.source_info() == {"x": SOURCE["x"], "m": SOURCE["m"]} + + +@pytest.mark.parametrize("backing", BACKINGS) +def test_select_keeps_only_the_source_info_of_the_selected_columns( + backing: str, +) -> None: + selected = data(backing).select("x", "m") + assert_source_follows(selected, ["x", "m"]) + + +@pytest.mark.parametrize("backing", BACKINGS) +def test_rename_renames_the_source_info(backing: str) -> None: + renamed = data(backing).rename({"s": "t"}) + assert_source_follows(renamed, ["x", "t", "m"]) + assert renamed.source_info()["t"] == SOURCE["s"] + + +@pytest.mark.parametrize("backing", BACKINGS) +def test_update_and_copy_keep_the_source_info(backing: str) -> None: + assert_source_follows(data(backing).update(x=7), ["x", "s", "m"]) + assert_source_follows(data(backing).copy(), ["x", "s", "m"]) + + +@pytest.mark.parametrize("backing", BACKINGS) +def test_with_columns_adds_empty_source_info_for_the_new_column(backing: str) -> None: + extended = data(backing).with_columns(n=3) + assert_source_follows(extended, ["x", "s", "m", "n"]) + assert extended.source_info()["n"] is None + + +@pytest.mark.parametrize("backing", BACKINGS) +def test_a_column_dropped_then_added_again_has_no_stale_source_info( + backing: str, +) -> None: + again = data(backing).drop("s").with_columns(s="q") + assert again.source_info()["s"] is None + table = again.as_table(columns={"source": True}) + assert table.column(f"{constants.SOURCE_PREFIX}s").to_pylist() == [None] + + +def test_tag_drop_keeps_the_system_tags_which_belong_to_the_row() -> None: + """System tags describe the row, not a tag column: dropping a tag keeps them.""" + system_tag = f"{constants.SYSTEM_TAG_SOURCE_ID_PREFIX}::abc" + tag = Tag({"a": 1, "b": "q"}, system_tags={system_tag: "src"}) + + dropped = tag.drop("b") + + assert dropped.keys() == ("a",) + table = dropped.as_table(columns={"system_tags": True}) + assert table.column_names == ["a", system_tag] + assert dropped.system_tags() == {system_tag: "src"} diff --git a/tests/test_pipeline/test_sync_async_stored_records.py b/tests/test_pipeline/test_sync_async_stored_records.py new file mode 100644 index 000000000..a095f634f --- /dev/null +++ b/tests/test_pipeline/test_sync_async_stored_records.py @@ -0,0 +1,133 @@ +"""Sync and async runs store the same pipeline records after a select or drop (ITL-1040). + +The async paths of ``SelectDataColumns`` and ``DropDataColumns`` drop columns datagram by +datagram. Before ITL-1040 the dropped columns' source info survived there, so an async run +stored an extra ``_source_`` in the function node's records that a sync run +did not: the stored table's schema depended on the orchestrator. +""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pyarrow as pa +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.core.operators import DropDataColumns, SelectDataColumns +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.sources import ArrowTableSource + +STORES: dict[str, Callable[[Path], Any]] = { + "in_memory": lambda _: InMemoryArrowDatabase(), + "delta": lambda path: DeltaTableDatabase(path), +} + +# Columns that differ between two runs by design: the output datagram's uuid, the token +# that carries it, and the time the row was computed. +PER_RUN_COLUMNS = ("__data_id", "_source_y", "__pod_ts") + + +@function_pod(output_keys="y") +def times_ten(x: int) -> int: + """Return ``10 * x``.""" + return 10 * x + + +def source() -> ArrowTableSource: + """Tag ``k``; data ``x`` (the pod's input) and ``s``, ``m`` (not its input).""" + table = pa.table( + { + "k": ["a", "b", "c"], + "x": [1, 2, 3], + "s": ["p", "q", "r"], + "m": [0.5, 1.5, 2.5], + }, + schema=pa.schema( + [ + pa.field("k", pa.large_string(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + pa.field("s", pa.large_string(), nullable=False), + pa.field("m", pa.float64(), nullable=False), + ] + ), + ) + return ArrowTableSource(table, tag_columns=["k"], source_id="src") + + +def automatic_select(src: ArrowTableSource) -> Any: + """``pod(src)`` selects the pod's input ``x`` itself.""" + return times_ten.pod(src, label="node") + + +def explicit_select(src: ArrowTableSource) -> Any: + """``SelectDataColumns(["x"])`` before the pod.""" + return times_ten.pod(SelectDataColumns(["x"])(src), label="node") + + +def explicit_drop(src: ArrowTableSource) -> Any: + """``DropDataColumns(["s", "m"])`` before the pod.""" + return times_ten.pod(DropDataColumns(["s", "m"])(src), label="node") + + +PIPELINES = { + "automatic_select": automatic_select, + "explicit_select": explicit_select, + "explicit_drop": explicit_drop, +} + + +def stored_records( + pipeline: str, orchestrator: str, store_kind: str, directory: Path +) -> pa.Table: + """The function node's stored records after a run, rows in tag order.""" + job = PipelineJob(name="itl1040", store=STORES[store_kind](directory)) + with job: + PIPELINES[pipeline](source()) + job.run( + orchestrator=SyncPipelineOrchestrator() + if orchestrator == "sync" + else AsyncPipelineOrchestrator() + ) + records = job.nodes["node"].get_all_records(all_info=True) + assert records is not None + return records.sort_by("k") + + +@pytest.mark.parametrize("store_kind", list(STORES)) +@pytest.mark.parametrize("pipeline", list(PIPELINES)) +def test_async_stores_the_records_a_sync_run_stores( + pipeline: str, store_kind: str, tmp_path: Path +) -> None: + sync = stored_records(pipeline, "sync", store_kind, tmp_path / "sync") + async_ = stored_records(pipeline, "async", store_kind, tmp_path / "async") + + assert async_.schema == sync.schema + assert "_source_s" not in async_.column_names + assert "_source_m" not in async_.column_names + assert async_.drop_columns(list(PER_RUN_COLUMNS)).equals( + sync.drop_columns(list(PER_RUN_COLUMNS)) + ) + + +@pytest.mark.parametrize("pipeline", ["explicit_select", "explicit_drop"]) +def test_the_operator_node_table_is_the_same_on_both_orchestrators( + pipeline: str, +) -> None: + """The select or drop node itself shows the same columns after either run.""" + tables = [] + for orchestrator in (SyncPipelineOrchestrator, AsyncPipelineOrchestrator): + job = PipelineJob(name="itl1040", store=InMemoryArrowDatabase()) + with job: + src = source() + if pipeline == "explicit_select": + SelectDataColumns(["x"])(src, label="op") + else: + DropDataColumns(["s", "m"])(src, label="op") + job.run(orchestrator=orchestrator()) + tables.append(job.nodes["op"].as_table(all_info=True).sort_by("k")) + assert tables[1].schema == tables[0].schema + assert tables[1].equals(tables[0]) From 597fbb8d3864955e2630bd198064467fb871cdd9 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 22:27:29 -0800 Subject: [PATCH 2/4] docs(design_issues): ITL-1040 entry and changelog fragment Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1040.md | 3 +++ design_issues/ITL-1040-DG2.md | 29 +++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 changelog.d/ITL-1040.md create mode 100644 design_issues/ITL-1040-DG2.md diff --git a/changelog.d/ITL-1040.md b/changelog.d/ITL-1040.md new file mode 100644 index 000000000..0361c31d0 --- /dev/null +++ b/changelog.d/ITL-1040.md @@ -0,0 +1,3 @@ +### Fixed + +- Async job runs no longer store an extra `_source_` column for a data column dropped before a function pod, by the automatic select of `pod(stream)` or by an explicit `SelectDataColumns` or `DropDataColumns` (ITL-1040). `Data.drop` and `Data.select` kept the source info of the removed columns, and `Data.as_table(columns={"source": True})` still showed it. Sync and async runs now store the same pipeline records. No hash, record id or stored path changes. A Delta pipeline table written by an async run before this fix keeps the extra column, and rows written after the fix have null in it. diff --git a/design_issues/ITL-1040-DG2.md b/design_issues/ITL-1040-DG2.md new file mode 100644 index 000000000..7c2d21097 --- /dev/null +++ b/design_issues/ITL-1040-DG2.md @@ -0,0 +1,29 @@ +## `src/orcapod/core/datagrams/tag_data.py` + +### DG2 — A dropped data column kept its source info, so async runs stored an extra column +**Status:** resolved +**Severity:** high +**Issue:** ITL-1040 + +`Data` did not override `Datagram.drop` or `Datagram.select`, so both left the removed +columns' entries in `Data._source_info`. The source-info table that `as_table(source=True)` +and `arrow_schema(source=True)` use was built from all of `_source_info`, while `as_dict` and +`source_info()` filtered it to the current columns. + +The async paths of `SelectDataColumns` and `DropDataColumns` drop columns one datagram at a +time with `data.drop`. Their sync paths drop the `_source_` columns from the table +explicitly. So an async run stored an extra `_source_` in the function node's +pipeline records that a sync run did not, and the stored table's schema depended on the +orchestrator. This happened after the automatic select in front of a pod, and after any +explicit select or drop. On one store, mixing orchestrators then either widened a Delta table +(with nulls) or failed an in-memory commit with a schema mismatch. The cell values were always +the row's own. The row-alignment suite found it (ITL-1034). + +**Fix:** `Data.drop` and `Data.select` remove the source info of the columns they remove, and +the source-info table holds only the entries of current columns. Sync and async runs now +store identical records (`tests/test_pipeline/test_sync_async_stored_records.py`). No hash, +record id or stored path depends on source-info columns. + +A Delta table that an async run wrote before the fix keeps its extra column. Rows written +after the fix have null in it, which is what a sync run appending to such a table already did +before the fix. From 6820febccb0098be5af763e9411af054d2043288 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 22:39:13 -0800 Subject: [PATCH 3/4] fix(datagrams): select keeps source info in column order; ITL-838 xfails removed; ITL-1040 evidence Data.select orders the kept source info as the columns it holds (the in-place Index path restores column order with select). The strict ITL-838 xfails in test_async_sync_equivalence pass now and are removed. The append measurement is committed under superpowers/specs/itl-1040. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1040.md | 2 +- design_issues/ITL-1040-DG2.md | 4 +- src/orcapod/core/datagrams/tag_data.py | 10 +- superpowers/specs/itl-1040/README.md | 28 ++ superpowers/specs/itl-1040/measure.py | 248 ++++++++++++++++++ .../test_data_source_info_follows_columns.py | 12 +- .../operators/test_async_sync_equivalence.py | 23 +- 7 files changed, 301 insertions(+), 26 deletions(-) create mode 100644 superpowers/specs/itl-1040/README.md create mode 100644 superpowers/specs/itl-1040/measure.py diff --git a/changelog.d/ITL-1040.md b/changelog.d/ITL-1040.md index 0361c31d0..16fb70240 100644 --- a/changelog.d/ITL-1040.md +++ b/changelog.d/ITL-1040.md @@ -1,3 +1,3 @@ ### Fixed -- Async job runs no longer store an extra `_source_` column for a data column dropped before a function pod, by the automatic select of `pod(stream)` or by an explicit `SelectDataColumns` or `DropDataColumns` (ITL-1040). `Data.drop` and `Data.select` kept the source info of the removed columns, and `Data.as_table(columns={"source": True})` still showed it. Sync and async runs now store the same pipeline records. No hash, record id or stored path changes. A Delta pipeline table written by an async run before this fix keeps the extra column, and rows written after the fix have null in it. +- Async job runs no longer store an extra `_source_` column for a data column dropped before a function pod, by the automatic select of `pod(stream)` or by an explicit `SelectDataColumns` or `DropDataColumns` (ITL-1040, ITL-838). `Data.drop` and `Data.select` kept the source info of the removed columns, and `Data.as_table(columns={"source": True})` still showed it. A selected `Data`'s source info is now in the order of its columns. Sync and async runs now store the same pipeline records. No hash, record id or stored path changes. Existing tables are not migrated in this release. A Delta pipeline table written by an async run before this fix keeps the extra column, and rows written from now on have null in it. Migrating stores written by earlier versions is the job of the migration tool planned after v0.3 (ITL-1048). diff --git a/design_issues/ITL-1040-DG2.md b/design_issues/ITL-1040-DG2.md index 7c2d21097..4f17208d7 100644 --- a/design_issues/ITL-1040-DG2.md +++ b/design_issues/ITL-1040-DG2.md @@ -17,7 +17,7 @@ pipeline records that a sync run did not, and the stored table's schema depended orchestrator. This happened after the automatic select in front of a pod, and after any explicit select or drop. On one store, mixing orchestrators then either widened a Delta table (with nulls) or failed an in-memory commit with a schema mismatch. The cell values were always -the row's own. The row-alignment suite found it (ITL-1034). +the row's own. The row-alignment suite found it (ITL-1034). It had been reported for `Data` alone as ITL-838. **Fix:** `Data.drop` and `Data.select` remove the source info of the columns they remove, and the source-info table holds only the entries of current columns. Sync and async runs now @@ -26,4 +26,4 @@ record id or stored path depends on source-info columns. A Delta table that an async run wrote before the fix keeps its extra column. Rows written after the fix have null in it, which is what a sync run appending to such a table already did -before the fix. +before the fix. No migration in this release; migrating stores written by earlier versions is the job of the migration tool planned after v0.3 (ITL-1048). diff --git a/src/orcapod/core/datagrams/tag_data.py b/src/orcapod/core/datagrams/tag_data.py index 49aebd51d..ab324e0ef 100644 --- a/src/orcapod/core/datagrams/tag_data.py +++ b/src/orcapod/core/datagrams/tag_data.py @@ -480,10 +480,16 @@ def as_table( return table def select(self, *column_names: str) -> Self: - """Return a copy holding only the given data columns and their source info.""" + """Return a copy holding only the given data columns and their source info. + + The source info is in the order of the columns the copy holds. + """ new_p = super().select(*column_names) + # Datagram.keys: the data columns, also for an EmptyData, whose keys() raises. new_p._source_info = { - k: v for k, v in self._source_info.items() if k in column_names + k: self._source_info[k] + for k in Datagram.keys(new_p) + if k in self._source_info } new_p._source_info_table = None return new_p diff --git a/superpowers/specs/itl-1040/README.md b/superpowers/specs/itl-1040/README.md new file mode 100644 index 000000000..2276bbc0e --- /dev/null +++ b/superpowers/specs/itl-1040/README.md @@ -0,0 +1,28 @@ +# ITL-1040 measurement: appending to a pipeline table an async run wrote before the fix + +Evidence for the ITL-1040 decision. These scripts are not tests, and the test run does not +collect them. + +`measure.py` runs each step as a subcommand (see its docstring). The job is source (`k`; `x`, +`s`) → `times_ten(x)`; the pod's automatic select drops `s`. An in-memory store is carried +from one build to another by dumping its tables to Arrow files and reloading them. + +- `phase1 ` (run on `main` before the fix): async runs write rows `r0..r3` to Delta + stores and to a dumped in-memory store. It also measures the baseline on `main`: a sync run + appending rows `r4, r5` to an async-written table. +- `phase1b` (run on `main` before the fix): the reverse baseline, an async run appending to + a sync-written table. +- `phase2 ` (run on the fixed build): async and sync runs append `r4, r5` to the + tables phase 1 wrote. +- `operator_node`: a cached operator node downstream of an async select. Are its stored + records the same after a sync and an async run? + +Results (November 12, 2026): + +| Store | Who appends | Result | +|---|---|---| +| Delta | fixed build, async or sync | succeeds; the old `_source_s` column stays, old rows keep their tokens, new rows get NULL | +| in memory | fixed build, async or sync | `StoreWriteError` (schema mismatch); cannot happen across processes, since an in-memory store dies with its process | +| Delta | `main`, sync after async | succeeds; new rows get NULL in `_source_s` (the same state as the fixed build) | +| Delta | `main`, async after sync | succeeds; the OLD rows get NULL in `_source_s` | +| in memory | `main`, sync after async, or async after sync | fails (`StoreWriteError` / `ExceptionGroup`) | diff --git a/superpowers/specs/itl-1040/measure.py b/superpowers/specs/itl-1040/measure.py new file mode 100644 index 000000000..eae1c79d1 --- /dev/null +++ b/superpowers/specs/itl-1040/measure.py @@ -0,0 +1,248 @@ +"""ITL-1040 measurement: appending to a pipeline table an async run wrote before the fix. + +Usage (from the repository root): + + uv run python superpowers/specs/itl-1040/measure.py phase1 # on main, unfixed + uv run python superpowers/specs/itl-1040/measure.py phase1b # on main, unfixed + uv run python superpowers/specs/itl-1040/measure.py phase2 # on the fixed build + uv run python superpowers/specs/itl-1040/measure.py operator_node # either build + +See README.md in this folder for what each phase does and the results. +""" + +from __future__ import annotations + +import json +import shutil +import sys +import tempfile +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pyarrow as pa +import pyarrow.feather as feather + +from orcapod import PipelineJob, function_pod +from orcapod.core.operators import Join, SelectDataColumns +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.sources import ArrowTableSource +from orcapod.types import CacheMode + +DUMP_ID = "__dump_record_id" + + +@function_pod(output_keys="y") +def times_ten(x: int) -> int: + """Return ``10 * x``.""" + return 10 * x + + +def source(n: int) -> ArrowTableSource: + """``n`` rows: tag ``k``; data ``x`` (the pod's input) and ``s`` (dropped).""" + table = pa.table( + { + "k": [f"r{i}" for i in range(n)], + "x": list(range(n)), + "s": [f"s{i}" for i in range(n)], + }, + schema=pa.schema( + [ + pa.field("k", pa.large_string(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + pa.field("s", pa.large_string(), nullable=False), + ] + ), + ) + return ArrowTableSource( + table, tag_columns=["k"], source_id="src", record_id_column="x" + ) + + +def run(store: Any, n: int, mode: str) -> tuple[PipelineJob, Any]: + """Run ``times_ten`` over ``n`` rows on ``store`` (``mode`` "sync" or "async").""" + job = PipelineJob(name="m", store=store) + with job: + times_ten.pod(source(n), label="n") # the automatic select of x drops s + orchestrator = ( + AsyncPipelineOrchestrator() if mode == "async" else SyncPipelineOrchestrator() + ) + return job, job.run(orchestrator=orchestrator) + + +def dump_memory(store: InMemoryArrowDatabase, out: Path) -> None: + """Write every table of ``store`` to ``out`` as Arrow files.""" + out.mkdir(parents=True, exist_ok=True) + index = [] + for i, path in enumerate(store.list_table_paths()): + table = store.get_all_records(path, record_id_column=DUMP_ID) + assert table is not None + feather.write_feather(table, out / f"{i}.arrow") + index.append(list(path)) + (out / "paths.json").write_text(json.dumps(index)) + + +def load_memory(out: Path) -> InMemoryArrowDatabase: + """An in-memory store holding the tables ``dump_memory`` wrote.""" + store = InMemoryArrowDatabase() + for i, path in enumerate(json.loads((out / "paths.json").read_text())): + table = feather.read_table(out / f"{i}.arrow") + store.add_records(tuple(path), table, record_id_column=DUMP_ID, flush=True) + return store + + +def describe(job: PipelineJob, label: str) -> None: + """Print the node's stored source-info columns and its table.""" + node = job.nodes["n"] + records = node.get_all_records(all_info=True) + assert records is not None + source_columns = [c for c in records.column_names if c.startswith("_source")] + print( + f" [{label}] records: {records.num_rows} rows; source columns {source_columns}" + ) + if "_source_s" in records.column_names: + values = dict( + zip( + records.column("k").to_pylist(), records.column("_source_s").to_pylist() + ) + ) + print(f" [{label}] _source_s by row: {dict(sorted(values.items()))}") + table = node.as_table(all_info=True) + print(f" [{label}] as_table(all_info): {table.num_rows} rows") + print(f" [{label}] y: {sorted(table.column('y').to_pylist())}") + + +def attempt(label: str, append: Callable[[], tuple[PipelineJob, Any]]) -> None: + """Run ``append`` and describe the result, or print how it failed.""" + print(label) + try: + job, summary = append() + except BaseException as exc: # noqa: BLE001 # the measurement reports any failure + inner = exc.exceptions[0] if isinstance(exc, BaseExceptionGroup) else exc + print(" FAILS:", type(exc).__name__, "/", type(inner).__name__) + print(" ", str(inner).splitlines()[0][:220]) + return + print(" run:", str(summary).splitlines()[-1].strip()) + describe(job, label) + + +def phase1(out: Path) -> None: + """On main: async runs write r0..r3; the sync-after-async baseline.""" + shutil.rmtree(out, ignore_errors=True) + for name in ("delta_then_async", "delta_then_sync", "delta_main_sync_append"): + job, _ = run(DeltaTableDatabase(out / name), 4, "async") + describe(job, f"main, async write, {name}") + memory = InMemoryArrowDatabase() + job, _ = run(memory, 4, "async") + describe(job, "main, async write, memory") + dump_memory(memory, out / "memory_dump") + attempt( + "baseline on main: sync run appends r4, r5 to the async-written Delta table", + lambda: run(DeltaTableDatabase(out / "delta_main_sync_append"), 6, "sync"), + ) + other = InMemoryArrowDatabase() + run(other, 4, "async") + attempt( + "baseline on main: sync run appends r4, r5 to the async-written in-memory table", + lambda: run(other, 6, "sync"), + ) + + +def phase1b() -> None: + """On main: an async run appends r4, r5 to a sync-written table.""" + for kind in ("delta", "memory"): + store: Any = ( + DeltaTableDatabase(Path(tempfile.mkdtemp())) + if kind == "delta" + else InMemoryArrowDatabase() + ) + run(store, 4, "sync") + attempt( + f"baseline on main: async run appends to a sync-written {kind} table", + lambda store=store: run(store, 6, "async"), + ) + + +def phase2(out: Path) -> None: + """On the fixed build: append r4, r5 to the tables phase 1 wrote.""" + attempt( + "delta, fixed async append", + lambda: run(DeltaTableDatabase(out / "delta_then_async"), 6, "async"), + ) + attempt( + "delta, fixed sync append", + lambda: run(DeltaTableDatabase(out / "delta_then_sync"), 6, "sync"), + ) + attempt( + "memory, fixed async append", + lambda: run(load_memory(out / "memory_dump"), 6, "async"), + ) + attempt( + "memory, fixed sync append", + lambda: run(load_memory(out / "memory_dump"), 6, "sync"), + ) + + +def operator_node() -> None: + """A cached Join downstream of an async select: stored records, sync vs async.""" + + def stored(orchestrator: Any) -> pa.Table: + a = pa.table( + {"k": ["a", "b"], "x": [1, 2], "s": ["p", "q"]}, + schema=pa.schema( + [ + pa.field("k", pa.large_string(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + pa.field("s", pa.large_string(), nullable=False), + ] + ), + ) + b = pa.table( + {"k": ["a", "b"], "z": [10, 20]}, + schema=pa.schema( + [ + pa.field("k", pa.large_string(), nullable=False), + pa.field("z", pa.int64(), nullable=False), + ] + ), + ) + store = InMemoryArrowDatabase() + job = PipelineJob(name="j", store=store) + with job: + selected = SelectDataColumns(["x"])( + ArrowTableSource(a, tag_columns=["k"], source_id="a") + ) + Join()( + selected, + ArrowTableSource(b, tag_columns=["k"], source_id="b"), + label="op", + ) + job.nodes["op"].attach_databases( + pipeline_database=store.at("j"), cache_mode=CacheMode.LOG + ) + job.run(orchestrator=orchestrator) + records = job.nodes["op"].get_all_records(all_info=True) + assert records is not None + return records.sort_by("k") + + sync = stored(SyncPipelineOrchestrator()) + async_ = stored(AsyncPipelineOrchestrator()) + for name, table in (("sync", sync), ("async", async_)): + columns = [c for c in table.column_names if c.startswith("_source")] + print(f"{name}: stored operator records, source-info columns {columns}") + print("stored operator tables equal:", sync.equals(async_)) + + +if __name__ == "__main__": + command = sys.argv[1] + if command == "phase1": + phase1(Path(sys.argv[2])) + elif command == "phase1b": + phase1b() + elif command == "phase2": + phase2(Path(sys.argv[2])) + elif command == "operator_node": + operator_node() + else: + raise SystemExit(f"unknown command {command!r}") diff --git a/tests/test_core/datagrams/test_data_source_info_follows_columns.py b/tests/test_core/datagrams/test_data_source_info_follows_columns.py index 52040547d..2c06a22d9 100644 --- a/tests/test_core/datagrams/test_data_source_info_follows_columns.py +++ b/tests/test_core/datagrams/test_data_source_info_follows_columns.py @@ -1,4 +1,4 @@ -"""A ``Data``'s source info follows its data columns in every view (ITL-1040). +"""A ``Data``'s source info follows its data columns in every view (ITL-1040, ITL-838). ``Data.drop`` and ``Data.select`` used to remove a data column but keep its source info, which ``as_dict`` filtered out and ``as_table`` / ``arrow_schema`` did not. The async paths @@ -52,6 +52,8 @@ def assert_source_follows(d: Data, expected_columns: list[str]) -> None: assert d.keys() == tuple(expected_columns) config = {"source": True} assert source_columns(d.as_table(columns=config).column_names) == expected + assert source_columns(d.as_table(all_info=True).column_names) == expected + assert source_columns(d.arrow_schema(all_info=True).names) == expected assert source_columns(d.arrow_schema(columns=config).names) == expected assert source_columns(list(d.as_dict(columns=config))) == expected assert source_columns(list(d.schema(columns=config))) == expected @@ -74,6 +76,14 @@ def test_select_keeps_only_the_source_info_of_the_selected_columns( assert_source_follows(selected, ["x", "m"]) +@pytest.mark.parametrize("backing", BACKINGS) +def test_select_orders_the_source_info_as_the_columns(backing: str) -> None: + """Whatever column order ``select`` gives, the source info follows it.""" + reordered = data(backing).select("m", "x") + assert set(reordered.keys()) == {"m", "x"} + assert_source_follows(reordered, list(reordered.keys())) + + @pytest.mark.parametrize("backing", BACKINGS) def test_rename_renames_the_source_info(backing: str) -> None: renamed = data(backing).rename({"s": "t"}) diff --git a/tests/test_core/operators/test_async_sync_equivalence.py b/tests/test_core/operators/test_async_sync_equivalence.py index ed95210d5..9bb419820 100644 --- a/tests/test_core/operators/test_async_sync_equivalence.py +++ b/tests/test_core/operators/test_async_sync_equivalence.py @@ -643,26 +643,9 @@ def test_drop_nulls_without_tag_columns() -> None: assert data.source_info() == sync_data.source_info() -# Pre-existing provenance defects, not type re-inference: out of scope for -# ITL-837 and tracked elsewhere. Strict, so a fix flags them. -_DROPPED_SOURCE_INFO = ( - "ITL-838: Data.select / Data.drop keep the dropped columns' source info, so the " - "async row's source-info schema lists columns it no longer has" -) -PROVENANCE_XFAIL = { - "select_data_columns": _DROPPED_SOURCE_INFO, - "drop_data_columns": _DROPPED_SOURCE_INFO, -} -PROVENANCE_CASES = [ - pytest.param( - c, - id=c.label, - marks=[pytest.mark.xfail(strict=True, reason=PROVENANCE_XFAIL[c.label])] - if c.label in PROVENANCE_XFAIL - else [], - ) - for c in CASES -] +# Every case: the async path carries the sync path's source info (ITL-838 / ITL-1040 fixed +# the select and drop cases, whose dropped columns' source info used to survive). +PROVENANCE_CASES = [pytest.param(c, id=c.label) for c in CASES] def test_group_by_keeps_non_key_tags_as_tags_on_both_paths() -> None: From 3f107ac759f52c0053c29df272ac24bccf510fa4 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 23:37:18 -0800 Subject: [PATCH 4/4] fix(datagrams): a Data's source-info table is in the order of its data columns (ITL-1040) Source info given in another order than the columns (or left in that order by drop) was shown in insertion order by as_table and arrow_schema, while as_dict, keys and schema used the column order. The table is now built in data-column order. Found by CodeRabbit on #350. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/datagrams/tag_data.py | 11 +++++++---- .../test_data_source_info_follows_columns.py | 17 +++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/orcapod/core/datagrams/tag_data.py b/src/orcapod/core/datagrams/tag_data.py index ab324e0ef..42f8cab0f 100644 --- a/src/orcapod/core/datagrams/tag_data.py +++ b/src/orcapod/core/datagrams/tag_data.py @@ -367,10 +367,13 @@ def _ensure_source_info_table(self) -> "pa.Table": if self._source_info_table is None: import pyarrow as _pa - # Only the source info of the data columns this Data holds: a column it - # no longer has has no source info in any view (ITL-1040). - data_keys = set(super().keys()) - current = {k: v for k, v in self._source_info.items() if k in data_keys} + # Only the source info of the data columns this Data holds, in the order of + # those columns, as every other view gives it (ITL-1040). + current = { + k: self._source_info[k] + for k in super().keys() + if k in self._source_info + } if current: prefixed = { f"{constants.SOURCE_PREFIX}{k}": v for k, v in current.items() diff --git a/tests/test_core/datagrams/test_data_source_info_follows_columns.py b/tests/test_core/datagrams/test_data_source_info_follows_columns.py index 2c06a22d9..66259e673 100644 --- a/tests/test_core/datagrams/test_data_source_info_follows_columns.py +++ b/tests/test_core/datagrams/test_data_source_info_follows_columns.py @@ -125,3 +125,20 @@ def test_tag_drop_keeps_the_system_tags_which_belong_to_the_row() -> None: table = dropped.as_table(columns={"system_tags": True}) assert table.column_names == ["a", system_tag] assert dropped.system_tags() == {system_tag: "src"} + + +@pytest.mark.parametrize("operation", ["construct", "drop", "select"]) +def test_source_info_given_out_of_column_order_follows_the_columns( + operation: str, +) -> None: + """Source info given in another order than the columns is shown in column order.""" + d = Data( + {"x": 1, "s": "p", "m": 2.5}, + source_info={"m": "src::row_0::m", "s": "src::row_0::s", "x": "src::row_0::x"}, + ) + if operation == "drop": + d = d.drop("s") + elif operation == "select": + d = d.select("x", "m") + expected = [c for c in ("x", "s", "m") if c in d.keys()] + assert_source_follows(d, expected)