From 43bb9375139c64c0d03c5ec44f6c9891cfbd7f16 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 15:29:57 -0800 Subject: [PATCH 01/12] fix(function_node): as_table keeps each row's Arrow types (ITL-928) A function node's as_table() rebuilt its table from tag.as_dict() Python values typed by the node's Python output schema, so an int8 or uint32 tag came back int64, a naive timestamp[us] came back timestamp[us, tz=UTC], and a time, duration, decimal, uuid or Path tag made it raise. It now builds the table from each row's own Arrow tables, which also restores the _source_ and _context_key columns that as_table(all_info=True) of the same pod outside a job has. The zero-row table is unchanged. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/function_node.py | 52 ++-- .../2026-11-12-itl-928-node-tag-types.md | 129 ++++++++++ .../test_function_node_arrow_types.py | 230 ++++++++++++++++++ .../test_function_pod_auto_select.py | 8 +- 4 files changed, 386 insertions(+), 33 deletions(-) create mode 100644 superpowers/plans/2026-11-12-itl-928-node-tag-types.md create mode 100644 tests/test_core/function_pod/test_function_node_arrow_types.py diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 888d0b0df..b2b6abd20 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -455,51 +455,39 @@ def as_table( all_info: bool = False, ) -> "pa.Table": if self._cached_output_table is None: - all_tags: list[dict] = [] - all_data: list[dict] = [] - # Derive Arrow schemas from the declared output_schema upfront so that - # the cached table always has a consistent, well-formed schema regardless - # of whether any rows were produced. - tag_python_schema, data_python_schema = self.output_schema(all_info=True) - for tag, data in self.iter_data(): - all_tags.append(tag.as_dict(all_info=True)) - all_data.append(data.as_dict(all_info=True)) - - converter = self.data_context.type_converter - tag_arrow_schema = converter.python_schema_to_arrow_schema( - tag_python_schema - ) - data_arrow_schema = converter.python_schema_to_arrow_schema( - data_python_schema - ) - - if not all_tags: + rows = list(self.iter_data()) + if not rows: # No data was produced (node has not been run yet or produced zero # outputs). Build a zero-row table whose schema is derived from the # pod's declared output so that callers always receive a well-formed # schema regardless of whether the node has actually executed. + tag_python_schema, data_python_schema = self.output_schema( + all_info=True + ) self._cached_output_table = pa.Table.from_pylist( [], - schema=converter.python_schema_to_arrow_schema( + schema=self.data_context.type_converter.python_schema_to_arrow_schema( tag_python_schema + data_python_schema ), ) else: - struct_data = converter.python_dicts_to_struct_dicts( - all_data, python_schema=data_python_schema + # Built from each row's own Arrow tables, so every column keeps the + # type its row has: a tag the type of the input's tag, never a type + # inferred from Python values (ITL-928). + # A row's meta columns (a computed row's ``__computed`` flag) are its + # own bookkeeping, not part of the node's output, as for the pod's + # stream outside a job. + tag_table = arrow_utils.concat_tables_by_name( + [tag.as_table(columns={"system_tags": True}) for tag, _ in rows] ) - all_tags_as_tables: pa.Table = pa.Table.from_pylist( - all_tags, schema=tag_arrow_schema - ) - if constants.CONTEXT_KEY in all_tags_as_tables.column_names: - all_tags_as_tables = all_tags_as_tables.drop( - [constants.CONTEXT_KEY] - ) - all_data_as_tables: pa.Table = pa.Table.from_pylist( - struct_data, schema=data_arrow_schema + data_table = arrow_utils.concat_tables_by_name( + [ + data.as_table(columns={"source": True, "context": True}) + for _, data in rows + ] ) self._cached_output_table = arrow_utils.hstack_tables( - all_tags_as_tables, all_data_as_tables + tag_table, data_table ) column_config = ColumnConfig.handle_config(columns, all_info=all_info) diff --git a/superpowers/plans/2026-11-12-itl-928-node-tag-types.md b/superpowers/plans/2026-11-12-itl-928-node-tag-types.md new file mode 100644 index 000000000..d12bb6443 --- /dev/null +++ b/superpowers/plans/2026-11-12-itl-928-node-tag-types.md @@ -0,0 +1,129 @@ +# ITL-928: function node and Delta store keep tag Arrow types — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** A function node's `as_table()` returns the input's exact (large-normalized) Arrow types, and the Delta store gives back the Arrow types it was given, so joins after a function node work and a re-run on a Delta store writes no duplicate pipeline rows. + +**Architecture:** Two independent defects (audit sent to the coordinator on 2026-11-12, `.untracked/ask1.md`). +Part A: `FunctionNodeBase.as_table` (`src/orcapod/core/nodes/function_node.py`) builds its +table from each row's Arrow tables instead of Python values plus a type inferred from the +Python output schema. Part B: `DeltaTableDatabase` records a column's original Arrow type in +top-level field metadata when Delta would store it differently, writes unsigned integers (and, +per Decision 1 (i), ns timestamps, date64, time, duration) as a same-width integer view, and +casts back on every read. + +**Tech Stack:** pyarrow, deltalake (delta-rs), pytest, `uv run`. + +## Status / decisions (keep updated for a successor) + +- 2026-11-12: Part A APPROVED by the coordinator (own commits, do not push until told). +- Part B: with the developer (stored-format decision). Recommended: Decision 1 (i) integer + view for ns / date64 / time / duration; Decision 2 (i) document that Delta stores written + before the fix must be rebuilt. **Write no Delta code until the coordinator replies.** B's + tests are prepared as strict xfails. +- Audit scripts: `.untracked/measure.py` (tags, all read points, `[sync|async] [memory|delta|sqlite]`), + `.untracked/measure2.py` (data columns, all-info columns, operator node, CachedSource), + `.untracked/rerun_ids.py` (consequence 2), `.untracked/repro_flush_poison.py`. + +## Files + +- Modify: `src/orcapod/core/nodes/function_node.py` (`FunctionNodeBase.as_table`, ~l.451-503). +- Create: `tests/test_core/function_pod/test_function_node_arrow_types.py` (Part A, plus + store read-back guards; Delta read-back cases are strict xfails until Part B). +- Part B (after approval): modify `src/orcapod/databases/delta_lake_databases.py` (write: + record type + integer view; reads: cast back; append check compares converted types); + create `tests/test_databases/test_delta_arrow_type_roundtrip.py`; loaded-job test in + `tests/test_pipeline/loaded_job/`. +- `changelog.d/ITL-928.md`, `design_issues/ITL-928-FN1.md` (and a DT entry for Part B). + +## The type matrix (tests) + +int8, int16, int32, int64, uint8, uint16, uint32, uint64, float32, float64, bool, +timestamp s/ms/us/ns naive, timestamp us UTC, timestamp us America/New_York, date32, date64, +time32[s], time64[us], duration[s], duration[us], string, large_string, binary, +decimal128(10,2), uuid, Path. Expected type = `arrow_utils.normalize_to_large_types(source type)`. + +Delta changes on main (Part B): every uint, timestamp s/ms/ns, timestamp with a non-UTC zone; +cannot write date64, time32/64, duration. + +### Task 1: failing tests for Part A + +**Files:** Create `tests/test_core/function_pod/test_function_node_arrow_types.py` + +- [ ] Parametrize over the matrix x {in-memory, Delta} x {sync, async}: source with tag `k` + (3 distinct values, non-null) and data `x: int64` -> `double.pod` in a `PipelineJob` -> run. + Assert `job.nodes["double"].as_table().schema.field("k").type` is the expected type. + Delta + (date64, time, duration): strict xfail (store cannot write them; Part B). +- [ ] `as_table(all_info=True)` of the node after the run has the same schema as the same + pod's stream outside a job (`double.pod(src).as_table(all_info=True)`): `_source_y` and + `_context_key` present (in-memory store, a few types). +- [ ] Join after a function node: `job.nodes["double"].join(other_source)` on a `uint32` and a + `timestamp[us]` tag returns 3 rows, `k` typed as the source's. +- [ ] A node whose rows mix store-loaded and freshly computed entries (run with 2 rows, then + a new job with 3 rows on the same in-memory store) returns 3 rows with the input types. +- [ ] Run: `uv run pytest tests/test_core/function_pod/test_function_node_arrow_types.py -q`; + expected: failures for every type other than int64, float64, bool, ts[us,UTC], date32, + string, large_string, binary; errors for time/duration/decimal/uuid/Path. + +### Task 2: implement Part A + +**Files:** `src/orcapod/core/nodes/function_node.py` + +- [ ] Replace the non-empty branch of `FunctionNodeBase.as_table`: + +```python +rows = list(self.iter_data()) +if not rows: + ... # unchanged zero-row table from the declared output schema (reported, not fixed) +else: + # Each row's own Arrow tables: the input's tag types and the data's stored types, + # never a type inferred from Python values (ITL-928). + tag_table = arrow_utils.concat_tables_by_name( + [tag.as_table(all_info=True) for tag, _ in rows] + ) + if constants.CONTEXT_KEY in tag_table.column_names: + tag_table = tag_table.drop([constants.CONTEXT_KEY]) + data_table = arrow_utils.concat_tables_by_name( + [data.as_table(all_info=True) for _, data in rows] + ) + self._cached_output_table = arrow_utils.hstack_tables(tag_table, data_table) +``` + + Keep nullability of `_source_*` fields consistent with `FunctionPodStream.as_table` + (check with the all_info test; adjust field flags without a cast if needed). +- [ ] Run Task 1 tests: pass (except the strict xfails). Run + `uv run pytest tests/test_core tests/test_pipeline -q -x`. +- [ ] Commit `fix(function_node): as_table keeps each row's Arrow types (ITL-928)`. + +### Task 3: failing tests for Part B (strict xfail until approved) + +**Files:** `tests/test_core/function_pod/test_function_node_arrow_types.py`, +`tests/test_databases/test_delta_arrow_type_roundtrip.py` + +- [ ] Store read-back over the matrix x {in-memory, Delta} x {sync, async}: after the run, + `get_all_records()` and a fresh job's `iter_data()` / `as_table()` on the same store keep + the type. In-memory passes today (guard); Delta + changed types: strict xfail "ITL-928 part B". +- [ ] Re-run writes nothing: source(uint32 / ts[s] / ts[ns] / ts[us, NY] tag) -> pod -> pod, + run twice on one store; a fresh job reads 3 rows from the second node and + `get_all_records(all_info=True)` of each node still has 3 rows. Delta: strict xfail. +- [ ] `DeltaTableDatabase` round trip per type, a `uint32` value above int32 max, a + `timestamp[ns]` with sub-microsecond digits; `OperatorJobNode` REPLAY and `CachedSource` + on Delta keep types. Strict xfail. +- [ ] Saved-and-loaded job on a Delta store (separate process, as `loaded_job`) reads the + input tag types. Strict xfail. +- [ ] Commit `test(delta): Arrow type round trip through the Delta store (ITL-928 part B, xfail)`. + +### Task 4 (after approval only): implement Part B + +- [ ] Write side (`add_records` / `_flush_by_key`): for each top-level column whose storage + type the Delta writer converts (`_as_delta_writer_stores`), record the original type in + field metadata (one key, IPC-serialized one-field schema), write unsigned and the types of + Decision 1 as a same-width integer view; the append check compares converted types. +- [ ] Read side (every read method): cast/view each recorded column back, drop the key. +- [ ] Remove the xfail marks; full suite; commit `fix(delta): ...`. + +### Task 5: finishing + +- [ ] `design_issues/ITL-928-FN1.md` (+ DT entry), `changelog.d/ITL-928.md`. +- [ ] ruff format/check, basedpyright exit code, full `uv run pytest tests/`, golden tests unchanged. +- [ ] Merge origin/main, push once when the coordinator says, PR, readiness pipeline. diff --git a/tests/test_core/function_pod/test_function_node_arrow_types.py b/tests/test_core/function_pod/test_function_node_arrow_types.py new file mode 100644 index 000000000..e578a91ba --- /dev/null +++ b/tests/test_core/function_pod/test_function_node_arrow_types.py @@ -0,0 +1,230 @@ +"""A function node in a job keeps the Arrow type of every tag column (ITL-928). + +A function pod never touches tags, so a function node's tag columns must have the +input's exact Arrow types (large-normalized) whatever the store, the orchestrator and +the read: ``as_table()``, ``iter_data()``, ``get_all_records()``, a fresh job on the same +store, and a re-run. Before ITL-928 a node's ``as_table()`` rebuilt its table from Python +values typed by the node's Python output schema, so an ``int8`` tag came back ``int64`` +and a naive ``timestamp[us]`` came back ``timestamp[us, tz=UTC]``. +""" + +from __future__ import annotations + +import datetime as dt +import decimal +import pathlib +import uuid +from collections.abc import Callable +from typing import Any + +import pyarrow as pa +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.contexts import resolve_context +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.pipeline import AsyncPipelineOrchestrator +from orcapod.sources import ArrowTableSource +from orcapod.utils import arrow_utils + +_T0 = dt.datetime(2026, 3, 24, 17, 54, 17) +_DAYS = [_T0 + dt.timedelta(days=i) for i in range(3)] + + +def _logical(py_type: type, values: list[Any]) -> pa.Array: + """A column of a logical type, built by the type converter.""" + converter = resolve_context(None).type_converter + table = converter.python_dicts_to_arrow_table( + [{"v": v} for v in values], python_schema={"v": py_type} + ) + return table.column("v").combine_chunks() + + +# One column of three distinct, non-null values per Arrow type. +TAG_COLUMNS: dict[str, Callable[[], pa.Array]] = { + "int8": lambda: pa.array([1, 2, 3], pa.int8()), + "int16": lambda: pa.array([1, 2, 3], pa.int16()), + "int32": lambda: pa.array([1, 2, 3], pa.int32()), + "int64": lambda: pa.array([1, 2, 3], pa.int64()), + "uint8": lambda: pa.array([1, 2, 3], pa.uint8()), + "uint16": lambda: pa.array([1, 2, 3], pa.uint16()), + "uint32": lambda: pa.array([1, 2, 3], pa.uint32()), + "uint64": lambda: pa.array([1, 2, 3], pa.uint64()), + "float32": lambda: pa.array([1.5, 2.5, 3.5], pa.float32()), + "float64": lambda: pa.array([1.5, 2.5, 3.5], pa.float64()), + "bool": lambda: pa.array([True, False, True], pa.bool_()), + "timestamp[s]": lambda: pa.array(_DAYS, pa.timestamp("s")), + "timestamp[ms]": lambda: pa.array(_DAYS, pa.timestamp("ms")), + "timestamp[us]": lambda: pa.array(_DAYS, pa.timestamp("us")), + "timestamp[ns]": lambda: pa.array(_DAYS, pa.timestamp("ns")), + "timestamp[us, UTC]": lambda: pa.array(_DAYS, pa.timestamp("us", tz="UTC")), + "timestamp[us, New_York]": lambda: pa.array( + _DAYS, pa.timestamp("us", tz="America/New_York") + ), + "date32": lambda: pa.array([d.date() for d in _DAYS], pa.date32()), + "date64": lambda: pa.array([d.date() for d in _DAYS], pa.date64()), + "time32[s]": lambda: pa.array([dt.time(1), dt.time(2), dt.time(3)], pa.time32("s")), + "time64[us]": lambda: pa.array( + [dt.time(1), dt.time(2), dt.time(3)], pa.time64("us") + ), + "duration[s]": lambda: pa.array( + [dt.timedelta(seconds=i) for i in (1, 2, 3)], pa.duration("s") + ), + "duration[us]": lambda: pa.array( + [dt.timedelta(seconds=i) for i in (1, 2, 3)], pa.duration("us") + ), + "string": lambda: pa.array(["a", "b", "c"], pa.string()), + "large_string": lambda: pa.array(["a", "b", "c"], pa.large_string()), + "binary": lambda: pa.array([b"a", b"b", b"c"], pa.binary()), + "decimal128(10, 2)": lambda: pa.array( + [decimal.Decimal(v) for v in ("1.25", "2.50", "3.75")], pa.decimal128(10, 2) + ), + "uuid": lambda: _logical(uuid.UUID, [uuid.UUID(int=i) for i in (1, 2, 3)]), + "Path": lambda: _logical(pathlib.Path, [pathlib.Path(p) for p in "abc"]), +} + +# Types the Delta store cannot write at all (part B of ITL-928). +DELTA_CANNOT_WRITE = { + "date64", + "time32[s]", + "time64[us]", + "duration[s]", + "duration[us]", +} +# Types the Delta store writes as another type, so a read back differs (part B). +DELTA_CHANGES = { + "uint8", + "uint16", + "uint32", + "uint64", + "timestamp[s]", + "timestamp[ms]", + "timestamp[ns]", + "timestamp[us, New_York]", +} +PART_B = "ITL-928 part B: the Delta store does not keep this Arrow type" + + +@function_pod(output_keys="y") +def double(x: int) -> int: + """Twice ``x``.""" + return 2 * x + + +@function_pod(output_keys="z") +def increment(y: int) -> int: + """``y`` plus one.""" + return y + 1 + + +def _source_table(name: str, data_column: str = "x") -> pa.Table: + """A three-row table with tag ``k`` of the named type and an ``int64`` data column.""" + k = TAG_COLUMNS[name]() + schema = pa.schema( + [ + pa.field("k", k.type, nullable=False), + pa.field(data_column, pa.int64(), nullable=False), + ] + ) + return pa.Table.from_arrays([k, pa.array([1, 2, 3], pa.int64())], schema=schema) + + +def _expected(name: str) -> pa.DataType: + """The type a pass-through tag column must have: the large-normalized input type.""" + return arrow_utils.normalize_to_large_types(TAG_COLUMNS[name]().type) + + +def _new_store(kind: str, tmp_path: pathlib.Path): + if kind == "delta": + return DeltaTableDatabase(tmp_path / "store") + return InMemoryArrowDatabase() + + +def _run(job: PipelineJob, mode: str) -> None: + if mode == "async": + job.run(orchestrator=AsyncPipelineOrchestrator()) + else: + job.run() + + +def _build(store, table: pa.Table, *, chained: bool = False) -> PipelineJob: + """A job: source -> ``double`` (-> ``increment`` when chained).""" + with PipelineJob(name="types", store=store) as job: + src = ArrowTableSource(table, tag_columns=["k"], source_id="src") + out = double.pod(src, label="double") + if chained: + increment.pod(out, label="increment") + return job + + +def _xfail_delta(request: pytest.FixtureRequest, name: str, store: str) -> None: + if store == "delta" and name in DELTA_CANNOT_WRITE: + request.applymarker(pytest.mark.xfail(reason=PART_B, strict=True)) + + +# --------------------------------------------------------------------------- +# as_table() after a run: the node's own rows (part A) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("mode", ["sync", "async"]) +@pytest.mark.parametrize("store", ["memory", "delta"]) +@pytest.mark.parametrize("name", list(TAG_COLUMNS)) +def test_as_table_after_run_keeps_tag_type( + name: str, store: str, mode: str, tmp_path: pathlib.Path, request +) -> None: + _xfail_delta(request, name, store) + job = _build(_new_store(store, tmp_path), _source_table(name)) + _run(job, mode) + + table = job.nodes["double"].as_table() + + assert table.schema.field("k").type == _expected(name) + assert table.num_rows == 3 + + +@pytest.mark.parametrize("name", ["int64", "uint32", "timestamp[us]"]) +def test_as_table_all_info_matches_the_pod_outside_a_job(name: str) -> None: + """With every column group, the node's table equals the pod's stream outside a job.""" + table = _source_table(name) + outside = double.pod( + ArrowTableSource(table, tag_columns=["k"], source_id="src") + ).as_table(all_info=True) + job = _build(InMemoryArrowDatabase(), table) + job.run() + + inside = job.nodes["double"].as_table(all_info=True) + + assert inside.schema == outside.schema + # A source-info token names the result record, which each computation makes anew. + source_info = [c for c in inside.column_names if c.startswith("_source_")] + assert source_info == ["_source_y"] + assert inside.drop(source_info).equals(outside.drop(source_info)) + + +@pytest.mark.parametrize("name", ["uint32", "timestamp[us]"]) +def test_join_after_a_function_node_on_a_typed_tag(name: str) -> None: + """The ITL-990 case: join a function node's output with another source on the tag.""" + job = _build(InMemoryArrowDatabase(), _source_table(name)) + job.run() + other = ArrowTableSource(_source_table(name, "w"), tag_columns=["k"], source_id="o") + + joined = job.nodes["double"].join(other).as_table() + + assert joined.num_rows == 3 + assert joined.schema.field("k").type == _expected(name) + + +def test_as_table_mixing_stored_and_new_rows_keeps_tag_type() -> None: + """Rows served from the store and rows computed now make one correctly typed table.""" + store = InMemoryArrowDatabase() + first = _build(store, _source_table("uint32").slice(0, 2)) + first.run() + second = _build(store, _source_table("uint32")) + second.run() + + table = second.nodes["double"].as_table() + + assert table.num_rows == 3 + assert table.schema.field("k").type == pa.uint32() + assert sorted(table.column("k").to_pylist()) == [1, 2, 3] diff --git a/tests/test_core/function_pod/test_function_pod_auto_select.py b/tests/test_core/function_pod/test_function_pod_auto_select.py index edca9d26a..10e4cf777 100644 --- a/tests/test_core/function_pod/test_function_pod_auto_select.py +++ b/tests/test_core/function_pod/test_function_pod_auto_select.py @@ -706,7 +706,13 @@ def test_sync_and_async_runs_give_the_same_rows() -> None: tables.append(job.nodes["add"].as_table(all_info=True).sort_by("k")) assert tables[0].schema == tables[1].schema - assert tables[0].to_pylist() == tables[1].to_pylist() + # A source-info token names the result record, which each run makes anew. + source_info = [c for c in tables[0].column_names if c.startswith("_source_")] + assert source_info == ["_source_total"] + assert ( + tables[0].drop(source_info).to_pylist() + == tables[1].drop(source_info).to_pylist() + ) assert [row["total"] for row in tables[0].to_pylist()] == [5, 9] From ecc879b70a4bb9cb03ff3946adbe70340dd7a8a8 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 15:29:57 -0800 Subject: [PATCH 02/12] fix(function_pod): a pod stream's table keeps logical-type tags (ITL-928) FunctionPodStream.as_table built its tag table from tag.as_dict() Python values, which raises 'Expected bytes, got a UUID object' for a uuid or Path tag. It now takes each row's tag Arrow table. Data columns are unchanged. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/function_pod.py | 22 +++++++++---------- .../test_function_node_arrow_types.py | 15 ++++++++++++- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/src/orcapod/core/function_pod.py b/src/orcapod/core/function_pod.py index cdbac803e..d6380f3d4 100644 --- a/src/orcapod/core/function_pod.py +++ b/src/orcapod/core/function_pod.py @@ -1103,16 +1103,15 @@ def as_table( all_info: bool = False, ) -> pa.Table: if self._cached_output_table is None: - all_tags = [] + tag_tables = [] all_data = [] - tag_schema, data_schema = None, None + data_schema = None for tag, data in self.iter_data(): - if tag_schema is None: - tag_schema = tag.arrow_schema(all_info=True) if data_schema is None: data_schema = data.arrow_schema(all_info=True) - # TODO: make use of arrow_compat dict - all_tags.append(tag.as_dict(all_info=True)) + # Tags are passed through untouched: each row's own Arrow table keeps + # the input's tag types, a logical type included (ITL-928). + tag_tables.append(tag.as_table(columns={"system_tags": True})) all_data.append(data.as_dict(all_info=True)) # TODO: re-verify the implemetation of this conversion @@ -1128,12 +1127,11 @@ def as_table( struct_data = converter.python_dicts_to_struct_dicts( all_data, python_schema=python_schema ) - all_tags_as_tables: pa.Table = pa.Table.from_pylist( - all_tags, schema=tag_schema - ) - # drop context key column from tags table (guard: column absent on empty stream) - if constants.CONTEXT_KEY in all_tags_as_tables.column_names: - all_tags_as_tables = all_tags_as_tables.drop([constants.CONTEXT_KEY]) + if tag_tables: + all_tags_as_tables = arrow_utils.concat_tables_by_name(tag_tables) + else: + # An empty stream has no row to take a tag table from. + all_tags_as_tables = pa.Table.from_pylist([]) all_data_as_tables: pa.Table = pa.Table.from_pylist( struct_data, schema=data_schema ) diff --git a/tests/test_core/function_pod/test_function_node_arrow_types.py b/tests/test_core/function_pod/test_function_node_arrow_types.py index e578a91ba..5f2c5e861 100644 --- a/tests/test_core/function_pod/test_function_node_arrow_types.py +++ b/tests/test_core/function_pod/test_function_node_arrow_types.py @@ -183,7 +183,20 @@ def test_as_table_after_run_keeps_tag_type( assert table.num_rows == 3 -@pytest.mark.parametrize("name", ["int64", "uint32", "timestamp[us]"]) +@pytest.mark.parametrize("name", list(TAG_COLUMNS)) +def test_pod_stream_outside_a_job_keeps_tag_type(name: str) -> None: + """The same pod outside a job: its stream's table keeps the tag type too.""" + stream = double.pod( + ArrowTableSource(_source_table(name), tag_columns=["k"], source_id="src") + ) + + table = stream.as_table() + + assert table.schema.field("k").type == _expected(name) + assert table.num_rows == 3 + + +@pytest.mark.parametrize("name", ["int64", "uint32", "timestamp[us]", "uuid", "Path"]) def test_as_table_all_info_matches_the_pod_outside_a_job(name: str) -> None: """With every column group, the node's table equals the pod's stream outside a job.""" table = _source_table(name) From 3105d3ddf63d1bee859e3fa69c35dca28dedaae9 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 15:31:51 -0800 Subject: [PATCH 03/12] test(delta): Arrow type round trip through the Delta store (ITL-928 part B, xfail) Strict expected failures until part B: a Delta read returns the written type for unsigned integers, s/ms/ns and zoned timestamps, nested columns, and stores date64, time and duration; a re-run on a Delta store writes no pipeline rows; a fresh job, a loaded job, an operator replay and a cached source read the tag type. Co-Authored-By: Claude Opus 5.5 --- .../2026-11-12-itl-928-node-tag-types.md | 4 + .../test_function_node_arrow_types.py | 29 ++ .../test_delta_arrow_type_roundtrip.py | 315 ++++++++++++++++++ 3 files changed, 348 insertions(+) create mode 100644 tests/test_databases/test_delta_arrow_type_roundtrip.py diff --git a/superpowers/plans/2026-11-12-itl-928-node-tag-types.md b/superpowers/plans/2026-11-12-itl-928-node-tag-types.md index d12bb6443..d46a354fb 100644 --- a/superpowers/plans/2026-11-12-itl-928-node-tag-types.md +++ b/superpowers/plans/2026-11-12-itl-928-node-tag-types.md @@ -21,6 +21,10 @@ casts back on every read. view for ns / date64 / time / duration; Decision 2 (i) document that Delta stores written before the fix must be rebuilt. **Write no Delta code until the coordinator replies.** B's tests are prepared as strict xfails. +- 2026-11-12: Part A committed (43bb9375), plus FunctionPodStream.as_table logical-type + tags (ecc879b7, approved by the coordinator as its own commit). Part B tests committed as + strict xfails (`tests/test_databases/test_delta_arrow_type_roundtrip.py`, + `test_store_read_back_keeps_tag_type`). Nothing pushed yet (coordinator: do not push). - Audit scripts: `.untracked/measure.py` (tags, all read points, `[sync|async] [memory|delta|sqlite]`), `.untracked/measure2.py` (data columns, all-info columns, operator node, CachedSource), `.untracked/rerun_ids.py` (consequence 2), `.untracked/repro_flush_poison.py`. diff --git a/tests/test_core/function_pod/test_function_node_arrow_types.py b/tests/test_core/function_pod/test_function_node_arrow_types.py index 5f2c5e861..9e96d02cf 100644 --- a/tests/test_core/function_pod/test_function_node_arrow_types.py +++ b/tests/test_core/function_pod/test_function_node_arrow_types.py @@ -241,3 +241,32 @@ def test_as_table_mixing_stored_and_new_rows_keeps_tag_type() -> None: assert table.num_rows == 3 assert table.schema.field("k").type == pa.uint32() assert sorted(table.column("k").to_pylist()) == [1, 2, 3] + + +# --------------------------------------------------------------------------- +# Reads from the store: get_all_records() and a fresh job (part B on Delta) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("store", ["memory", "delta"]) +@pytest.mark.parametrize("name", list(TAG_COLUMNS)) +def test_store_read_back_keeps_tag_type( + name: str, store: str, tmp_path: pathlib.Path, request +) -> None: + if store == "delta" and name in DELTA_CANNOT_WRITE | DELTA_CHANGES: + request.applymarker(pytest.mark.xfail(reason=PART_B, strict=True)) + database = _new_store(store, tmp_path) + table = _source_table(name) + _build(database, table).run() + if store == "delta": + # A new store object sees the committed tables, as a new process would. + database = _new_store(store, tmp_path) + + fresh = _build(database, table).nodes["double"] + + records = fresh.get_all_records() + assert records is not None + assert records.schema.field("k").type == _expected(name) + tag, _ = next(iter(fresh.iter_data())) + assert tag.as_table().schema.field("k").type == _expected(name) + assert fresh.as_table().schema.field("k").type == _expected(name) diff --git a/tests/test_databases/test_delta_arrow_type_roundtrip.py b/tests/test_databases/test_delta_arrow_type_roundtrip.py new file mode 100644 index 000000000..0a71ff729 --- /dev/null +++ b/tests/test_databases/test_delta_arrow_type_roundtrip.py @@ -0,0 +1,315 @@ +"""The Delta store gives back the Arrow types it was given (ITL-928, part B). + +Delta Lake has no unsigned integer, nanosecond, second or millisecond timestamp, or +zoned (other than UTC) timestamp type, and the Delta writer converts such columns. A +read must still return the column's written type (large-normalized) and values: a tag +read back with another type changes the record ids computed from it, so a re-run +writes its pipeline rows a second time. +""" + +from __future__ import annotations + +import datetime as dt +import decimal +import pathlib + +import pyarrow as pa +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.core.operators import SelectDataColumns +from orcapod.core.sources import CachedSource +from orcapod.databases import DeltaTableDatabase +from orcapod.sources import ArrowTableSource +from orcapod.types import CacheMode +from orcapod.utils import arrow_utils + +PART_B = "ITL-928 part B: the Delta store does not keep this Arrow type" + +_T = [ + dt.datetime(2026, 3, 24, 17, 54, 17, 123456) + dt.timedelta(days=i) + for i in range(3) +] + +# Column name -> three values of that type, chosen within what the type holds. +COLUMNS: dict[str, pa.Array] = { + "int8": pa.array([-1, 0, 1], pa.int8()), + "int64": pa.array([-1, 0, 1], pa.int64()), + "uint8": pa.array([0, 1, 2], pa.uint8()), + "uint16": pa.array([0, 1, 2], pa.uint16()), + "uint32": pa.array([0, 1, 2], pa.uint32()), + "uint64": pa.array([0, 1, 2], pa.uint64()), + "float32": pa.array([0.5, 1.5, 2.5], pa.float32()), + "bool": pa.array([True, False, True], pa.bool_()), + "timestamp[s]": pa.array([t.replace(microsecond=0) for t in _T], pa.timestamp("s")), + "timestamp[ms]": pa.array( + [t.replace(microsecond=123000) for t in _T], pa.timestamp("ms") + ), + "timestamp[us]": pa.array(_T, pa.timestamp("us")), + "timestamp[us, UTC]": pa.array(_T, pa.timestamp("us", tz="UTC")), + "timestamp[us, New_York]": pa.array(_T, pa.timestamp("us", tz="America/New_York")), + "date32": pa.array([t.date() for t in _T], pa.date32()), + "large_string": pa.array(["a", "b", "c"], pa.large_string()), + "large_binary": pa.array([b"a", b"b", b"c"], pa.large_binary()), + "decimal128(10, 2)": pa.array( + [decimal.Decimal(v) for v in ("1.25", "2.50", "3.75")], pa.decimal128(10, 2) + ), + "large_list": pa.array([[0], [1, 2], []], pa.large_list(pa.uint32())), + "struct": pa.array( + [{"u": i, "t": _T[i].replace(microsecond=0)} for i in range(3)], + pa.struct([("u", pa.uint16()), ("t", pa.timestamp("ms"))]), + ), +} +# Types the Delta writer converts on main: a read gives another type back. +CHANGED = { + "uint8", + "uint16", + "uint32", + "uint64", + "timestamp[s]", + "timestamp[ms]", + "timestamp[us, New_York]", + "large_list", + "struct", +} + + +def _written_and_read(tmp_path: pathlib.Path, column: pa.Array) -> pa.Table: + """Write one three-row column to a Delta store and read it back from a new store object.""" + DeltaTableDatabase(tmp_path).add_records( + ("t",), + pa.table({"id": pa.array([b"a", b"b", b"c"], pa.large_binary()), "c": column}), + record_id_column="id", + flush=True, + ) + read = DeltaTableDatabase(tmp_path).get_all_records(("t",), record_id_column="id") + assert read is not None + return read.sort_by("id") + + +@pytest.mark.parametrize("name", list(COLUMNS)) +def test_a_read_returns_the_written_type_and_values( + name: str, tmp_path: pathlib.Path, request: pytest.FixtureRequest +) -> None: + if name in CHANGED: + request.applymarker(pytest.mark.xfail(reason=PART_B, strict=True)) + column = COLUMNS[name] + + read = _written_and_read(tmp_path, column) + + assert read.schema.field("c").type == arrow_utils.normalize_to_large_types( + column.type + ) + assert read.column("c").to_pylist() == column.to_pylist() + + +@pytest.mark.xfail(reason=PART_B, strict=True) +@pytest.mark.parametrize( + "column", + [ + pa.array([0, 2**31, 2**32 - 1], pa.uint32()), + pa.array([0, 2**63, 2**64 - 1], pa.uint64()), + ], + ids=["uint32", "uint64"], +) +def test_an_unsigned_value_above_the_signed_range_round_trips( + column: pa.Array, tmp_path: pathlib.Path +) -> None: + read = _written_and_read(tmp_path, column) + + assert read.schema.field("c").type == column.type + assert read.column("c").to_pylist() == column.to_pylist() + + +@pytest.mark.xfail(reason=PART_B, strict=True) +def test_nanosecond_timestamps_keep_their_digits(tmp_path: pathlib.Path) -> None: + column = pa.array( + [1_700_000_000_123_456_789, 1, 1_700_000_000_000_000_001], pa.timestamp("ns") + ) + + read = _written_and_read(tmp_path, column) + + assert read.schema.field("c").type == pa.timestamp("ns") + assert read.column("c").equals(column) + + +@pytest.mark.xfail(reason=PART_B, strict=True) +@pytest.mark.parametrize( + "column", + [ + pa.array([dt.date(2026, 1, i) for i in (1, 2, 3)], pa.date64()), + pa.array([dt.time(i) for i in (1, 2, 3)], pa.time32("s")), + pa.array([dt.time(i) for i in (1, 2, 3)], pa.time64("us")), + pa.array([dt.timedelta(seconds=i) for i in (1, 2, 3)], pa.duration("us")), + ], + ids=["date64", "time32[s]", "time64[us]", "duration[us]"], +) +def test_types_delta_has_no_equivalent_for_are_stored( + column: pa.Array, tmp_path: pathlib.Path +) -> None: + read = _written_and_read(tmp_path, column) + + assert read.schema.field("c").type == column.type + assert read.column("c").to_pylist() == column.to_pylist() + + +# --------------------------------------------------------------------------- +# Through a job: re-runs, operator replay, cached sources, a loaded job +# --------------------------------------------------------------------------- + + +@function_pod(output_keys="y") +def double(x: int) -> int: + """Twice ``x``.""" + return 2 * x + + +@function_pod(output_keys="z") +def increment(y: int) -> int: + """``y`` plus one.""" + return y + 1 + + +def _source_table(k: pa.Array) -> pa.Table: + schema = pa.schema( + [ + pa.field("k", k.type, nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] + ) + return pa.Table.from_arrays([k, pa.array([1, 2, 3], pa.int64())], schema=schema) + + +TAGS = { + "int8": pa.array([1, 2, 3], pa.int8()), + "uint32": pa.array([1, 2, 3], pa.uint32()), + "timestamp[s]": pa.array(_T, pa.timestamp("s")), + "timestamp[ns]": pa.array(_T, pa.timestamp("ns")), + "timestamp[us, New_York]": pa.array(_T, pa.timestamp("us", tz="America/New_York")), +} + + +def _chained_job(root: pathlib.Path, table: pa.Table) -> PipelineJob: + with PipelineJob(name="j", store=DeltaTableDatabase(root)) as job: + src = ArrowTableSource(table, tag_columns=["k"], source_id="src") + increment.pod(double.pod(src, label="double"), label="increment") + return job + + +@pytest.mark.parametrize("mode", ["sync", "async"]) +@pytest.mark.parametrize("name", list(TAGS)) +def test_a_rerun_on_the_same_store_writes_no_pipeline_rows( + name: str, mode: str, tmp_path: pathlib.Path, request: pytest.FixtureRequest +) -> None: + """The ITL-990 case: source -> pod -> pod, run twice; a fresh job reads 3 rows.""" + if name != "int8": + request.applymarker(pytest.mark.xfail(reason=PART_B, strict=True)) + from orcapod.pipeline import AsyncPipelineOrchestrator + + table = _source_table(TAGS[name]) + for _ in range(2): + job = _chained_job(tmp_path, table) + if mode == "async": + job.run(orchestrator=AsyncPipelineOrchestrator()) + else: + job.run() + + fresh = _chained_job(tmp_path, table) + for label in ("double", "increment"): + node = fresh.nodes[label] + assert node.as_table().num_rows == 3 + assert node.as_table().schema.field("k").type == TAGS[name].type + records = node.get_all_records(all_info=True) + assert records is not None + assert records.num_rows == 3 + + +@pytest.mark.parametrize("name", list(TAGS)) +def test_a_fresh_job_reads_the_tag_type( + name: str, tmp_path: pathlib.Path, request: pytest.FixtureRequest +) -> None: + if name != "int8": + request.applymarker(pytest.mark.xfail(reason=PART_B, strict=True)) + table = _source_table(TAGS[name]) + _chained_job(tmp_path, table).run() + + fresh = _chained_job(tmp_path, table).nodes["double"] + + assert fresh.as_table().schema.field("k").type == TAGS[name].type + tag, _ = next(iter(fresh.iter_data())) + assert tag.as_table().schema.field("k").type == TAGS[name].type + records = fresh.get_all_records() + assert records is not None + assert records.schema.field("k").type == TAGS[name].type + + +@pytest.mark.parametrize("name", list(TAGS)) +def test_a_loaded_job_reads_the_tag_type( + name: str, tmp_path: pathlib.Path, request: pytest.FixtureRequest +) -> None: + if name != "int8": + request.applymarker(pytest.mark.xfail(reason=PART_B, strict=True)) + job = _chained_job(tmp_path / "store", _source_table(TAGS[name])) + job.run() + job.save(tmp_path / "job.json") + + loaded = PipelineJob.load( + tmp_path / "job.json", store=DeltaTableDatabase(tmp_path / "store") + ) + + for label in ("double", "increment"): + assert loaded.nodes[label].as_table().schema.field("k").type == TAGS[name].type + + +@pytest.mark.parametrize("name", list(TAGS)) +def test_an_operator_replay_reads_the_tag_type( + name: str, tmp_path: pathlib.Path, request: pytest.FixtureRequest +) -> None: + if name != "int8": + request.applymarker(pytest.mark.xfail(reason=PART_B, strict=True)) + table = pa.Table.from_arrays( + [TAGS[name], pa.array([1, 2, 3], pa.int64()), pa.array([4, 5, 6], pa.int64())], + schema=pa.schema( + [ + pa.field("k", TAGS[name].type, nullable=False), + pa.field("x", pa.int64(), nullable=False), + pa.field("w", pa.int64(), nullable=False), + ] + ), + ) + + def build(mode: CacheMode) -> PipelineJob: + store = DeltaTableDatabase(tmp_path) + with PipelineJob(name="o", store=store) as job: + src = ArrowTableSource(table, tag_columns=["k"], source_id="src") + SelectDataColumns(["x"])(src, label="sel") + # OperatorNode.attach_databases is the public way to turn on LOG / REPLAY mode. + job.nodes["sel"].attach_databases( + pipeline_database=store.at("o"), cache_mode=mode + ) + return job + + build(CacheMode.LOG).run() + + replayed = build(CacheMode.REPLAY).nodes["sel"].as_table() + + assert replayed.schema.field("k").type == TAGS[name].type + + +@pytest.mark.parametrize("name", list(TAGS)) +def test_a_cached_source_reads_the_tag_type( + name: str, tmp_path: pathlib.Path, request: pytest.FixtureRequest +) -> None: + if name != "int8": + request.applymarker(pytest.mark.xfail(reason=PART_B, strict=True)) + table = _source_table(TAGS[name]) + + def cached() -> CachedSource: + return CachedSource( + ArrowTableSource(table, tag_columns=["k"], source_id="src"), + cache_database=DeltaTableDatabase(tmp_path), + ) + + cached().as_table() + + assert cached().as_table().schema.field("k").type == TAGS[name].type From 1c36e9ba8d4b4f1b33391356ec7c52dc3bceced4 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 15:33:42 -0800 Subject: [PATCH 04/12] docs(design_issues): ITL-928 FN1 entry and changelog fragment; type-annotate the new tests Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-928.md | 4 +++ design_issues/ITL-928-FN1.md | 25 +++++++++++++++++++ .../test_function_node_arrow_types.py | 4 +-- .../test_delta_arrow_type_roundtrip.py | 13 +++++----- 4 files changed, 38 insertions(+), 8 deletions(-) create mode 100644 changelog.d/ITL-928.md create mode 100644 design_issues/ITL-928-FN1.md diff --git a/changelog.d/ITL-928.md b/changelog.d/ITL-928.md new file mode 100644 index 000000000..78d60e494 --- /dev/null +++ b/changelog.d/ITL-928.md @@ -0,0 +1,4 @@ +### Fixed + +- A function node's `as_table()` keeps the Arrow type of every tag column. Before, it rebuilt the table from Python values, so after `job.run()` an `int8` or `uint32` tag came back as `int64`, `float32` as `double`, a naive `timestamp[us]` as `timestamp[us, tz=UTC]` and `date64` as `date32`, and a `time`, `duration`, `decimal`, `uuid` or `Path` tag made it raise. Joining a function node's output with another source on such a tag no longer fails with "Join inputs disagree on the Arrow type". `as_table(all_info=True)` of a function node now also has the `_source_` and `_context_key` columns, as the same pod's stream outside a job does. +- The table of a function pod's stream (the pod applied outside a job) no longer raises for a `uuid` or `Path` tag ("Expected bytes, got a 'UUID' object"). diff --git a/design_issues/ITL-928-FN1.md b/design_issues/ITL-928-FN1.md new file mode 100644 index 000000000..b17992700 --- /dev/null +++ b/design_issues/ITL-928-FN1.md @@ -0,0 +1,25 @@ +## `src/orcapod/core/nodes/function_node.py` + +### FN1 — a function node's table rebuilt its tags from Python values with inferred types +**Status:** resolved +**Severity:** critical +**Issue:** ITL-928 + +`FunctionNodeBase.as_table` built its table from `tag.as_dict()` and `data.as_dict()` Python +values, typed by the node's Python output schema through the type converter. A function pod +never touches tags, yet the tag columns came back with types inferred from Python: every +integer width as `int64`, `float32` as `double`, every naive or zoned timestamp as +`timestamp[us, tz=UTC]`, `date64` as `date32`; a `time`, `duration`, `decimal`, `uuid` or +`Path` tag made it raise. It happened on every store, sync and async, while `iter_data()`, +`get_all_records()` and the stored pipeline rows kept the input types. Joining a node's output +with another source on such a tag raised "Join inputs disagree on the Arrow type". The same +build also dropped the `_source_` and `_context_key` columns that the pod's stream +outside a job has. `FunctionPodStream.as_table`, the same pod outside a job, built its tag +table from Python values too (typed with the tag's own Arrow schema), so only a `uuid` or +`Path` tag broke it ("Expected bytes, got a 'UUID' object"). + +**Fix:** both build their tag table from each row's own Arrow table (`tag.as_table()`), and the +node its data table from `data.as_table()` (source info and context, no meta columns), +concatenated by name. No stored value, record id or hash changes. The zero-row table of a +node that has not run is still built from the Python output schema (the ITL-912 problem, for +function nodes). diff --git a/tests/test_core/function_pod/test_function_node_arrow_types.py b/tests/test_core/function_pod/test_function_node_arrow_types.py index 9e96d02cf..ebd19c09d 100644 --- a/tests/test_core/function_pod/test_function_node_arrow_types.py +++ b/tests/test_core/function_pod/test_function_node_arrow_types.py @@ -31,7 +31,7 @@ _DAYS = [_T0 + dt.timedelta(days=i) for i in range(3)] -def _logical(py_type: type, values: list[Any]) -> pa.Array: +def _logical(py_type: type, values: list[Any]) -> pa.Array[Any]: """A column of a logical type, built by the type converter.""" converter = resolve_context(None).type_converter table = converter.python_dicts_to_arrow_table( @@ -41,7 +41,7 @@ def _logical(py_type: type, values: list[Any]) -> pa.Array: # One column of three distinct, non-null values per Arrow type. -TAG_COLUMNS: dict[str, Callable[[], pa.Array]] = { +TAG_COLUMNS: dict[str, Callable[[], pa.Array[Any]]] = { "int8": lambda: pa.array([1, 2, 3], pa.int8()), "int16": lambda: pa.array([1, 2, 3], pa.int16()), "int32": lambda: pa.array([1, 2, 3], pa.int32()), diff --git a/tests/test_databases/test_delta_arrow_type_roundtrip.py b/tests/test_databases/test_delta_arrow_type_roundtrip.py index 0a71ff729..7ce48be4e 100644 --- a/tests/test_databases/test_delta_arrow_type_roundtrip.py +++ b/tests/test_databases/test_delta_arrow_type_roundtrip.py @@ -12,6 +12,7 @@ import datetime as dt import decimal import pathlib +from typing import Any import pyarrow as pa import pytest @@ -32,7 +33,7 @@ ] # Column name -> three values of that type, chosen within what the type holds. -COLUMNS: dict[str, pa.Array] = { +COLUMNS: dict[str, pa.Array[Any]] = { "int8": pa.array([-1, 0, 1], pa.int8()), "int64": pa.array([-1, 0, 1], pa.int64()), "uint8": pa.array([0, 1, 2], pa.uint8()), @@ -74,7 +75,7 @@ } -def _written_and_read(tmp_path: pathlib.Path, column: pa.Array) -> pa.Table: +def _written_and_read(tmp_path: pathlib.Path, column: pa.Array[Any]) -> pa.Table: """Write one three-row column to a Delta store and read it back from a new store object.""" DeltaTableDatabase(tmp_path).add_records( ("t",), @@ -113,7 +114,7 @@ def test_a_read_returns_the_written_type_and_values( ids=["uint32", "uint64"], ) def test_an_unsigned_value_above_the_signed_range_round_trips( - column: pa.Array, tmp_path: pathlib.Path + column: pa.Array[Any], tmp_path: pathlib.Path ) -> None: read = _written_and_read(tmp_path, column) @@ -130,7 +131,7 @@ def test_nanosecond_timestamps_keep_their_digits(tmp_path: pathlib.Path) -> None read = _written_and_read(tmp_path, column) assert read.schema.field("c").type == pa.timestamp("ns") - assert read.column("c").equals(column) + assert read.column("c").to_pylist() == column.to_pylist() @pytest.mark.xfail(reason=PART_B, strict=True) @@ -145,7 +146,7 @@ def test_nanosecond_timestamps_keep_their_digits(tmp_path: pathlib.Path) -> None ids=["date64", "time32[s]", "time64[us]", "duration[us]"], ) def test_types_delta_has_no_equivalent_for_are_stored( - column: pa.Array, tmp_path: pathlib.Path + column: pa.Array[Any], tmp_path: pathlib.Path ) -> None: read = _written_and_read(tmp_path, column) @@ -170,7 +171,7 @@ def increment(y: int) -> int: return y + 1 -def _source_table(k: pa.Array) -> pa.Table: +def _source_table(k: pa.Array[Any]) -> pa.Table: schema = pa.schema( [ pa.field("k", k.type, nullable=False), From fe18942770a61f51f527b5bcf1ac9fec3d25cc70 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 15:51:53 -0800 Subject: [PATCH 05/12] fix(function_node): rows disagreeing on a column's Arrow type raise, naming it (ITL-928) A function node's table never takes a type one of its rows does not have. On a Delta store, until it keeps unsigned and non-microsecond or zoned timestamp types, rows read from the store and rows computed in the same run can disagree: as_table() now raises InputValidationError naming the node, the column and both types, where main returned a wrong type. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-928.md | 4 ++ design_issues/ITL-928-FN1.md | 6 ++- src/orcapod/core/function_pod.py | 2 +- src/orcapod/core/nodes/function_node.py | 53 +++++++++++++++---- .../test_function_node_arrow_types.py | 23 ++++++++ 5 files changed, 77 insertions(+), 11 deletions(-) diff --git a/changelog.d/ITL-928.md b/changelog.d/ITL-928.md index 78d60e494..df261ef41 100644 --- a/changelog.d/ITL-928.md +++ b/changelog.d/ITL-928.md @@ -2,3 +2,7 @@ - A function node's `as_table()` keeps the Arrow type of every tag column. Before, it rebuilt the table from Python values, so after `job.run()` an `int8` or `uint32` tag came back as `int64`, `float32` as `double`, a naive `timestamp[us]` as `timestamp[us, tz=UTC]` and `date64` as `date32`, and a `time`, `duration`, `decimal`, `uuid` or `Path` tag made it raise. Joining a function node's output with another source on such a tag no longer fails with "Join inputs disagree on the Arrow type". `as_table(all_info=True)` of a function node now also has the `_source_` and `_context_key` columns, as the same pod's stream outside a job does. - The table of a function pod's stream (the pod applied outside a job) no longer raises for a `uuid` or `Path` tag ("Expected bytes, got a 'UUID' object"). + +### Known limitations + +- The Delta store still gives back unsigned integer tags as signed (`uint32` as `int32`), `timestamp[s]`, `[ms]` and `[ns]` tags as `timestamp[us]` (dropping sub-microsecond digits), and a zoned tag other than UTC as UTC; it cannot store `date64`, `time` or `duration` columns. So on a Delta store, a re-run over such a tag writes the downstream nodes' pipeline rows again, and a function node whose rows mix rows read from the store with newly computed rows raises `InputValidationError` naming the column and both types (where it used to return a wrong type silently). The in-memory store is not affected. A fix of the Delta store follows (ITL-928). diff --git a/design_issues/ITL-928-FN1.md b/design_issues/ITL-928-FN1.md index b17992700..77fe85381 100644 --- a/design_issues/ITL-928-FN1.md +++ b/design_issues/ITL-928-FN1.md @@ -20,6 +20,10 @@ table from Python values too (typed with the tag's own Arrow schema), so only a **Fix:** both build their tag table from each row's own Arrow table (`tag.as_table()`), and the node its data table from `data.as_table()` (source info and context, no meta columns), -concatenated by name. No stored value, record id or hash changes. The zero-row table of a +concatenated by name; rows that disagree on a column's Arrow type raise +`InputValidationError` naming the column and both types, never a widened type. No stored value, record id or hash changes. The zero-row table of a node that has not run is still built from the Python output schema (the ITL-912 problem, for function nodes). +The Delta store's own type changes (unsigned to signed, `timestamp[s/ms/ns]` to `[us]`, a zone +to UTC) are the second part of ITL-928; until they are fixed, rows read from a Delta store and +rows computed in the same run can disagree, which now raises. diff --git a/src/orcapod/core/function_pod.py b/src/orcapod/core/function_pod.py index d6380f3d4..72fcd143c 100644 --- a/src/orcapod/core/function_pod.py +++ b/src/orcapod/core/function_pod.py @@ -1103,7 +1103,7 @@ def as_table( all_info: bool = False, ) -> pa.Table: if self._cached_output_table is None: - tag_tables = [] + tag_tables: list[pa.Table] = [] all_data = [] data_schema = None for tag, data in self.iter_data(): diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index b2b6abd20..cc239a521 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -132,6 +132,36 @@ def _build_record_id_preimage( ) +def _concat_row_tables(tables: "list[pa.Table]") -> "pa.Table": + """Concatenate one-row tables by column name, refusing a column whose types differ. + + Rows never make a column of a type one of them does not have: two rows whose + Arrow types for one column differ (other than by their large variants) are an + error, never widened or cast. + + Args: + tables: The rows' tables, in row order; not empty. + + Returns: + The concatenated table, with the first table's column order. + + Raises: + InputValidationError: If two rows give one column different Arrow types, + naming the column and both types. + """ + first_types = dict(zip(tables[0].schema.names, tables[0].schema.types)) + for table in tables[1:]: + for name, row_type in zip(table.schema.names, table.schema.types): + expected = first_types.get(name) + if ( + expected is not None + and arrow_utils.shared_arrow_type(expected, row_type) is None + ): + message = f"Rows disagree on the Arrow type of column {name!r}: {expected} and {row_type}" + raise InputValidationError(message) + return arrow_utils.concat_tables_by_name(tables) + + def _bare_digest(hash_string: str) -> str: """Return the bare hex digest of a stored ``"