From f4e48d88cfea4b4134b1713a9a19bd721fa26ff1 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 01:32:28 -0800 Subject: [PATCH] fix(function_node): a row read back from the store carries its stored __data_id (ITL-1060) Co-Authored-By: Claude Sonnet 5.5 --- changelog.d/ITL-1060.md | 3 + design_issues/ITL-1060-FN1.md | 23 +++ src/orcapod/core/nodes/function_node.py | 45 ++++-- ...026-11-13-itl-1060-readback-datagram-id.md | 9 ++ .../function_pod/test_readback_datagram_id.py | 136 ++++++++++++++++++ 5 files changed, 208 insertions(+), 8 deletions(-) create mode 100644 changelog.d/ITL-1060.md create mode 100644 design_issues/ITL-1060-FN1.md create mode 100644 superpowers/plans/2026-11-13-itl-1060-readback-datagram-id.md create mode 100644 tests/test_core/function_pod/test_readback_datagram_id.py diff --git a/changelog.d/ITL-1060.md b/changelog.d/ITL-1060.md new file mode 100644 index 000000000..7479d635a --- /dev/null +++ b/changelog.d/ITL-1060.md @@ -0,0 +1,3 @@ +### Fixed + +- A result row a function node reads back from its store now carries the stored `__data_id` as its `datagram_uuid` (ITL-1060). Before, every read-back row (sync, async cache-only, loaded node) got a fresh uuid, so it was not the datagram its own source-info token names and its id changed on every read. Nothing stored changes: the id is read only from the output of a computation or a result-cache lookup, which already carried the stored id. No hash, record id or golden value moves. diff --git a/design_issues/ITL-1060-FN1.md b/design_issues/ITL-1060-FN1.md new file mode 100644 index 000000000..69cacfc27 --- /dev/null +++ b/design_issues/ITL-1060-FN1.md @@ -0,0 +1,23 @@ +## `src/orcapod/core/nodes/function_node.py` — Rows read back from a function node's store + +### FN1 — A result row read back from the store got a fresh datagram id instead of its stored `__data_id` +**Status:** resolved +**Severity:** low +**Issue:** ITL-1060 + +`FunctionNode._load_cached_entries` built the rows it reads back (sync `iter_data`, the async +cache-only path, a loaded node) from a stream over the stored table with the `__data_id` +column dropped, so each `Data` took a fresh uuid7 on first use. The row's own source-info +token still named the stored id. The result cache lookup (`CachedFunctionPod`, a node's +computed rows) already passes the stored id and was correct. + +No stored value could differ because of it: `datagram_uuid` is read only on the OUTPUT of +`process_data` (the pipeline record's `__data_id`, the result record id, the post-run hook +payload), never on an input or a read-back row, and a source-info token is built from the new +output's own uuid at compute time. It was an in-memory identity defect. + +**Fix:** `_stored_rows_stream` also returns the stored `__data_id` of each row, and +`_load_cached_entries` rebuilds each `Data` with it as its `datagram_uuid` (same columns, +types, source info and context). Reading the same stored row twice gives the same id. The +operator node cache and `DerivedSource` store no `__data_id`, so there is no stored id to +carry. No hash, record id, golden value or stored table changes. diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 5d3660d5e..8d8979ee3 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -41,7 +41,7 @@ StreamHead, ) from orcapod.core.tracker import DEFAULT_TRACKER_MANAGER -from orcapod.core.datagrams.tag_data import EmptyData, Tag +from orcapod.core.datagrams.tag_data import Data, EmptyData, Tag from orcapod.errors import ( EphemeralResultMissingError, InputValidationError, @@ -215,6 +215,26 @@ def _bare_digest(hash_string: str) -> str: _TAG_NODE_INPUT_REF = "_tag_node_input_ref" +def _with_datagram_id(data: Data, datagram_id: uuid.UUID) -> Data: + """Return ``data`` rebuilt with ``datagram_id`` as its datagram id. + + A stored row read back is built by a stream, which gives every row a fresh id. + The rebuilt datagram has the same columns, types, source info and data context. + + Args: + data: The datagram a stream built from a stored row. + datagram_id: The id the row was stored under. + + Returns: + A datagram equal to ``data`` whose ``datagram_uuid`` is ``datagram_id``. + """ + return Data( + data.as_table(columns={"source": True}), + record_uuid=datagram_id, + data_context=data.data_context_key, + ) + + def _executor_supports_concurrent( data_function: DataFunctionProtocol, ) -> bool: @@ -2849,7 +2869,7 @@ def _stored_tag_keys(self, fetched: _JoinedRecords) -> tuple[str, ...]: def _stored_rows_stream( self, fetched: _JoinedRecords - ) -> tuple[list[bytes], ArrowTableStream] | None: + ) -> tuple[list[bytes], ArrowTableStream, list[uuid.UUID]] | None: """Return the persistent stored rows of *fetched* as a stream. The stream is built over the stored Arrow records with the internal @@ -2858,8 +2878,9 @@ def _stored_rows_stream( greatest recomputation index. Returns: - The base entry ID of each row, in stream order, and the stream; or - ``None`` when no persistent row is stored. + The base entry ID of each row, in stream order, the stream, and the + stored ``__data_id`` of each row (the id of the result record, in + stream order); or ``None`` when no persistent row is stored. """ joined = fetched.table if joined.num_rows == 0: @@ -2900,10 +2921,14 @@ def _stored_rows_stream( if c.startswith(constants.META_PREFIX) or c == constants.NODE_CONTENT_HASH_COL ] + data_ids = [ + uuid.UUID(bytes=bytes(cast(bytes, record_id))) + for record_id in joined.column(constants.DATA_RECORD_ID).to_pylist() + ] stream = ArrowTableStream( joined.drop(drop_cols), tag_columns=self._stored_tag_keys(fetched) ) - return base_entry_ids, stream + return base_entry_ids, stream, data_ids def _load_cached_entries( self, @@ -2945,9 +2970,13 @@ def _load_cached_entries( # --- Process normal (fully matched) rows --- stored = self._stored_rows_stream(fetched) if stored is not None: - base_entry_ids_col, stream = stored - for base_eid, (tag, data) in zip(base_entry_ids_col, stream.iter_data()): - loaded[base_eid] = (tag, data) + base_entry_ids_col, stream, data_ids = stored + for base_eid, data_id, (tag, data) in zip( + base_entry_ids_col, data_ids, stream.iter_data(), strict=True + ): + # A stored result is the datagram its own provenance names: it + # carries the stored ``__data_id`` as its datagram id (ITL-1060). + loaded[base_eid] = (tag, _with_datagram_id(data, data_id)) # --- Process EmptyData tokens (ephemeral misses) --- for base_eid, empty_data in fetched.empty_data_tokens.items(): diff --git a/superpowers/plans/2026-11-13-itl-1060-readback-datagram-id.md b/superpowers/plans/2026-11-13-itl-1060-readback-datagram-id.md new file mode 100644 index 000000000..25fcd8002 --- /dev/null +++ b/superpowers/plans/2026-11-13-itl-1060-readback-datagram-id.md @@ -0,0 +1,9 @@ +# ITL-1060 plan: read-back rows carry their stored `__data_id` + +Worker notes. Bug: `FunctionNode._load_cached_entries` builds Data via `ArrowTableStream.iter_data` +over a table with `__data_id` dropped, so each row got a fresh uuid7. +Fix: `_stored_rows_stream` also returns the stored ids; `_load_cached_entries` rebuilds each Data +with `record_uuid` (`_with_datagram_id`). All node read paths (sync, async cache-only, loaded) +go through `_load_cached_entries`. Pod-level result cache already passes `record_uuid`. +Evidence on use: `datagram_uuid` is read only on `process_data` outputs. Tests: +tests/test_core/function_pod/test_readback_datagram_id.py. diff --git a/tests/test_core/function_pod/test_readback_datagram_id.py b/tests/test_core/function_pod/test_readback_datagram_id.py new file mode 100644 index 000000000..421f53129 --- /dev/null +++ b/tests/test_core/function_pod/test_readback_datagram_id.py @@ -0,0 +1,136 @@ +"""A result row read back from the store carries its stored ``__data_id`` (ITL-1060).""" + +from __future__ import annotations + +import uuid + +import pyarrow as pa + +from orcapod import PipelineJob, function_pod +from orcapod.databases import InMemoryArrowDatabase +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.sources import ArrowTableSource + + +@function_pod(output_keys="y") +def times_ten(x: int) -> int: + """Return ``10 * x``.""" + return 10 * x + + +def source() -> ArrowTableSource: + """Three rows, not in tag order.""" + table = pa.table( + {"k": ["c", "a", "b"], "x": [3, 1, 2]}, + schema=pa.schema( + [ + pa.field("k", pa.large_string(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] + ), + ) + return ArrowTableSource(table, tag_columns=["k"], source_id="src") + + +def build(store: InMemoryArrowDatabase) -> PipelineJob: + job = PipelineJob(name="itl1060", store=store) + with job: + times_ten.pod(source(), label="node") + return job + + +def stored_ids(job: PipelineJob) -> dict[str, uuid.UUID]: + """Stored ``__data_id`` by tag ``k``.""" + records = job.nodes["node"].get_all_records(all_info=True) + assert records is not None + return { + k: uuid.UUID(bytes=bytes(i)) + for k, i in zip( + records.column("k").to_pylist(), + records.column("__data_id").to_pylist(), + strict=True, + ) + } + + +def read_back_ids(job: PipelineJob) -> dict[str, uuid.UUID]: + """Datagram id of each row the node yields, by tag ``k``.""" + return { + str(tag.as_dict()["k"]): data.datagram_uuid + for tag, data in job.nodes["node"].iter_data() + } + + +def test_rows_read_back_by_a_second_run_carry_their_stored_id() -> None: + store = InMemoryArrowDatabase() + _ = build(store).run(orchestrator=SyncPipelineOrchestrator()) + + second = build(store) + _ = second.run(orchestrator=SyncPipelineOrchestrator()) + stored = stored_ids(second) + assert len(stored) == 3 + assert read_back_ids(second) == stored + # reading the same stored row twice gives the same id + assert read_back_ids(second) == read_back_ids(second) + + +def test_async_read_back_carries_the_stored_id() -> None: + store = InMemoryArrowDatabase() + _ = build(store).run(orchestrator=SyncPipelineOrchestrator()) + + second = build(store) + _ = second.run(orchestrator=AsyncPipelineOrchestrator()) + assert read_back_ids(second) == stored_ids(second) + + +def test_a_fresh_job_on_the_store_reads_the_stored_ids() -> None: + store = InMemoryArrowDatabase() + first = build(store) + _ = first.run(orchestrator=SyncPipelineOrchestrator()) + stored = stored_ids(first) + + fresh = build(store) # never run: every row comes from the store + assert read_back_ids(fresh) == stored + + +def test_a_pod_cached_row_read_back_carries_the_id_it_was_stored_under() -> None: + from orcapod.core.cached_function_pod import CachedFunctionPod + from orcapod.core.streams.arrow_table_stream import ArrowTableStream + + table = pa.table( + {"k": ["c", "a", "b"], "x": [3, 1, 2]}, + schema=pa.schema( + [ + pa.field("k", pa.large_string(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] + ), + ) + + store = InMemoryArrowDatabase() + + def ids() -> dict[str, uuid.UUID]: + pod = CachedFunctionPod( + function_pod=times_ten.pod, # pyright: ignore[reportArgumentType] # FunctionPod lacks from_config, as in CachedFunctionPod.process + result_database=store, + ) + stream = pod.process(ArrowTableStream(table, tag_columns=["k"])) + return {str(t.as_dict()["k"]): d.datagram_uuid for t, d in stream.iter_data()} + + first = ids() + assert ids() == first # the second read takes every row from the cache + + +def test_a_read_back_row_is_the_same_row_apart_from_its_id() -> None: + store = InMemoryArrowDatabase() + first = build(store) + _ = first.run(orchestrator=SyncPipelineOrchestrator()) + computed = {t.as_dict()["k"]: d for t, d in first.nodes["node"].iter_data()} + second = build(store) + for tag, data in second.nodes["node"].iter_data(): + before = computed[tag.as_dict()["k"]] + assert data.content_hash() == before.content_hash() + assert data.source_info() == before.source_info() + # the computed row also carries the ``__computed`` meta column; the rest is equal + columns = {"source": True, "context": True} + assert data.as_table(columns=columns).equals(before.as_table(columns=columns))