Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions changelog.d/ITL-1060.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
### Fixed

- A result row a function node reads back from its store now carries the stored `__data_id` as its `datagram_uuid` (ITL-1060). Before, every read-back row (sync, async cache-only, loaded node) got a fresh uuid, so it was not the datagram its own source-info token names and its id changed on every read. Nothing stored changes: the id is read only from the output of a computation or a result-cache lookup, which already carried the stored id. No hash, record id or golden value moves.
23 changes: 23 additions & 0 deletions design_issues/ITL-1060-FN1.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
## `src/orcapod/core/nodes/function_node.py` — Rows read back from a function node's store

### FN1 — A result row read back from the store got a fresh datagram id instead of its stored `__data_id`
**Status:** resolved
**Severity:** low
**Issue:** ITL-1060

`FunctionNode._load_cached_entries` built the rows it reads back (sync `iter_data`, the async
cache-only path, a loaded node) from a stream over the stored table with the `__data_id`
column dropped, so each `Data` took a fresh uuid7 on first use. The row's own source-info
token still named the stored id. The result cache lookup (`CachedFunctionPod`, a node's
computed rows) already passes the stored id and was correct.

No stored value could differ because of it: `datagram_uuid` is read only on the OUTPUT of
`process_data` (the pipeline record's `__data_id`, the result record id, the post-run hook
payload), never on an input or a read-back row, and a source-info token is built from the new
output's own uuid at compute time. It was an in-memory identity defect.

**Fix:** `_stored_rows_stream` also returns the stored `__data_id` of each row, and
`_load_cached_entries` rebuilds each `Data` with it as its `datagram_uuid` (same columns,
types, source info and context). Reading the same stored row twice gives the same id. The
operator node cache and `DerivedSource` store no `__data_id`, so there is no stored id to
carry. No hash, record id, golden value or stored table changes.
45 changes: 37 additions & 8 deletions src/orcapod/core/nodes/function_node.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
StreamHead,
)
from orcapod.core.tracker import DEFAULT_TRACKER_MANAGER
from orcapod.core.datagrams.tag_data import EmptyData, Tag
from orcapod.core.datagrams.tag_data import Data, EmptyData, Tag
from orcapod.errors import (
EphemeralResultMissingError,
InputValidationError,
Expand Down Expand Up @@ -215,6 +215,26 @@ def _bare_digest(hash_string: str) -> str:
_TAG_NODE_INPUT_REF = "_tag_node_input_ref"


def _with_datagram_id(data: Data, datagram_id: uuid.UUID) -> Data:
"""Return ``data`` rebuilt with ``datagram_id`` as its datagram id.

A stored row read back is built by a stream, which gives every row a fresh id.
The rebuilt datagram has the same columns, types, source info and data context.

Args:
data: The datagram a stream built from a stored row.
datagram_id: The id the row was stored under.

Returns:
A datagram equal to ``data`` whose ``datagram_uuid`` is ``datagram_id``.
"""
return Data(
data.as_table(columns={"source": True}),
record_uuid=datagram_id,
data_context=data.data_context_key,
)


def _executor_supports_concurrent(
data_function: DataFunctionProtocol,
) -> bool:
Expand Down Expand Up @@ -2849,7 +2869,7 @@ def _stored_tag_keys(self, fetched: _JoinedRecords) -> tuple[str, ...]:

def _stored_rows_stream(
self, fetched: _JoinedRecords
) -> tuple[list[bytes], ArrowTableStream] | None:
) -> tuple[list[bytes], ArrowTableStream, list[uuid.UUID]] | None:
"""Return the persistent stored rows of *fetched* as a stream.

The stream is built over the stored Arrow records with the internal
Expand All @@ -2858,8 +2878,9 @@ def _stored_rows_stream(
greatest recomputation index.

Returns:
The base entry ID of each row, in stream order, and the stream; or
``None`` when no persistent row is stored.
The base entry ID of each row, in stream order, the stream, and the
stored ``__data_id`` of each row (the id of the result record, in
stream order); or ``None`` when no persistent row is stored.
"""
joined = fetched.table
if joined.num_rows == 0:
Expand Down Expand Up @@ -2900,10 +2921,14 @@ def _stored_rows_stream(
if c.startswith(constants.META_PREFIX)
or c == constants.NODE_CONTENT_HASH_COL
]
data_ids = [
uuid.UUID(bytes=bytes(cast(bytes, record_id)))
for record_id in joined.column(constants.DATA_RECORD_ID).to_pylist()
]
stream = ArrowTableStream(
joined.drop(drop_cols), tag_columns=self._stored_tag_keys(fetched)
)
return base_entry_ids, stream
return base_entry_ids, stream, data_ids

def _load_cached_entries(
self,
Expand Down Expand Up @@ -2945,9 +2970,13 @@ def _load_cached_entries(
# --- Process normal (fully matched) rows ---
stored = self._stored_rows_stream(fetched)
if stored is not None:
base_entry_ids_col, stream = stored
for base_eid, (tag, data) in zip(base_entry_ids_col, stream.iter_data()):
loaded[base_eid] = (tag, data)
base_entry_ids_col, stream, data_ids = stored
for base_eid, data_id, (tag, data) in zip(
base_entry_ids_col, data_ids, stream.iter_data(), strict=True
):
# A stored result is the datagram its own provenance names: it
# carries the stored ``__data_id`` as its datagram id (ITL-1060).
loaded[base_eid] = (tag, _with_datagram_id(data, data_id))

# --- Process EmptyData tokens (ephemeral misses) ---
for base_eid, empty_data in fetched.empty_data_tokens.items():
Expand Down
9 changes: 9 additions & 0 deletions superpowers/plans/2026-11-13-itl-1060-readback-datagram-id.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# ITL-1060 plan: read-back rows carry their stored `__data_id`

Worker notes. Bug: `FunctionNode._load_cached_entries` builds Data via `ArrowTableStream.iter_data`
over a table with `__data_id` dropped, so each row got a fresh uuid7.
Fix: `_stored_rows_stream` also returns the stored ids; `_load_cached_entries` rebuilds each Data
with `record_uuid` (`_with_datagram_id`). All node read paths (sync, async cache-only, loaded)
go through `_load_cached_entries`. Pod-level result cache already passes `record_uuid`.
Evidence on use: `datagram_uuid` is read only on `process_data` outputs. Tests:
tests/test_core/function_pod/test_readback_datagram_id.py.
136 changes: 136 additions & 0 deletions tests/test_core/function_pod/test_readback_datagram_id.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
"""A result row read back from the store carries its stored ``__data_id`` (ITL-1060)."""

from __future__ import annotations

import uuid

import pyarrow as pa

from orcapod import PipelineJob, function_pod
from orcapod.databases import InMemoryArrowDatabase
from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator
from orcapod.sources import ArrowTableSource


@function_pod(output_keys="y")
def times_ten(x: int) -> int:
"""Return ``10 * x``."""
return 10 * x


def source() -> ArrowTableSource:
"""Three rows, not in tag order."""
table = pa.table(
{"k": ["c", "a", "b"], "x": [3, 1, 2]},
schema=pa.schema(
[
pa.field("k", pa.large_string(), nullable=False),
pa.field("x", pa.int64(), nullable=False),
]
),
)
return ArrowTableSource(table, tag_columns=["k"], source_id="src")


def build(store: InMemoryArrowDatabase) -> PipelineJob:
job = PipelineJob(name="itl1060", store=store)
with job:
times_ten.pod(source(), label="node")
return job


def stored_ids(job: PipelineJob) -> dict[str, uuid.UUID]:
"""Stored ``__data_id`` by tag ``k``."""
records = job.nodes["node"].get_all_records(all_info=True)
assert records is not None
return {
k: uuid.UUID(bytes=bytes(i))
for k, i in zip(
records.column("k").to_pylist(),
records.column("__data_id").to_pylist(),
strict=True,
)
}


def read_back_ids(job: PipelineJob) -> dict[str, uuid.UUID]:
"""Datagram id of each row the node yields, by tag ``k``."""
return {
str(tag.as_dict()["k"]): data.datagram_uuid
for tag, data in job.nodes["node"].iter_data()
}


def test_rows_read_back_by_a_second_run_carry_their_stored_id() -> None:
store = InMemoryArrowDatabase()
_ = build(store).run(orchestrator=SyncPipelineOrchestrator())

second = build(store)
_ = second.run(orchestrator=SyncPipelineOrchestrator())
stored = stored_ids(second)
assert len(stored) == 3
assert read_back_ids(second) == stored
# reading the same stored row twice gives the same id
assert read_back_ids(second) == read_back_ids(second)


def test_async_read_back_carries_the_stored_id() -> None:
store = InMemoryArrowDatabase()
_ = build(store).run(orchestrator=SyncPipelineOrchestrator())

second = build(store)
_ = second.run(orchestrator=AsyncPipelineOrchestrator())
assert read_back_ids(second) == stored_ids(second)


def test_a_fresh_job_on_the_store_reads_the_stored_ids() -> None:
store = InMemoryArrowDatabase()
first = build(store)
_ = first.run(orchestrator=SyncPipelineOrchestrator())
stored = stored_ids(first)

fresh = build(store) # never run: every row comes from the store
assert read_back_ids(fresh) == stored


def test_a_pod_cached_row_read_back_carries_the_id_it_was_stored_under() -> None:
from orcapod.core.cached_function_pod import CachedFunctionPod
from orcapod.core.streams.arrow_table_stream import ArrowTableStream

table = pa.table(
{"k": ["c", "a", "b"], "x": [3, 1, 2]},
schema=pa.schema(
[
pa.field("k", pa.large_string(), nullable=False),
pa.field("x", pa.int64(), nullable=False),
]
),
)

store = InMemoryArrowDatabase()

def ids() -> dict[str, uuid.UUID]:
pod = CachedFunctionPod(
function_pod=times_ten.pod, # pyright: ignore[reportArgumentType] # FunctionPod lacks from_config, as in CachedFunctionPod.process
result_database=store,
)
stream = pod.process(ArrowTableStream(table, tag_columns=["k"]))
return {str(t.as_dict()["k"]): d.datagram_uuid for t, d in stream.iter_data()}

first = ids()
assert ids() == first # the second read takes every row from the cache


def test_a_read_back_row_is_the_same_row_apart_from_its_id() -> None:
store = InMemoryArrowDatabase()
first = build(store)
_ = first.run(orchestrator=SyncPipelineOrchestrator())
computed = {t.as_dict()["k"]: d for t, d in first.nodes["node"].iter_data()}
second = build(store)
for tag, data in second.nodes["node"].iter_data():
before = computed[tag.as_dict()["k"]]
assert data.content_hash() == before.content_hash()
assert data.source_info() == before.source_info()
# the computed row also carries the ``__computed`` meta column; the rest is equal
columns = {"source": True, "context": True}
assert data.as_table(columns=columns).equals(before.as_table(columns=columns))
Loading