Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions changelog.d/ITL-1045.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
### Fixed

- `as_table` on a function pod stream, a pod-cached stream or a function node no longer returns rows from one computation with content hashes from another (ITL-1045). When the upstream was reset (`clear_cache()` on it) between `as_table()` and `as_table(columns={"content_hash": True})` and the function gives different values on a second call, the cached table was kept and its rows were given the hashes of the recomputed rows. A stale stream or node now clears its cache first and returns the current rows with their own hashes. No hash, stored table or public API changes.
19 changes: 19 additions & 0 deletions design_issues/ITL-1045-FP1.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
## `src/orcapod/core/function_pod.py` — Table assembly of a function pod stream and node

### FP1 — `as_table` built rows and content hashes in two passes: a stale upstream between them gave old rows with new rows' hashes
**Status:** resolved
**Severity:** medium
**Issue:** ITL-1045

`FunctionPodStream.as_table` (and `_CachedFunctionPodStream`, `FunctionNodeBase.as_table`, the
same structure) used its cached table without a staleness check, then built the content hashes
in a second `iter_data()` pass. `iter_data()` clears a stale cache. So when the upstream was
reset between `as_table()` and `as_table(columns={"content_hash": True})` (a non-deterministic
function, `clear_cache()` on the upstream), the old cached rows got the hashes of the new rows,
and no row's `_content_hash` was the hash of its own data.

**Fix:** `as_table` clears a stale cache first, before any of it is used, so the table and the
hashes come from the same rows and the call returns the current rows. For a job node no
public sequence changes the values between the two passes (its results come from the store),
so the defect was latent there; the same few lines fix its structure. No hash, golden value,
stored table or public API changes.
6 changes: 6 additions & 0 deletions src/orcapod/core/function_pod.py
Original file line number Diff line number Diff line change
Expand Up @@ -1335,6 +1335,12 @@ def as_table(
columns: ColumnConfig | dict[str, Any] | None = None,
all_info: bool = False,
) -> pa.Table:
# A stale cache is cleared before any of it is used: the table and the content
# hashes are then built from the same rows, and they are the current ones
# (ITL-1045). ``iter_data`` clears a stale cache too, but only after the
# cached table would already have been read.
if self.is_stale:
self.clear_cache()
if self._cached_output_table is None:
self._cached_output_table = self._table_from_pairs(list(self.iter_data()))
assert self._cached_output_table is not None, (
Expand Down
6 changes: 6 additions & 0 deletions src/orcapod/core/nodes/function_node.py
Original file line number Diff line number Diff line change
Expand Up @@ -548,6 +548,12 @@ def as_table(
columns: ColumnConfig | dict[str, Any] | None = None,
all_info: bool = False,
) -> "pa.Table":
# A stale cache is cleared before any of it is used: the table and the content
# hashes are then built from the same rows, and they are the current ones
# (ITL-1045). ``iter_data`` clears a stale cache too, but only after the
# cached table would already have been read.
if self.is_stale:
self.clear_cache()
if self._cached_output_table is None:
self._cached_output_table = self._table_from_pairs(list(self.iter_data()))

Expand Down
8 changes: 8 additions & 0 deletions superpowers/plans/2026-11-13-itl-1045-as-table-two-passes.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# ITL-1045 plan: as_table never mixes two computations

Bug: `as_table` used `_cached_output_table` without a staleness check; the content-hash pass
(`iter_data`) cleared the stale cache and recomputed. Fix: `if self.is_stale: self.clear_cache()`
at the top of `FunctionPodStream.as_table` and `FunctionNodeBase.as_table` (the cached pod stream
inherits). Tests: tests/test_core/function_pod/test_as_table_stale_between_passes.py (pod
stream, cached pod stream, head/iter_data, job node latent). Job node: latent (results come from
the store; checked unfixed: aligned).
131 changes: 131 additions & 0 deletions tests/test_core/function_pod/test_as_table_stale_between_passes.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
"""``as_table`` returns the current rows with their own content hashes (ITL-1045).

A table is built from one pass over the rows and its content hashes from another. If the
upstream is reset between a cached ``as_table()`` and ``as_table(content_hash=True)``, the
call must return the rows a fresh read gives, each with the hash of its own data.
"""

from __future__ import annotations

import itertools
from typing import Any, cast

import pyarrow as pa

from orcapod import function_pod
from orcapod.core.cached_function_pod import CachedFunctionPod
from orcapod.databases import InMemoryArrowDatabase
from orcapod.sources import ArrowTableSource

COUNTER = itertools.count()


@function_pod(output_keys="y")
def stamp(x: int) -> int:
"""Not deterministic: a call counter."""
return 100 * next(COUNTER) + x


@function_pod(output_keys="z")
def plus_one(y: int) -> int:
"""Return ``y + 1``."""
return y + 1


def source() -> ArrowTableSource:
"""Three rows, not in tag order."""
table = pa.table(
{"k": ["c", "a", "b"], "x": [3, 1, 2]},
schema=pa.schema(
[
pa.field("k", pa.large_string(), nullable=False),
pa.field("x", pa.int64(), nullable=False),
]
),
)
return ArrowTableSource(table, tag_columns=["k"], source_id="src")


def chain(cached: bool = False) -> tuple[Any, Any]:
s1: Any = stamp.pod(source())
if cached:
pod = CachedFunctionPod(
function_pod=plus_one.pod, # pyright: ignore[reportArgumentType] # FunctionPod lacks from_config, as in CachedFunctionPod.process
result_database=InMemoryArrowDatabase(),
)
return s1, pod.process(s1)
return s1, plus_one.pod(s1)


def assert_rows_carry_their_own_hash(stream: Any, table: pa.Table) -> None:
hasher = stream.data_context.arrow_hasher
for i in range(table.num_rows):
own = hasher.hash_table(table.slice(i, 1).select(["z"])).to_string()
assert table.column("_content_hash")[i].as_py() == own, f"row {i}"


def by_tag(table: pa.Table) -> dict[str, int]:
keys = cast("list[str]", table.column("k").to_pylist())
values = cast("list[int]", table.column("z").to_pylist())
return dict(zip(keys, values, strict=True))


def test_stale_between_two_calls_returns_current_rows_with_their_own_hashes() -> None:
s1, s2 = chain()
s2.as_table()
s1.clear_cache()
assert s2.is_stale

table = s2.as_table(columns={"content_hash": True})

assert_rows_carry_their_own_hash(s2, table)
# the rows are the current ones: a read now gives the same values
assert by_tag(table) == by_tag(s2.as_table())


def test_cached_pod_stream_stale_between_two_calls() -> None:
s1, s2 = chain(cached=True)
s2.as_table()
s1.clear_cache()

table = s2.as_table(columns={"content_hash": True})

assert_rows_carry_their_own_hash(s2, table)
assert by_tag(table) == by_tag(s2.as_table())


def test_head_and_iter_data_show_the_current_rows_after_the_reset() -> None:
s1, s2 = chain()
s2.as_table()
s1.clear_cache()
table = s2.as_table(columns={"content_hash": True})
current = by_tag(table)

s1.clear_cache()
rows = {t.as_dict()["k"]: d.as_dict()["z"] for t, d in s2.iter_data()}
assert rows == by_tag(s2.as_table())
assert rows != current # the reset recomputed: the counter moved on

s1.clear_cache()
head = s2.head(10, columns={"content_hash": True})
assert head.table.num_rows in (0, 3)
if head.table.num_rows:
assert_rows_carry_their_own_hash(s2, head.table)


def test_job_node_stale_between_two_calls_keeps_rows_and_hashes_aligned() -> None:
from orcapod import PipelineJob

job = PipelineJob(name="itl1045", store=InMemoryArrowDatabase())
with job:
plus_one.pod(stamp.pod(source(), label="st"), label="n")
_ = job.run()
node = job.nodes["n"]
_ = node.as_table()
job.nodes["st"].clear_cache() # public: the node goes stale
assert node.is_stale

table = node.as_table(columns={"content_hash": True})

assert_rows_carry_their_own_hash(node, table)
assert by_tag(table) == by_tag(node.as_table())
Loading