diff --git a/changelog.d/ITL-1045.md b/changelog.d/ITL-1045.md new file mode 100644 index 000000000..445cf7c5a --- /dev/null +++ b/changelog.d/ITL-1045.md @@ -0,0 +1,3 @@ +### Fixed + +- `as_table` on a function pod stream, a pod-cached stream or a function node no longer returns rows from one computation with content hashes from another (ITL-1045). When the upstream was reset (`clear_cache()` on it) between `as_table()` and `as_table(columns={"content_hash": True})` and the function gives different values on a second call, the cached table was kept and its rows were given the hashes of the recomputed rows. A stale stream or node now clears its cache first and returns the current rows with their own hashes. No hash, stored table or public API changes. diff --git a/design_issues/ITL-1045-FP1.md b/design_issues/ITL-1045-FP1.md new file mode 100644 index 000000000..5462f436b --- /dev/null +++ b/design_issues/ITL-1045-FP1.md @@ -0,0 +1,19 @@ +## `src/orcapod/core/function_pod.py` — Table assembly of a function pod stream and node + +### FP1 — `as_table` built rows and content hashes in two passes: a stale upstream between them gave old rows with new rows' hashes +**Status:** resolved +**Severity:** medium +**Issue:** ITL-1045 + +`FunctionPodStream.as_table` (and `_CachedFunctionPodStream`, `FunctionNodeBase.as_table`, the +same structure) used its cached table without a staleness check, then built the content hashes +in a second `iter_data()` pass. `iter_data()` clears a stale cache. So when the upstream was +reset between `as_table()` and `as_table(columns={"content_hash": True})` (a non-deterministic +function, `clear_cache()` on the upstream), the old cached rows got the hashes of the new rows, +and no row's `_content_hash` was the hash of its own data. + +**Fix:** `as_table` clears a stale cache first, before any of it is used, so the table and the +hashes come from the same rows and the call returns the current rows. For a job node no +public sequence changes the values between the two passes (its results come from the store), +so the defect was latent there; the same few lines fix its structure. No hash, golden value, +stored table or public API changes. diff --git a/src/orcapod/core/function_pod.py b/src/orcapod/core/function_pod.py index 9e170ad26..5ee7b32a6 100644 --- a/src/orcapod/core/function_pod.py +++ b/src/orcapod/core/function_pod.py @@ -1335,6 +1335,12 @@ def as_table( columns: ColumnConfig | dict[str, Any] | None = None, all_info: bool = False, ) -> pa.Table: + # A stale cache is cleared before any of it is used: the table and the content + # hashes are then built from the same rows, and they are the current ones + # (ITL-1045). ``iter_data`` clears a stale cache too, but only after the + # cached table would already have been read. + if self.is_stale: + self.clear_cache() if self._cached_output_table is None: self._cached_output_table = self._table_from_pairs(list(self.iter_data())) assert self._cached_output_table is not None, ( diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index dca13d7d4..4e38858b3 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -548,6 +548,12 @@ def as_table( columns: ColumnConfig | dict[str, Any] | None = None, all_info: bool = False, ) -> "pa.Table": + # A stale cache is cleared before any of it is used: the table and the content + # hashes are then built from the same rows, and they are the current ones + # (ITL-1045). ``iter_data`` clears a stale cache too, but only after the + # cached table would already have been read. + if self.is_stale: + self.clear_cache() if self._cached_output_table is None: self._cached_output_table = self._table_from_pairs(list(self.iter_data())) diff --git a/superpowers/plans/2026-11-13-itl-1045-as-table-two-passes.md b/superpowers/plans/2026-11-13-itl-1045-as-table-two-passes.md new file mode 100644 index 000000000..e99a57cad --- /dev/null +++ b/superpowers/plans/2026-11-13-itl-1045-as-table-two-passes.md @@ -0,0 +1,8 @@ +# ITL-1045 plan: as_table never mixes two computations + +Bug: `as_table` used `_cached_output_table` without a staleness check; the content-hash pass +(`iter_data`) cleared the stale cache and recomputed. Fix: `if self.is_stale: self.clear_cache()` +at the top of `FunctionPodStream.as_table` and `FunctionNodeBase.as_table` (the cached pod stream +inherits). Tests: tests/test_core/function_pod/test_as_table_stale_between_passes.py (pod +stream, cached pod stream, head/iter_data, job node latent). Job node: latent (results come from +the store; checked unfixed: aligned). diff --git a/tests/test_core/function_pod/test_as_table_stale_between_passes.py b/tests/test_core/function_pod/test_as_table_stale_between_passes.py new file mode 100644 index 000000000..782a6dfeb --- /dev/null +++ b/tests/test_core/function_pod/test_as_table_stale_between_passes.py @@ -0,0 +1,131 @@ +"""``as_table`` returns the current rows with their own content hashes (ITL-1045). + +A table is built from one pass over the rows and its content hashes from another. If the +upstream is reset between a cached ``as_table()`` and ``as_table(content_hash=True)``, the +call must return the rows a fresh read gives, each with the hash of its own data. +""" + +from __future__ import annotations + +import itertools +from typing import Any, cast + +import pyarrow as pa + +from orcapod import function_pod +from orcapod.core.cached_function_pod import CachedFunctionPod +from orcapod.databases import InMemoryArrowDatabase +from orcapod.sources import ArrowTableSource + +COUNTER = itertools.count() + + +@function_pod(output_keys="y") +def stamp(x: int) -> int: + """Not deterministic: a call counter.""" + return 100 * next(COUNTER) + x + + +@function_pod(output_keys="z") +def plus_one(y: int) -> int: + """Return ``y + 1``.""" + return y + 1 + + +def source() -> ArrowTableSource: + """Three rows, not in tag order.""" + table = pa.table( + {"k": ["c", "a", "b"], "x": [3, 1, 2]}, + schema=pa.schema( + [ + pa.field("k", pa.large_string(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] + ), + ) + return ArrowTableSource(table, tag_columns=["k"], source_id="src") + + +def chain(cached: bool = False) -> tuple[Any, Any]: + s1: Any = stamp.pod(source()) + if cached: + pod = CachedFunctionPod( + function_pod=plus_one.pod, # pyright: ignore[reportArgumentType] # FunctionPod lacks from_config, as in CachedFunctionPod.process + result_database=InMemoryArrowDatabase(), + ) + return s1, pod.process(s1) + return s1, plus_one.pod(s1) + + +def assert_rows_carry_their_own_hash(stream: Any, table: pa.Table) -> None: + hasher = stream.data_context.arrow_hasher + for i in range(table.num_rows): + own = hasher.hash_table(table.slice(i, 1).select(["z"])).to_string() + assert table.column("_content_hash")[i].as_py() == own, f"row {i}" + + +def by_tag(table: pa.Table) -> dict[str, int]: + keys = cast("list[str]", table.column("k").to_pylist()) + values = cast("list[int]", table.column("z").to_pylist()) + return dict(zip(keys, values, strict=True)) + + +def test_stale_between_two_calls_returns_current_rows_with_their_own_hashes() -> None: + s1, s2 = chain() + s2.as_table() + s1.clear_cache() + assert s2.is_stale + + table = s2.as_table(columns={"content_hash": True}) + + assert_rows_carry_their_own_hash(s2, table) + # the rows are the current ones: a read now gives the same values + assert by_tag(table) == by_tag(s2.as_table()) + + +def test_cached_pod_stream_stale_between_two_calls() -> None: + s1, s2 = chain(cached=True) + s2.as_table() + s1.clear_cache() + + table = s2.as_table(columns={"content_hash": True}) + + assert_rows_carry_their_own_hash(s2, table) + assert by_tag(table) == by_tag(s2.as_table()) + + +def test_head_and_iter_data_show_the_current_rows_after_the_reset() -> None: + s1, s2 = chain() + s2.as_table() + s1.clear_cache() + table = s2.as_table(columns={"content_hash": True}) + current = by_tag(table) + + s1.clear_cache() + rows = {t.as_dict()["k"]: d.as_dict()["z"] for t, d in s2.iter_data()} + assert rows == by_tag(s2.as_table()) + assert rows != current # the reset recomputed: the counter moved on + + s1.clear_cache() + head = s2.head(10, columns={"content_hash": True}) + assert head.table.num_rows in (0, 3) + if head.table.num_rows: + assert_rows_carry_their_own_hash(s2, head.table) + + +def test_job_node_stale_between_two_calls_keeps_rows_and_hashes_aligned() -> None: + from orcapod import PipelineJob + + job = PipelineJob(name="itl1045", store=InMemoryArrowDatabase()) + with job: + plus_one.pod(stamp.pod(source(), label="st"), label="n") + _ = job.run() + node = job.nodes["n"] + _ = node.as_table() + job.nodes["st"].clear_cache() # public: the node goes stale + assert node.is_stale + + table = node.as_table(columns={"content_hash": True}) + + assert_rows_carry_their_own_hash(node, table) + assert by_tag(table) == by_tag(node.as_table())