From 9954640ad2efc052d397fe85093951035b82fb31 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 02:07:33 -0800 Subject: [PATCH 1/7] docs(cache): record approved rerun reuse design and benchmark --- bench/bench_pipeline_cached_rerun.py | 94 +++++++++++++++++++ design_issues/ITL-1063-CR1.md | 20 ++++ .../plans/2026-11-13-itl-1063-cache-reuse.md | 46 +++++++++ .../2026-11-13-pipeline-cache-reuse-design.md | 79 ++++++++++++++++ 4 files changed, 239 insertions(+) create mode 100644 bench/bench_pipeline_cached_rerun.py create mode 100644 design_issues/ITL-1063-CR1.md create mode 100644 superpowers/plans/2026-11-13-itl-1063-cache-reuse.md create mode 100644 superpowers/specs/2026-11-13-pipeline-cache-reuse-design.md diff --git a/bench/bench_pipeline_cached_rerun.py b/bench/bench_pipeline_cached_rerun.py new file mode 100644 index 000000000..61622cee9 --- /dev/null +++ b/bench/bench_pipeline_cached_rerun.py @@ -0,0 +1,94 @@ +"""Measure work retained by a job, independently of function execution cost. + +Run with ``uv run python bench/bench_pipeline_cached_rerun.py --rows 1000``. +Counters, rather than timing thresholds, distinguish reuse from fast recomputation. +""" + +from __future__ import annotations + +import argparse +import json +import time +from unittest.mock import patch + +from orcapod import PipelineJob, function_pod +from orcapod.core.operators.group_by import GroupBy +from orcapod.databases import InMemoryArrowDatabase +from orcapod.hashing.arrow_hashers import StarfixArrowHasher +from orcapod.pipeline import AsyncPipelineOrchestrator +from orcapod.sources import DictSource + + +def measure(rows: int, grouped: bool, asynchronous: bool) -> None: + """Print cold and warm run times and counts for one independent job.""" + counts = {"functions": 0, "arrow_hashes": 0, "group_by": 0} + + @function_pod(output_keys="y") + def increment(x: int) -> int: + counts["functions"] += 1 + return x + 1 + + @function_pod(output_keys="y") + def total(x: list[int]) -> int: + counts["functions"] += 1 + return sum(x) + + source = DictSource( + data=[{"g": i // 100, "k": i, "x": i} for i in range(rows)], + tag_columns=["g", "k"], + source_id="benchmark", + ) + with PipelineJob(name="cache_benchmark", store=InMemoryArrowDatabase()) as job: + if grouped: + total.pod(source.group_by(["g"]).drop_tag_columns(["k"]), label="result") + else: + increment.pod(source, label="result") + + original_hash = StarfixArrowHasher.hash_table + original_group = GroupBy.unary_static_process + + def hash_table(self, *args, **kwargs): + counts["arrow_hashes"] += 1 + return original_hash(self, *args, **kwargs) + + def group(self, *args, **kwargs): + counts["group_by"] += 1 + return original_group(self, *args, **kwargs) + + with ( + patch.object(StarfixArrowHasher, "hash_table", hash_table), + patch.object(GroupBy, "unary_static_process", group), + ): + for run in ("cold", "warm", "warm_again"): + counts.update(functions=0, arrow_hashes=0, group_by=0) + started = time.perf_counter() + summary = job.run( + orchestrator=AsyncPipelineOrchestrator() if asynchronous else None + ) + elapsed = time.perf_counter() - started + if not summary.ok: + raise RuntimeError(str(summary)) + print( + json.dumps( + { + "rows": rows, + "grouped": grouped, + "async": asynchronous, + "run": run, + "seconds": elapsed, + **counts, + } + ), + flush=True, + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--rows", type=int, default=1000) + options = parser.parse_args() + if options.rows < 1: + parser.error("--rows must be positive") + for is_grouped in (False, True): + for is_async in (False, True): + measure(options.rows, is_grouped, is_async) diff --git a/design_issues/ITL-1063-CR1.md b/design_issues/ITL-1063-CR1.md new file mode 100644 index 000000000..1ee293255 --- /dev/null +++ b/design_issues/ITL-1063-CR1.md @@ -0,0 +1,20 @@ +## `src/orcapod/pipeline/` — completed work on unchanged reruns + +### CR1 — Job reruns rebuild rows and hashes despite retained function results +**Status:** in progress +**Severity:** high +**Issue:** ITL-1063 + +An unchanged job visits every node again. The sync orchestrator materializes retained +`(Tag, Data)` objects as a new Arrow stream, losing their memoized hashes. Operator +execution constructs a new stream despite holding a previous output. Async function +execution reloads stored results on every run. SpiralDB sources pin a table version but +rescan it and rebuild provenance on each read. + +Measured at `89160c47`, a 1,000-row flat job's warm sync run performs 2,000 Arrow hashes +and its async run 1,000, with no user-function calls. Grouping the source into 10 rows still +reruns GroupBy and performs 20 hashes on both orchestrators. A retained standalone stream +read reuses its objects and does not repeat this work. + +Approved design: `superpowers/specs/2026-11-13-pipeline-cache-reuse-design.md`. +Reproduction: `uv run python bench/bench_pipeline_cached_rerun.py --rows 1000`. diff --git a/superpowers/plans/2026-11-13-itl-1063-cache-reuse.md b/superpowers/plans/2026-11-13-itl-1063-cache-reuse.md new file mode 100644 index 000000000..78c4112ae --- /dev/null +++ b/superpowers/plans/2026-11-13-itl-1063-cache-reuse.md @@ -0,0 +1,46 @@ +# ITL-1063 Pipeline Cache Reuse Implementation Plan + +> **For agentic workers:** Use sensei:executing-plans task by task; independently owned source and execution changes may run in parallel under sensei:dispatching-parallel-agents. + +**Goal:** Unchanged pipeline reruns reuse retained rows and their hashes while changed or failed work is processed correctly. + +**Architecture:** Completed reads and node execution retain results against the exact upstream state. Unknown freshness, changed bindings/configuration and incomplete runs must never qualify as complete. Existing persistent per-row caching remains the recovery path. + +**Tech Stack:** Python, Arrow, pytest, uv, existing sync and channel-based async orchestration. + +## 1. Execution reuse and row preservation + +Files: `src/orcapod/pipeline/sync_orchestrator.py`, `async_orchestrator.py`, focused internal cache helper if needed, `src/orcapod/core/nodes/{function_node,operator_node,source_node}.py`, `tests/test_pipeline/test_execution_cache_reuse.py`. + +- [ ] Add regression cases for repeated flat/grouped jobs, sync/async/cross-orchestrator execution. Count `StarfixArrowHasher.hash_table`, `GroupBy.unary_static_process`, and user function calls after the first successful run. Assert unchanged row values, provenance and summary counts. +- [ ] Run `uv run pytest tests/test_pipeline/test_execution_cache_reuse.py -q` and confirm redundant computation failures. +- [ ] Preserve `(Tag, Data)` references when wrapping execution buffers, including data context; do not transfer hashes across transformations or drop source/system columns. +- [ ] Add a completed-execution reuse path with conservative upstream validity checks. Reuse empty outputs too; do not use timestamps or source identity hashes alone as proof for mutable/dynamic sources. Preserve lifecycle hooks and side-effect stale-store checks. +- [ ] Avoid unconditional async DB loads when the same valid completed result is retained. Do not change streaming behavior for first runs or force a barrier onto dynamic paths. +- [ ] Test source rebinding, explicit clear/invalidation, partial and failed runs, observer counts, side effects, and materialize_results=False. Preserve per-row recovery for incomplete runs. +- [ ] Run new tests plus `tests/test_pipeline`, `tests/test_core/nodes`, `tests/test_side_effects` and channel tests. Fix failures in scope. + +## 2. Pinned Spiral snapshot retention + +Files: `src/orcapod/core/sources/spiraldb_table_source.py`, `tests/test_core/sources/test_spiraldb_table_source_streaming.py`, `docs/concepts/sources.md`. + +- [ ] Add fake-connector scan-count regressions: lazy construction, complete repeat reads, mixed table/row/async reads, failed/partial reads, empty pinned snapshot and concurrent readers. +- [ ] Run focused tests to demonstrate repeated scans. +- [ ] Retain successfully completed pinned-source read buffers with reusable row objects or Arrow streams. Publish only on completion; concurrent readers must not expose partial cache state. Unpinned/unknown table versions must stay conservative. No hidden repinning or hash changes. +- [ ] Keep head bounded and lazy before a full read; allow it to use retained rows afterward. Document retention memory cost and constructing a new source to select a new snapshot. +- [ ] Run Spiral source tests and the existing live-test collection without requiring live credentials. + +## 3. Integration, documentation and delivery + +Files: `design_issues/ITL-1063-CR1.md`, `changelog.d/ITL-1063.md`, approved design and this plan. + +- [ ] Review interactions between source snapshots, node invalidation, sync/async switching and existing row readback caches. +- [ ] Record implementation decisions and limitations in the approved design; add the authorized design-issue entry and changelog. +- [ ] Benchmark flat and grouped repeated jobs with function/operator/hash counters and compare to baseline. Use operation-count assertions, not flaky timing thresholds, in tests. +- [ ] Run `uv run ruff check .`, `uv run ruff format --check .`, `uv run basedpyright`, and `uv run pytest tests/`; inspect exit codes and logs. +- [ ] Request independent review; resolve correctness findings and rerun affected checks. +- [ ] Commit with conventional message, push issue branch, create PR against main linking ITL-1063, and inspect CI. Include measurements and decisions; do not merge. + +## Authorization and decisions + +Developer approved the design, issue creation and implementation through PR. Existing isolated cowfish workspace is used. No additional confirmation for routine implementation decisions; record them for the developer's return. diff --git a/superpowers/specs/2026-11-13-pipeline-cache-reuse-design.md b/superpowers/specs/2026-11-13-pipeline-cache-reuse-design.md new file mode 100644 index 000000000..bfcc36a35 --- /dev/null +++ b/superpowers/specs/2026-11-13-pipeline-cache-reuse-design.md @@ -0,0 +1,79 @@ +# Reuse completed work across pipeline runs + +Status: proposed for developer approval. Implementation and PR requested November 13, 2026. + +## Problem and evidence + +On main at 89160c47, a repeated job can hit every function-result cache while repeating +source reads, operators, row reconstruction and hashes. A 10,000-row source grouped into +100 function inputs took 329 ms on a warm job run, with one GroupBy execution and 200 Arrow +hashes; rereading the retained function output took 38 microseconds and performed neither. +The sync orchestrator reconstructs Data objects whose per-object hash cache is empty. +The async function path unconditionally reloads stored results. SpiralDB sources pin a +version at construction but scan it and rebuild provenance on every read. + +## Recommended approach + +Reuse completed node outputs against the upstream state that produced them, on both sync +and async execution paths. Keep the existing result database cache for new job objects, +changed inputs and recovery. Preserve row objects and their memoized hashes across sync +orchestration boundaries; table materialization remains available for operators that need +it. A cached result must retain its original schema, data context, system tags and per-row +source information. + +An execution cache is valid only after successful completion and only for the same inputs +and configuration. Rebinding a source, clearing a cache, changing an upstream generation +or using a dynamic source must prevent inappropriate whole-node reuse. Partial or failed +runs retain successful per-row work through the existing recovery mechanism but must not +be marked complete. Empty successful outputs are valid completed results. + +Side-effect completion checks, pending-write failures, observer lifecycle notifications +and RunSummary accounting must still run as required. An optimization must not skip a +side-effect safety check or silently turn a failed run into a successful cached run. + +For SpiralDB, keep construction lazy and retain a successfully completed read of the +source object's pinned version. Repeated iteration and table access reuse that snapshot. +Constructing and binding a new source reads a new snapshot. A failed or interrupted read +must not publish a truncated cache. The implementation must explicitly handle concurrent +sync/async readers and the unversioned empty-table case; unknown freshness is not proof +that a source is unchanged. Document the memory cost of retaining the read rather than +claiming bounded-memory streaming after caching is introduced. + +Do not change hash definitions, stored record identifiers or database layouts. Avoid +rewriting database indexing or query pushdown as part of this PR. + +## Alternatives considered + +1. Preserve hashes only: small, but sources, grouping and joins still repeat, leaving the + dominant ediop costs intact. +2. Skip the entire job after its first run: fast, but obscures per-node invalidation, + dynamic sources, side effects and failure recovery. +3. Reuse completed node/stream work (recommended): matches the intended stream semantics + and makes invalidation local and testable. + +## Verification + +Use counters and object identity assertions rather than elapsed-time thresholds: + +- An unchanged job run twice performs no new function calls, operator computation or + hashes of previously processed rows; exercise sync, async and switching orchestrators. +- Retained output rows preserve values, schemas, provenance and hashes, including joins, + grouping, filtering, empty output and logical types. +- A source rebind or explicit invalidation recomputes affected work and serves changed + results; dynamic/polling sources continue to observe new input. +- A failed or partial run retries missing work without losing completed rows. +- A pinned Spiral source is constructed without scans; its completed snapshot is scanned + once across repeated reads; interrupted/failed reads do not masquerade as complete. +- Side-effect guards and observer summaries retain their existing behavior. +- New job objects continue to recover results from the persistent store. + +Repeat the general flat and grouped benchmarks, then run relevant regression suites and +the repository's required lint, format, type and test checks. Include measured results and +the snapshot-memory tradeoff in the PR description. + +## Tracking and delivery + +No matching open Linear issue was found in the cache/rerun searches. Create a dedicated +issue and design-issue entry once approved, use the issue's branch name, write the detailed +implementation plan, add failing regressions, implement, review and open a PR against main. +No deployment, release or merge is included. From c58d6878d6a772305eb3041da7088b8c57df34df Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 02:10:24 -0800 Subject: [PATCH 2/7] fix(sources): retain completed pinned Spiral snapshots --- docs/concepts/sources.md | 25 ++- .../core/sources/spiraldb_table_source.py | 110 +++++++++--- .../sources/test_spiraldb_table_source.py | 2 +- .../test_spiraldb_table_source_streaming.py | 164 +++++++++++++++++- 4 files changed, 266 insertions(+), 35 deletions(-) diff --git a/docs/concepts/sources.md b/docs/concepts/sources.md index dc7075718..e210c5a23 100644 --- a/docs/concepts/sources.md +++ b/docs/concepts/sources.md @@ -115,14 +115,23 @@ Things to know about Spiral tables: Building a `SpiralDBTableSource` reads no rows. It reads the table's schema, key columns and table id, which takes about a second whatever the table's size. `output_schema()`, `keys()`, `content_hash()` and `pipeline_hash()` read no rows either. Rows are read when you ask for -them, and every read scans the table again: +them. The first complete read of a pinned version is retained for later reads: - `iter_data()` and a pipeline job hand the rows on in batches of at most `batch_rows` rows (default 8,192). SpiralDB sends a scan in batches that follow its stored fragments, - whose sizes cannot be chosen; one of them is held while its rows are handed on. + whose sizes cannot be chosen. Completed batch streams are retained in memory, and + their row objects and hashes are reused by later sync and async reads. - `as_table()`, `as_df()`, `as_polars_df()`, `as_pandas_df()` and `as_lazy_frame()` read the rows into memory and return them all. +Retention uses memory proportional to the full snapshot, even with `iter_data()`; +`batch_rows` bounds processing slices, not retained memory. Row objects are built only +when iteration needs them, so a table-only read retains Arrow streams without eagerly +constructing all row objects. Failed, cancelled, or partially consumed scans do not +publish a snapshot. Concurrent initial readers may scan independently; the first +complete read atomically publishes the retained snapshot. When no table version can +be pinned, each read scans again rather than reusing potentially stale rows. + To read part of a table, say so when you build the source. SpiralDB applies both restrictions before any row is transferred: @@ -148,13 +157,15 @@ SpiralDB records for the table. That count can include deleted rows that are not compacted, so a table near the cap can be refused although a scan would return fewer rows; the error says so. The cap is there because a pipeline job keeps every row it processes in memory, about 5 KB per row for each -node, so 100,000 rows cost about 0.5 GB per node. A table of millions of rows can be iterated -with `max_rows=None`, but it cannot yet be run through a job on a laptop. +node, so 100,000 rows cost about 0.5 GB per node. Removing the cap with `max_rows=None` +also permits a source to retain a snapshot of any size; it does not provide bounded-memory +iteration. **Looking at a table in a notebook.** A Spiral source shown at the end of a notebook cell (its `head()`, see [Looking at data](streams.md#looking-at-data)) reads only its first 10 rows, in one scan to which the row cap does not apply, so a table of any size can be looked -at; `head(n)` reads `n`. The header line says what is shown, for +at; `head(n)` reads `n`. After a complete snapshot has been retained, previews use +that snapshot and report its exact row count without another scan. The header line says what is shown, for example `SpiralDBTableSource[spike_event] — first 10 rows of about 63,349` (the table's recorded row count, read without scanning), `all 4 rows` when the source reads no more than that, or `first 10 matching rows (where unit_id=106); the table has about 63,349 rows` with @@ -169,8 +180,8 @@ job, and `as_table()` after an async run) returns every row that any job with th pipeline has read from that table, not only the rows of the latest restriction. The rows sent downstream during a run are only the rows that run read. -**Running again.** A second run scans the table again and computes nothing for the rows it -has already seen. A source object keeps reading the version its table had when it was +**Running again.** A second read reuses the completed pinned snapshot and its row +objects; a second run computes nothing for the rows it has already seen. A source object keeps reading the version its table had when it was built, so rows added to the Spiral table afterwards are seen only by a new source: build the source (and the job around it) again, and the added rows are new keys, the only rows computed. diff --git a/src/orcapod/core/sources/spiraldb_table_source.py b/src/orcapod/core/sources/spiraldb_table_source.py index eaf1abf8d..d5b04ec24 100644 --- a/src/orcapod/core/sources/spiraldb_table_source.py +++ b/src/orcapod/core/sources/spiraldb_table_source.py @@ -1,9 +1,9 @@ """SpiralDBTableSource — a read-only RootSource that streams a SpiralDB table. The source is built from the table's metadata alone (schema, key columns, SpiralDB's -table id): no row is read until one is asked for. Rows then arrive in batches, so a -read holds one of SpiralDB's batches at a time whatever the table's size. The table's -primary-key (key-schema) columns are the tag columns by default. +table id): no row is read until one is asked for. Rows arrive in batches, and a +completed pinned snapshot is retained for reuse. The table's primary-key +(key-schema) columns are the tag columns by default. Requires the ``spiraldb`` optional extra: ``pip install orcapod[spiraldb]``. Authentication is handled externally — run ``spiral login`` once; pyspiral keeps @@ -36,8 +36,9 @@ import asyncio import dataclasses import datetime +import threading from collections.abc import AsyncIterator, Collection, Generator, Iterator, Mapping -from contextlib import contextmanager +from contextlib import closing, contextmanager from typing import TYPE_CHECKING, Any, NamedTuple from orcapod.core.sources.base import RootSource @@ -158,12 +159,14 @@ class SpiralDBTableSource(RootSource): ``content_hash``, ``pipeline_hash`` and ``to_config`` never read a row either. Rows are read when asked for (``iter_data``, ``async_iter_data``, ``as_table`` and - the dataframe methods). Each read is one scan of the table in key order. SpiralDB - sends it in batches whose sizes follow its stored fragments (they cannot be - chosen); the source hands each on in slices of at most ``batch_rows`` rows, so - iterating holds one SpiralDB batch, and the slice being processed, at a time. - ``as_table`` and the dataframe methods hold every row they return. Nothing is - cached: every read scans again. + the dataframe methods). The first complete read scans the table in key order. + SpiralDB sends batches whose sizes follow its stored fragments (they cannot be + chosen); the source hands each on in slices of at most ``batch_rows`` rows. + A pinned source retains those streams and their lazily built row objects for + later reads, using memory proportional to the snapshot. Failed or partial reads + are not retained. Concurrent initial readers may scan independently; the first + complete read publishes the retained snapshot. Sources without a pinned version + scan on every read because their freshness cannot be established. Display. ``head(n)`` and the notebook display (its ``head()``) read only the first ``n`` rows (``PREVIEW_ROWS``, 10, by default), in one scan to which ``max_rows`` does @@ -277,6 +280,8 @@ def __init__( self._limit = limit self._max_rows = max_rows self._batch_rows = batch_rows + self._retained_streams: tuple[ArrowTableStream, ...] | None = None + self._snapshot_lock: threading.Lock = threading.Lock() self._given_asof = asof self._tags_as_data: dict[str, str] = dict(tags_as_data) if tags_as_data else {} location = f"SpiralDB project {project_id!r}, dataset {dataset!r}" @@ -485,18 +490,44 @@ def _key_tokens(self, table: "pa.Table") -> list[str]: for i in range(keys.num_rows) ] - def _batch_streams(self) -> Generator["ArrowTableStream", None, None]: - """Scan the table once and yield its rows as streams of at most ``batch_rows`` rows. - - Each SpiralDB batch is held while its slices are yielded. + def _batch_streams( + self, cancelled: threading.Event | None = None + ) -> Generator["ArrowTableStream", None, None]: + """Reuse a completed pinned snapshot, or retain a successful full scan. - Raises: - SourceTooLargeError: If the read would return, or returns, more than - ``max_rows`` rows. - ValueError: If a batch's columns differ from the source's schema, or a - key column holds a null. + Concurrent initial readers scan independently. Publication is atomic and + first-completion wins; no lock is held across a scan or a consumer yield. + Closing or failing a partial read leaves no retained snapshot. """ - return self._read_batches(limit=self._limit, max_rows=self._max_rows) + with self._snapshot_lock: + retained = self._retained_streams + if retained is not None: + yield from retained + return + streams: list[ArrowTableStream] = [] + with closing( + self._read_batches(limit=self._limit, max_rows=self._max_rows) + ) as batches: + for stream in batches: + if self._asof is not None: + streams.append(stream) + yield stream + if self._asof is not None: + with self._snapshot_lock: + if self._retained_streams is None and ( + cancelled is None or not cancelled.is_set() + ): + self._retained_streams = tuple(streams) + + def _stream_rows( + self, stream: ArrowTableStream + ) -> Iterator[tuple[TagProtocol, DataProtocol]]: + """Initialize a batch's row memo before concurrent readers can reuse it.""" + # ArrowTableStream initializes its memo only after iteration completes. + # Materialize just this batch under the lock; never hold it while yielding. + with self._snapshot_lock: + rows = list(stream.iter_data()) + yield from rows def _read_batches( self, *, limit: int | None, max_rows: int | None @@ -571,23 +602,26 @@ def _read_batches( ) def iter_data(self) -> Iterator["tuple[TagProtocol, DataProtocol]"]: - """Scan the table and yield its ``(tag, data)`` pairs, one batch held at a time.""" - for stream in self._batch_streams(): - yield from stream.iter_data() + """Yield ``(tag, data)`` pairs, reusing a completed pinned snapshot.""" + with closing(self._batch_streams()) as streams: + for stream in streams: + yield from self._stream_rows(stream) async def async_iter_data( self, ) -> AsyncIterator["tuple[TagProtocol, DataProtocol]"]: """Yield the table's ``(tag, data)`` pairs, reading each batch off the event loop.""" - streams = self._batch_streams() + cancelled = threading.Event() + streams = self._batch_streams(cancelled) try: while True: stream = await asyncio.to_thread(next, streams, None) if stream is None: break - for pair in stream.iter_data(): + for pair in self._stream_rows(stream): yield pair finally: + cancelled.set() try: streams.close() except ValueError: @@ -601,7 +635,7 @@ def as_table( columns: ColumnConfig | dict[str, Any] | None = None, all_info: bool = False, ) -> "pa.Table": - """Scan the table and return its rows as one PyArrow table (all held in memory).""" + """Return one PyArrow table, reusing a completed pinned snapshot when available.""" return self._whole_table( self._batch_streams(), columns=columns, all_info=all_info ) @@ -650,6 +684,28 @@ def _head( schema_only = self._stream.as_table(columns=columns, all_info=all_info) return _Head(schema_only, False, self.row_count_estimate, True) head_limit = n if self._limit is None else min(n, self._limit) + with self._snapshot_lock: + retained = self._retained_streams + if retained is not None: + remaining = head_limit + total_rows = 0 + previews: list[ArrowTableStream] = [] + for stream in retained: + table = stream.as_table(columns={"source": True, "system_tags": True}) + total_rows += table.num_rows + if remaining > 0: + selected = table.slice(0, remaining) + previews.append( + ArrowTableStream( + selected, + tag_columns=self._tag_columns, + system_tag_columns=self._system_tag_columns, + data_context=self.data_context, + ) + ) + remaining -= selected.num_rows + rows = self._whole_table(iter(previews), columns=columns, all_info=all_info) + return _Head(rows, True, total_rows, False) rows = self._whole_table( self._read_batches(limit=head_limit, max_rows=None), columns=columns, @@ -668,6 +724,8 @@ def _head_note(self, head: "_Head") -> str: if not head.rows_read: return f"rows not read; the table has about {head.total_rows:,} rows" if not head.total_is_estimate: + if shown < head.total_rows: + return f"first {shown:,} rows of {head.total_rows:,}" return f"all {shown:,} row{'' if shown == 1 else 's'}" if self._where: condition = ", ".join(f"{c}={v!r}" for c, v in self._where.items()) diff --git a/tests/test_core/sources/test_spiraldb_table_source.py b/tests/test_core/sources/test_spiraldb_table_source.py index 3dcfc30db..f6e6f7381 100644 --- a/tests/test_core/sources/test_spiraldb_table_source.py +++ b/tests/test_core/sources/test_spiraldb_table_source.py @@ -250,11 +250,11 @@ def connect(**kwargs): with patch(_CONNECTOR, side_effect=connect): src = SpiralDBTableSource(_PROJECT_ID, _TABLE_NAME) - src.as_table() # iter_data is a generator function, so the read can be closed. unfinished = cast("Generator[Any, None, None]", src.iter_data()) next(unfinished) # a read that is not finished unfinished.close() + src.as_table() with pytest.raises(ValueError): SpiralDBTableSource(_PROJECT_ID, "no_such_table") assert len(opened) == 4 diff --git a/tests/test_core/sources/test_spiraldb_table_source_streaming.py b/tests/test_core/sources/test_spiraldb_table_source_streaming.py index 384ead58e..7f911b5b5 100644 --- a/tests/test_core/sources/test_spiraldb_table_source_streaming.py +++ b/tests/test_core/sources/test_spiraldb_table_source_streaming.py @@ -283,7 +283,7 @@ def test_every_read_of_one_source_uses_the_version_it_was_built_at(self, project source.as_table() list(source.iter_data()) - assert [scan.kwargs["asof"] for scan in fake_pyspiral.scan_log] == [version] * 2 + assert [scan.kwargs["asof"] for scan in fake_pyspiral.scan_log] == [version] def test_a_given_version_is_used(self, project): """A given ``asof`` is passed to the scan and kept in the config.""" @@ -727,3 +727,165 @@ def test_switched_off_preview_shows_the_schema_without_a_scan( assert head.table.num_rows == 0 assert head.table.num_columns == 3 assert "tags: subject_id, timestamp" in html + + +class TestCompletedSnapshotRetention: + """Only fully consumed, pinned reads are retained for subsequent consumers.""" + + def test_completed_read_reuses_scan_rows_and_hashes(self, project): + source = SpiralDBTableSource(_PROJECT, "event") + assert fake_pyspiral.scan_log == [] + first = list(source.iter_data()) + hashes = [data.content_hash() for _, data in first] + source.as_table(all_info=True) + + async def collect(): + return [pair async for pair in source.async_iter_data()] + + again = asyncio.run(collect()) + assert len(fake_pyspiral.scan_log) == 1 + assert all(a is b and x is y for (a, x), (b, y) in zip(first, again)) + assert [data.content_hash() for _, data in again] == hashes + + def test_table_first_read_initializes_shared_rows(self, project): + source = SpiralDBTableSource(_PROJECT, "event") + source.as_table() + first = list(source.iter_data()) + second = list(source.iter_data()) + assert len(fake_pyspiral.scan_log) == 1 + assert all(x is y for (_, x), (_, y) in zip(first, second)) + + def test_partial_read_is_not_published(self, project): + source = SpiralDBTableSource(_PROJECT, "event") + partial = source.iter_data() + next(partial) + partial.close() + assert len(list(source.iter_data())) == _ROWS + assert len(list(source.iter_data())) == _ROWS + assert len(fake_pyspiral.scan_log) == 2 + + def test_failed_read_retries(self, project, monkeypatch): + from orcapod.databases.spiraldb_connector import SpiralDBConnector + + original = SpiralDBConnector.scan_batches + fail = True + + def scan(connector, *args, **kwargs): + nonlocal fail + for batch in original(connector, *args, **kwargs): + yield batch + if fail: + fail = False + raise RuntimeError("interrupted scan") + + monkeypatch.setattr(SpiralDBConnector, "scan_batches", scan) + source = SpiralDBTableSource(_PROJECT, "event") + with pytest.raises(RuntimeError, match="interrupted scan"): + list(source.iter_data()) + assert len(list(source.iter_data())) == _ROWS + assert len(list(source.iter_data())) == _ROWS + assert len(fake_pyspiral.scan_log) == 2 + + def test_overlapping_readers_never_share_partial_snapshot(self, project): + source = SpiralDBTableSource(_PROJECT, "event") + partial = source.iter_data() + next(partial) + completed = list(source.iter_data()) + partial.close() + again = list(source.iter_data()) + assert len(completed) == len(again) == _ROWS + assert all(x is y for (_, x), (_, y) in zip(completed, again)) + assert len(fake_pyspiral.scan_log) == 2 + + def test_empty_pinned_snapshot_is_retained(self, project): + source = SpiralDBTableSource(_PROJECT, "event", where={"subject_id": "absent"}) + assert list(source.iter_data()) == [] + assert source.as_table().num_rows == 0 + assert len(fake_pyspiral.scan_log) == 1 + + def test_unpinned_source_keeps_reading(self, project, monkeypatch): + from orcapod.databases.spiraldb_connector import SpiralDBConnector + + monkeypatch.setattr(SpiralDBConnector, "get_table_version", lambda *args: None) + source = SpiralDBTableSource(_PROJECT, "event") + assert len(list(source.iter_data())) == _ROWS + project.table("default.event").write(_rows([6])) + assert len(list(source.iter_data())) == _ROWS + 1 + assert len(fake_pyspiral.scan_log) == 2 + + def test_async_partial_read_is_not_published(self, project): + source = SpiralDBTableSource(_PROJECT, "event") + + async def partial(): + rows = source.async_iter_data() + await anext(rows) + await rows.aclose() + + asyncio.run(partial()) + assert len(list(source.iter_data())) == _ROWS + assert len(list(source.iter_data())) == _ROWS + assert len(fake_pyspiral.scan_log) == 2 + + def test_concurrent_sync_and_async_consumers(self, project): + source = SpiralDBTableSource(_PROJECT, "event") + source.as_table() + + async def collect(): + async def asynchronous(): + return [pair async for pair in source.async_iter_data()] + + return await asyncio.gather( + asyncio.to_thread(lambda: list(source.iter_data())), asynchronous() + ) + + first, second = asyncio.run(collect()) + assert len(first) == len(second) == _ROWS + assert all(a is b and x is y for (a, x), (b, y) in zip(first, second)) + assert len(fake_pyspiral.scan_log) == 1 + + def test_cancellation_during_scan_does_not_publish(self, project, monkeypatch): + import threading + + from orcapod.databases.spiraldb_connector import SpiralDBConnector + + entered = threading.Event() + release = threading.Event() + finished = threading.Event() + original = SpiralDBConnector.scan_batches + + def scan(connector, *args, **kwargs): + try: + entered.set() + assert release.wait(5) + yield from original(connector, *args, **kwargs) + finally: + finished.set() + + monkeypatch.setattr(SpiralDBConnector, "scan_batches", scan) + source = SpiralDBTableSource(_PROJECT, "event") + + async def cancel(): + rows = source.async_iter_data() + task = asyncio.create_task(anext(rows)) + assert await asyncio.to_thread(entered.wait, 5) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + release.set() + + asyncio.run(cancel()) + # The cancelled worker may have read one batch, never a full snapshot. + assert len(list(source.iter_data())) == _ROWS + assert len(list(source.iter_data())) == _ROWS + assert finished.wait(5) + assert len(fake_pyspiral.scan_log) == 2 + + def test_head_reuses_completed_snapshot(self, project): + source = SpiralDBTableSource(_PROJECT, "event") + expected = source.head(2).as_table() + assert len(fake_pyspiral.scan_log) == 1 + source.as_table() + assert source.head(2).as_table().equals(expected) + assert source.head(2).total_rows == _ROWS + assert source.head(2).total_is_estimate is False + assert len(fake_pyspiral.scan_log) == 2 From f08ed22edc34366db440358b463feb722c670eb2 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 02:40:26 -0800 Subject: [PATCH 3/7] fix(pipeline): reuse completed work and retained row hashes --- changelog.d/ITL-1063.md | 12 + design_issues/ITL-1063-CR1.md | 13 +- src/orcapod/core/nodes/_execution_cache.py | 296 +++++++++++++++ src/orcapod/core/nodes/function_node.py | 50 ++- src/orcapod/core/nodes/operator_node.py | 54 +++ .../core/operators/static_output_pod.py | 10 +- .../core/sources/spiraldb_table_source.py | 6 +- .../core/streams/arrow_table_stream.py | 11 +- src/orcapod/pipeline/async_orchestrator.py | 35 +- src/orcapod/pipeline/sync_orchestrator.py | 80 +++- .../plans/2026-11-13-itl-1063-cache-reuse.md | 38 +- .../2026-11-13-pipeline-cache-reuse-design.md | 62 ++- .../operators/test_async_sync_equivalence.py | 2 - .../test_spiraldb_table_source_streaming.py | 26 +- .../test_execution_cache_reuse.py | 355 ++++++++++++++++++ .../test_spiral_execution_cache_reuse.py | 67 ++++ .../test_system_tag_run_stability.py | 4 +- .../test_tag_table_source_info.py | 12 + 18 files changed, 1082 insertions(+), 51 deletions(-) create mode 100644 changelog.d/ITL-1063.md create mode 100644 src/orcapod/core/nodes/_execution_cache.py create mode 100644 tests/test_pipeline/test_execution_cache_reuse.py create mode 100644 tests/test_pipeline/test_spiral_execution_cache_reuse.py diff --git a/changelog.d/ITL-1063.md b/changelog.d/ITL-1063.md new file mode 100644 index 000000000..8280d32de --- /dev/null +++ b/changelog.d/ITL-1063.md @@ -0,0 +1,12 @@ +### Fixed + +- Repeated pipeline runs reuse completed operator work and retained row hashes when + their input snapshots are unchanged. Sync execution preserves row objects across + node boundaries; async execution reuses valid in-memory function results. Unknown + or changing inputs, explicit invalidation and failed work remain subject to normal + execution and recovery. +- A SpiralDB source retains a successfully read pinned snapshot for later sync, + async and table reads. Construction remains lazy; partial, failed and cancelled + reads do not publish a snapshot. Retention uses memory proportional to the snapshot; + `batch_rows` bounds processing slices, not total retained memory. Construct a new + source to read a newer table version. diff --git a/design_issues/ITL-1063-CR1.md b/design_issues/ITL-1063-CR1.md index 1ee293255..d05cbcd5e 100644 --- a/design_issues/ITL-1063-CR1.md +++ b/design_issues/ITL-1063-CR1.md @@ -1,7 +1,7 @@ ## `src/orcapod/pipeline/` — completed work on unchanged reruns ### CR1 — Job reruns rebuild rows and hashes despite retained function results -**Status:** in progress +**Status:** resolved **Severity:** high **Issue:** ITL-1063 @@ -18,3 +18,14 @@ read reuses its objects and does not repeat this work. Approved design: `superpowers/specs/2026-11-13-pipeline-cache-reuse-design.md`. Reproduction: `uv run python bench/bench_pipeline_cached_rerun.py --rows 1000`. + +Completed graph execution now retains operator results and exact-row entry IDs against +known unchanged snapshots. Sync boundaries retain Arrow-decoded row views to preserve +declared-schema behavior; async functions reuse their valid in-memory results. Pinned +Spiral sources publish retained batch streams only after complete successful reads. +Unknown sources, direct node calls, changed bindings and failed work remain conservative. + +Validated with 11,625 passing tests (119 skipped, 19 expected failures), zero type-check +errors, clean Ruff checks and independent review. Warm benchmark runs perform zero new +Arrow hashes and zero GroupBy computations. Remaining traversal/materialization costs +and potential further optimizations are tracked by ITL-1064. diff --git a/src/orcapod/core/nodes/_execution_cache.py b/src/orcapod/core/nodes/_execution_cache.py new file mode 100644 index 000000000..29e4aec10 --- /dev/null +++ b/src/orcapod/core/nodes/_execution_cache.py @@ -0,0 +1,296 @@ +"""Completion proofs scoped to the exact graph inputs an orchestrator delivers. + +Source identity hashes do not prove freshness. Only known immutable snapshots +qualify; direct node calls, unknown sources and incomplete runs never qualify. +""" + +from __future__ import annotations + +import asyncio +import functools +import inspect +from collections.abc import Callable +from contextvars import ContextVar +from typing import Any, TypeVar, cast + +F = TypeVar("F", bound=Callable[..., Any]) + + +class _Identity: + """Retain an object and compare by identity, never by its content hash.""" + + def __init__(self, value: object) -> None: + self.value = value + + def __eq__(self, other: object) -> bool: + return isinstance(other, _Identity) and self.value is other.value + + +class _Run: + """Freeze source bindings, configuration and graph edges before any callbacks.""" + + def __init__(self, graph: Any, kinds: dict[str, tuple[type, ...]]) -> None: + self.kinds = kinds + self.succeeded: set[int] = set() + self.done: dict[int, asyncio.Event] = {} + self.nodes = {id(node): node for node in graph.topological_sort()} + self.aligned = {} + for node in self.nodes.values(): + upstreams = getattr(node, "upstreams", None) + if upstreams is None: + self.aligned[id(node)] = False + continue + order = {id(upstream): index for index, upstream in enumerate(upstreams)} + predecessors = sorted( + graph.predecessors(node), key=lambda p: order.get(id(p), 0) + ) + self.aligned[id(node)] = tuple(id(p) for p in predecessors) == tuple( + id(p) for p in upstreams + ) + self.initial = {key: self.definition(node) for key, node in self.nodes.items()} + + def definition(self, stream: Any) -> object | None: + """Describe the graph independently of whether its work has completed.""" + kind = type(stream) + if kind in self.kinds["source_node"]: + bound = self.definition(stream.bound_source) + return None if bound is None else (_Identity(stream), bound) + if kind in self.kinds["arrow"]: + return (_Identity(stream), stream.last_modified, stream.data_context_key) + if kind in self.kinds["memory_source"]: + underlying = self.definition(stream._stream) + return None if underlying is None else (_Identity(stream), underlying) + if kind in self.kinds["spiral"] and stream._asof is not None: + return (_Identity(stream), stream._asof, stream.data_context_key) + if kind not in self.kinds["computed"]: + return None + upstreams = tuple(self.definition(s) for s in stream.upstreams) + if not upstreams or any(s is None for s in upstreams): + return None + producer = stream.producer + return ( + upstreams, + getattr(stream, "_execution_generation", 0), + _Identity(producer), + producer.last_modified if producer is not None else None, + _Identity(getattr(stream, "_node_config", None)), + _Identity(getattr(stream, "_pipeline_database", None)), + _Identity(getattr(stream, "_ephemeral_cached_pod", None)), + stream.data_context_key, + ) + + def proof(self, node: Any) -> object | None: + """Require unchanged run-start inputs and successfully completed predecessors.""" + key = id(node) + if key not in self.nodes or not self.aligned[key]: + return None + definition = self.definition(node) + if definition is None or definition != self.initial[key]: + return None + for upstream in node.upstreams: + if type(upstream) in self.kinds["computed"]: + proof = self.proof(upstream) + if proof is None or proof != getattr( + upstream, "_completed_execution", None + ): + return None + return definition + + +_RUN: ContextVar[_Run | None] = ContextVar("execution_run", default=None) +_TICKET: ContextVar[object | None] = ContextVar("execution_ticket", default=None) +_ACTIVE: ContextVar[object | None] = ContextVar("execution_active", default=None) + + +def verified_graph_execution(method: F) -> F: + """Freeze a graph before execution; node calls still require a one-use ticket.""" + + def setup(args: tuple[Any, ...], kwargs: dict[str, Any]) -> _Run: + orchestrator = args[0] + graph = args[1] if len(args) > 1 else kwargs["graph"] + return _Run(graph, orchestrator._execution_snapshot_types()) + + if inspect.iscoroutinefunction(method): + + @functools.wraps(method) + async def asynchronous(*args: Any, **kwargs: Any) -> Any: + token = _RUN.set(setup(args, kwargs)) + try: + return await method(*args, **kwargs) + finally: + _RUN.reset(token) + + return cast(F, asynchronous) + + @functools.wraps(method) + def synchronous(*args: Any, **kwargs: Any) -> Any: + token = _RUN.set(setup(args, kwargs)) + try: + return method(*args, **kwargs) + finally: + _RUN.reset(token) + + return cast(F, synchronous) + + +def execute_node( + node: Any, method: Callable[..., Any], *args: Any, **kwargs: Any +) -> Any: + """Authorize one orchestrator-owned synchronous invocation.""" + token = _TICKET.set(node) + try: + result = method(*args, **kwargs) + run = _RUN.get() + if run is not None: + run.succeeded.add(id(node)) + return result + finally: + _TICKET.reset(token) + + +async def async_execute_node( + node: Any, method: Callable[..., Any], *args: Any, **kwargs: Any +) -> Any: + """Authorize one orchestrator-owned asynchronous invocation.""" + token = _TICKET.set(node) + run = _RUN.get() + try: + result = await method(*args, **kwargs) + if run is not None: + run.succeeded.add(id(node)) + return result + finally: + if run is not None: + run.done.setdefault(id(node), asyncio.Event()).set() + _TICKET.reset(token) + + +async def upstreams_succeeded(node: Any) -> bool: + """Wait for producer tasks, since channel EOF can also mean producer failure.""" + run = _RUN.get() + if run is None: + return False + for upstream in node.upstreams: + await run.done.setdefault(id(upstream), asyncio.Event()).wait() + return all(id(upstream) in run.succeeded for upstream in node.upstreams) + + +def is_current(node: Any) -> bool: + """Whether retained work matches this exact trusted graph invocation.""" + run = _RUN.get() + previous = getattr(node, "_completed_execution", None) + return ( + _ACTIVE.get() is node + and run is not None + and previous is not None + and previous == run.proof(node) + ) + + +def can_retain_rows(node: Any) -> bool: + """Keep row memoization only within an unchanged, known graph snapshot.""" + run = _RUN.get() + if run is None or _ACTIVE.get() is not node: + return False + initial = run.initial.get(id(node)) + return initial is not None and initial == run.definition(node) + + +def invalidate(node: Any) -> None: + """Forget completion and row memoization after an explicit cache clear.""" + node._completed_execution = None + node._execution_generation = getattr(node, "_execution_generation", 0) + 1 + node._sync_row_views = {} + node._entry_id_rows = {} + + +class _TrackingObserver: + """Forward hooks while remembering row failures across contextual observers.""" + + def __init__(self, observer: Any, failures: list[bool]) -> None: + self.observer = observer + self.failures = failures + + def __getattr__(self, name: str) -> Any: + return ( + getattr(self.observer, name) + if self.observer is not None + else lambda *a, **kw: None + ) + + def contextualize(self, *path: str) -> _TrackingObserver: + return ( + _TrackingObserver(self.observer.contextualize(*path), self.failures) + if self.observer is not None + else self + ) + + def on_data_crash(self, *args: Any, **kwargs: Any) -> None: + self.failures.append(True) + if self.observer is not None: + self.observer.on_data_crash(*args, **kwargs) + + +def publish_completion(node: Any, observer: Any) -> None: + """Publish success before output closes, never after failed or changed work.""" + run = _RUN.get() + node._completed_execution = ( + run.proof(node) + if run is not None and _ACTIVE.get() is node and not observer.failures + else None + ) + if node._completed_execution is None: + node._entry_id_rows = {} + + +def track_completion(method: F) -> F: + """Track completion without changing per-row execution, persistence or hooks.""" + + def setup(node: Any, kwargs: dict[str, Any]) -> _TrackingObserver: + if not is_current(node): + node._entry_id_rows = {} + observer = _TrackingObserver(kwargs.get("observer"), []) + kwargs["observer"] = observer + return observer + + if inspect.iscoroutinefunction(method): + + @functools.wraps(method) + async def asynchronous(node: Any, *args: Any, **kwargs: Any) -> Any: + trusted = _TICKET.get() is node + ticket = _TICKET.set(None) + active = _ACTIVE.set(node if trusted else None) + observer = setup(node, kwargs) + try: + result = await method(node, *args, **kwargs) + publish_completion(node, observer) + return result + except BaseException: + node._completed_execution = None + node._entry_id_rows = {} + raise + finally: + _ACTIVE.reset(active) + _TICKET.reset(ticket) + + return cast(F, asynchronous) + + @functools.wraps(method) + def synchronous(node: Any, *args: Any, **kwargs: Any) -> Any: + trusted = _TICKET.get() is node + ticket = _TICKET.set(None) + active = _ACTIVE.set(node if trusted else None) + observer = setup(node, kwargs) + try: + result = method(node, *args, **kwargs) + publish_completion(node, observer) + return result + except BaseException: + node._completed_execution = None + node._entry_id_rows = {} + raise + finally: + _ACTIVE.reset(active) + _TICKET.reset(ticket) + + return cast(F, synchronous) diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index dca13d7d4..92dd31e00 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -25,6 +25,13 @@ from collections.abc import Callable, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Literal, NamedTuple, cast +from orcapod.core.nodes._execution_cache import ( + publish_completion, + can_retain_rows, + invalidate, + is_current, + track_completion, +) from orcapod import contexts from orcapod.channels import Channel, ReadableChannel, WritableChannel from orcapod.config import OrcapodConfig @@ -41,7 +48,7 @@ StreamHead, ) from orcapod.core.tracker import DEFAULT_TRACKER_MANAGER -from orcapod.core.datagrams.tag_data import EmptyData, Tag +from orcapod.core.datagrams.tag_data import Data, EmptyData, Tag from orcapod.errors import ( EphemeralResultMissingError, InputValidationError, @@ -1484,6 +1491,7 @@ def from_descriptor( def clear_cache(self) -> None: """Clear in-memory output caches, content hash cache, and node identity path cache.""" + invalidate(self) super().clear_cache() self._cached_output_datas.clear() self._cached_output_table = None @@ -1836,6 +1844,7 @@ def execute_data( tag_out, result = self._process_data_internal(tag, data) return tag_out, result + @track_completion def execute( self, input_stream: StreamProtocol, @@ -2294,9 +2303,35 @@ def compute_base_entry_id( Returns: A bytes value in ``b"{method}:{digest}"`` format. """ - return self.data_context.arrow_hasher.hash_table( + if ( + not can_retain_rows(self) + or type(tag) is not Tag + or type(input_data) not in (Data, EmptyData) + ): + return self.data_context.arrow_hasher.hash_table( + self._build_entry_id_preimage(tag, input_data) + ).to_prefixed_digest() + # Hold the actual immutable rows, not bare ids that could be recycled. + rows = getattr(self, "_entry_id_rows", None) + if rows is None: + rows = self._entry_id_rows = {} + # A sync storage view shares its origin's exact Arrow table and hashes. + # Recognize that representation-only boundary across orchestrators too. + tag_identity = getattr(tag, "_stored_row_origin", tag) + row_identity = getattr(input_data, "_stored_row_origin", input_data) + key = (id(tag_identity), id(row_identity), self.data_context_key) + cached = rows.get(key) + if ( + cached is not None + and cached[0] is tag_identity + and cached[1] is row_identity + ): + return cached[2] + entry_id = self.data_context.arrow_hasher.hash_table( self._build_entry_id_preimage(tag, input_data) ).to_prefixed_digest() + rows[key] = (tag_identity, row_identity, entry_id) + return entry_id def compute_pipeline_entry_id( self, @@ -3246,6 +3281,7 @@ def run(self) -> None: # Async channel execution # ------------------------------------------------------------------ + @track_completion async def async_execute( self, input_channel: ReadableChannel[tuple[TagProtocol, DataProtocol]], @@ -3338,14 +3374,14 @@ async def async_execute( execution_pod = self._cached_function_pod # Load pipeline DB → in-memory cache keyed by base_entry_id. - loaded = self._load_cached_entries() + loaded = {} if is_current(self) else self._load_cached_entries() self._cached_output_datas.update(loaded) if loaded: self._cached_output_table = None self._cached_content_hash_column = None cached_by_base_entry_id: dict[ - bytes, tuple[TagProtocol, DataProtocol] - ] = dict(loaded) + bytes, tuple[TagProtocol, DataProtocol | None] + ] = dict(self._cached_output_datas) # Intermediate channels (bounded for backpressure). compute_channel: Channel[tuple[TagProtocol, DataProtocol]] = Channel( @@ -3379,7 +3415,8 @@ async def route_inputs() -> None: ctx_obs.on_data_end( node_label, tag, data, cached_data, cached=True ) - await output.send((cached_tag, cached_data)) + if cached_data is not None: + await output.send((cached_tag, cached_data)) else: correlation_key = uuid.uuid4().bytes input_store[correlation_key] = ( @@ -3557,6 +3594,7 @@ def create_data_logger(self, tag: Any, data: Any) -> Any: # the finally closes the output, so every downstream node, which stamps # after it has seen that close, is newer than this one (ITL-1042). self._update_modified_time() + publish_completion(self, observer) except Exception as run_error: self._commit_after_failure(run_error) raise diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index 624be3b4f..2535f4d42 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -22,6 +22,13 @@ from collections.abc import Collection, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Literal, cast +from orcapod.core.nodes._execution_cache import ( + publish_completion, + invalidate, + is_current, + track_completion, + upstreams_succeeded, +) from orcapod import contexts from orcapod.channels import Channel, ReadableChannel, WritableChannel from orcapod.config import OrcapodConfig @@ -741,6 +748,7 @@ def set_ephemeral_store(self, store: "ArrowDatabaseProtocol | None") -> None: def clear_cache(self) -> None: """Clear output caches and node identity path cache.""" + invalidate(self) super().clear_cache() # clears _node_identity_path_cache + _update_modified_time self._cached_output_stream = None self._cached_output_table = None @@ -1279,6 +1287,7 @@ def get_cached_output(self) -> StreamProtocol | None: self._replay_from_cache() return self._cached_output_stream + @track_completion def execute( self, *input_streams: StreamProtocol, @@ -1306,6 +1315,12 @@ def execute( ctx_obs.on_node_start(node_label, node_hash) + if is_current(self) and self._cached_output_stream is not None: + output = list(self._cached_output_stream.iter_data()) + self._update_modified_time() + ctx_obs.on_node_end(node_label, node_hash) + return output + # Check REPLAY cache first cached_output = self.get_cached_output() if cached_output is not None: @@ -1643,6 +1658,7 @@ def as_source(self, tags_as_data: "Mapping[str, str] | None" = None): # Async channel execution # ------------------------------------------------------------------ + @track_completion async def async_execute( self, inputs: Sequence[ReadableChannel[tuple[TagProtocol, DataProtocol]]], @@ -1695,6 +1711,23 @@ async def async_execute( ctx_obs.on_node_end(node_label, node_hash) return # finally block closes output + buffered_inputs: list[list[tuple[TagProtocol, DataProtocol]]] | None = None + if is_current(self) and self._cached_output_stream is not None: + # Drain every input concurrently. EOF alone is insufficient: an + # upstream closes its channel on failure too. Never deliver cached + # rows (and downstream side effects) before producers succeeded. + buffered_inputs = list( + await asyncio.gather(*(reader.collect() for reader in inputs)) + ) + if not await upstreams_succeeded(self): + return + if is_current(self): + for row in self._cached_output_stream.iter_data(): + await output.send(row) + self._update_modified_time() + ctx_obs.on_node_end(node_label, node_hash) + return + # Delegate to the operator, forward its rows downstream and keep them. intermediate: Channel[tuple[TagProtocol, DataProtocol]] = Channel() should_store = ( @@ -1715,10 +1748,30 @@ async def forward() -> None: collected.append(item) await output.send(item) + replay_channels: list[Channel[tuple[TagProtocol, DataProtocol]]] = [] + if buffered_inputs is not None: + replay_channels = [Channel(buffer_size=16) for _ in buffered_inputs] + inputs = [channel.reader for channel in replay_channels] + + async def feed_buffer( + rows: list[tuple[TagProtocol, DataProtocol]], + channel: Channel[tuple[TagProtocol, DataProtocol]], + ) -> None: + try: + for row in rows: + await channel.writer.send(row) + finally: + await channel.writer.close() + hashes = [s.pipeline_hash() for s in self._input_streams] operator = cast(StaticOutputOperatorPod, self._operator) try: async with asyncio.TaskGroup() as tg: + if buffered_inputs is not None: + for rows, channel in zip( + buffered_inputs, replay_channels, strict=True + ): + tg.create_task(feed_buffer(rows, channel)) tg.create_task( operator.async_execute( inputs, @@ -1755,5 +1808,6 @@ async def forward() -> None: self._update_modified_time() ctx_obs.on_node_end(node_label, node_hash) + publish_completion(self, observer) finally: await output.close() diff --git a/src/orcapod/core/operators/static_output_pod.py b/src/orcapod/core/operators/static_output_pod.py index 34ffa81db..5747aa1f4 100644 --- a/src/orcapod/core/operators/static_output_pod.py +++ b/src/orcapod/core/operators/static_output_pod.py @@ -277,7 +277,15 @@ def _materialize_to_stream( table, tag_keys = StaticOutputOperatorPod._rows_to_table(rows) # No ``source_info`` argument: the table's own ``_source_*`` columns # pass through, one value per row. - return ArrowTableStream(table, tag_columns=tag_keys) + stream = ArrowTableStream( + table, + tag_columns=tag_keys, + data_context=getattr( + rows[0][1], "data_context", rows[0][1].data_context_key + ), + ) + stream._cached_elements = list(rows) + return stream @staticmethod def _materialize_input_stream( diff --git a/src/orcapod/core/sources/spiraldb_table_source.py b/src/orcapod/core/sources/spiraldb_table_source.py index d5b04ec24..3a3869473 100644 --- a/src/orcapod/core/sources/spiraldb_table_source.py +++ b/src/orcapod/core/sources/spiraldb_table_source.py @@ -37,7 +37,7 @@ import dataclasses import datetime import threading -from collections.abc import AsyncIterator, Collection, Generator, Iterator, Mapping +from collections.abc import AsyncGenerator, Collection, Generator, Iterator, Mapping from contextlib import closing, contextmanager from typing import TYPE_CHECKING, Any, NamedTuple @@ -601,7 +601,7 @@ def _read_batches( provenance_tokens=tokens, ) - def iter_data(self) -> Iterator["tuple[TagProtocol, DataProtocol]"]: + def iter_data(self) -> Generator[tuple[TagProtocol, DataProtocol], None, None]: """Yield ``(tag, data)`` pairs, reusing a completed pinned snapshot.""" with closing(self._batch_streams()) as streams: for stream in streams: @@ -609,7 +609,7 @@ def iter_data(self) -> Iterator["tuple[TagProtocol, DataProtocol]"]: async def async_iter_data( self, - ) -> AsyncIterator["tuple[TagProtocol, DataProtocol]"]: + ) -> AsyncGenerator[tuple[TagProtocol, DataProtocol], None]: """Yield the table's ``(tag, data)`` pairs, reading each batch off the event loop.""" cancelled = threading.Event() streams = self._batch_streams(cancelled) diff --git a/src/orcapod/core/streams/arrow_table_stream.py b/src/orcapod/core/streams/arrow_table_stream.py index 721f59e36..e68ee4036 100644 --- a/src/orcapod/core/streams/arrow_table_stream.py +++ b/src/orcapod/core/streams/arrow_table_stream.py @@ -13,7 +13,12 @@ head_column_config, ) from orcapod.core.streams.head import StreamHead -from orcapod.protocols.core_protocols import PodProtocol, StreamProtocol, TagProtocol +from orcapod.protocols.core_protocols import ( + DataProtocol, + PodProtocol, + StreamProtocol, + TagProtocol, +) from orcapod.protocols.hashing_protocols import PipelineElementProtocol from orcapod.system_constants import constants from orcapod.types import ColumnConfig, Schema, SourceInfoValue @@ -140,7 +145,7 @@ def __init__( self._all_tag_schema = all_tag_schema self._data_schema = data_schema - self._cached_elements: list[tuple[TagProtocol, Data]] | None = None + self._cached_elements: list[tuple[TagProtocol, DataProtocol]] | None = None self._update_modified_time() # set modified time to now def identity_structure(self) -> Any: @@ -316,7 +321,7 @@ def clear_cache(self) -> None: """ self._cached_elements = None - def iter_data(self) -> Iterator[tuple[TagProtocol, Data]]: + def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: """Iterates over the rows of the stream as ``(Tag, Data)`` pairs. Every row of the backing table is yielded exactly once, in table order, diff --git a/src/orcapod/pipeline/async_orchestrator.py b/src/orcapod/pipeline/async_orchestrator.py index 07f53b7f7..261741066 100644 --- a/src/orcapod/pipeline/async_orchestrator.py +++ b/src/orcapod/pipeline/async_orchestrator.py @@ -19,6 +19,10 @@ from typing import TYPE_CHECKING, Any, Literal, TypeVar from orcapod.channels import BroadcastChannel, Channel +from orcapod.core.nodes._execution_cache import ( + async_execute_node, + verified_graph_execution, +) from orcapod.errors import StoreError, StoreWriteError, TypeIdentityError from orcapod.pipeline.result import OrchestratorResult from orcapod.utils.store_errors import keep_store_error @@ -59,6 +63,13 @@ class AsyncPipelineOrchestrator: buffer_size: Channel buffer size. Defaults to 64. """ + @staticmethod + def _execution_snapshot_types() -> dict[str, tuple[type, ...]]: + """Use the same immutable snapshot boundary as synchronous execution.""" + from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator + + return SyncPipelineOrchestrator._execution_snapshot_types() + def __init__( self, buffer_size: int = 64, @@ -173,6 +184,7 @@ async def run_async( pipeline_uri=pipeline_uri, ) + @verified_graph_execution async def _run_async( self, graph: "GraphProtocol[Any]", @@ -244,7 +256,12 @@ async def _run_async( _start_node( tg, node, - node.async_execute(writer, observer=effective_observer), + async_execute_node( + node, + node.async_execute, + writer, + observer=effective_observer, + ), ) elif is_function_node(node) or is_side_effect_function_node(node): predecessors = in_edges.get(node, []) @@ -256,7 +273,9 @@ async def _run_async( _start_node( tg, node, - node.async_execute( + async_execute_node( + node, + node.async_execute, input_reader, writer, observer=effective_observer, @@ -279,8 +298,12 @@ async def _run_async( _start_node( tg, node, - node.async_execute( - input_readers, writer, observer=effective_observer + async_execute_node( + node, + node.async_execute, + input_readers, + writer, + observer=effective_observer, ), ) elif is_side_effect_node(node): @@ -293,7 +316,9 @@ async def _run_async( _start_node( tg, node, - node.async_execute( + async_execute_node( + node, + node.async_execute, [input_reader], writer, observer=effective_observer, diff --git a/src/orcapod/pipeline/sync_orchestrator.py b/src/orcapod/pipeline/sync_orchestrator.py index 981706352..436f7628e 100644 --- a/src/orcapod/pipeline/sync_orchestrator.py +++ b/src/orcapod/pipeline/sync_orchestrator.py @@ -10,6 +10,7 @@ import uuid from typing import TYPE_CHECKING, Any, Literal +from orcapod.core.nodes._execution_cache import execute_node, verified_graph_execution from orcapod.errors import StoreError, TypeIdentityError from orcapod.pipeline.result import OrchestratorResult from orcapod.utils import arrow_utils @@ -41,12 +42,29 @@ class SyncPipelineOrchestrator: buffer management, and stream materialization between nodes. """ + @staticmethod + def _execution_snapshot_types() -> dict[str, tuple[type, ...]]: + """Known immutable source implementations and completion-aware node kinds.""" + from orcapod.core.nodes import FunctionJobNode, OperatorJobNode, SourceJobNode + from orcapod.core.sources import ArrowTableSource, DictSource, ListSource + from orcapod.core.sources.spiraldb_table_source import SpiralDBTableSource + from orcapod.core.streams.arrow_table_stream import ArrowTableStream + + return { + "source_node": (SourceJobNode,), + "computed": (FunctionJobNode, OperatorJobNode), + "arrow": (ArrowTableStream,), + "memory_source": (ArrowTableSource, DictSource, ListSource), + "spiral": (SpiralDBTableSource,), + } + def __init__( self, error_policy: Literal["continue", "fail_fast"] = "continue", ) -> None: self._error_policy = error_policy + @verified_graph_execution def run( self, graph: GraphProtocol[Any], @@ -89,14 +107,18 @@ def run( for node in topo_order: current = node if is_source_node(node): - buffers[node] = node.execute(observer=effective_observer) + buffers[node] = execute_node( + node, node.execute, observer=effective_observer + ) elif is_function_node(node) or is_side_effect_function_node(node): upstream_buf = self._gather_upstream(node, graph, buffers) upstream_node = list(graph.predecessors(node))[0] input_stream = self._materialize_as_stream( upstream_buf, upstream_node ) - buffers[node] = node.execute( + buffers[node] = execute_node( + node, + node.execute, input_stream, observer=effective_observer, error_policy=self._error_policy, @@ -108,8 +130,8 @@ def run( self._materialize_as_stream(buf, upstream_node) for buf, upstream_node in upstream_buffers ] - buffers[node] = node.execute( - *input_streams, observer=effective_observer + buffers[node] = execute_node( + node, node.execute, *input_streams, observer=effective_observer ) elif is_side_effect_node(node): upstream_buf = self._gather_upstream(node, graph, buffers) @@ -117,7 +139,9 @@ def run( input_stream = self._materialize_as_stream( upstream_buf, upstream_node ) - buffers[node] = node.execute( + buffers[node] = execute_node( + node, + node.execute, input_stream, observer=effective_observer, run_id=run_id, @@ -204,6 +228,7 @@ def _materialize_as_stream(buf: list[tuple[Any, Any]], upstream_node: Any) -> An from orcapod.core.streams.arrow_table_stream import ArrowTableStream if not buf: + upstream_node._sync_row_views = {} # Build an empty stream preserving declared field nullability. tag_schema, data_schema = upstream_node.output_schema( columns={"system_tags": True, "source": True} @@ -218,6 +243,7 @@ def _materialize_as_stream(buf: list[tuple[Any, Any]], upstream_node: Any) -> An tag_columns=tag_keys, producer=upstream_node.producer, upstreams=upstream_node.upstreams, + data_context=upstream_node.data_context, ) from orcapod.core.operators.static_output_pod import StaticOutputOperatorPod @@ -233,12 +259,54 @@ def _materialize_as_stream(buf: list[tuple[Any, Any]], upstream_node: Any) -> An producer = upstream_node.producer upstreams = upstream_node.upstreams - return ArrowTableStream( + stream = ArrowTableStream( full_table, tag_columns=user_tag_keys, producer=producer, upstreams=upstreams, + data_context=upstream_node.data_context, ) + # Sync boundaries have always exposed the Arrow-stored representation. + # A dict-born Data can still hold a subclass with fields omitted by its + # declared storage type. Keep a separate, memoized storage view instead of + # mutating the live row (async and standalone paths still see that row). + from orcapod.core.datagrams import Data, Tag + + previous = getattr(upstream_node, "_sync_row_views", {}) + retained = {} + stored_rows = [] + + def stored_view(row: Any, row_type: type) -> Any: + cached = previous.get(id(row)) + if cached is not None and cached[0] is row: + stored = cached[1] + elif type(row) is row_type: + # _rows_to_table above has materialized this exact Arrow table. + # Copying preserves source/meta/context and its cached hashes; + # only the view's Python representation is decoded anew. + stored = row.copy() + stored._data_dict = None + stored._data_python_schema = None + setattr( + stored, + "_stored_row_origin", + getattr(row, "_stored_row_origin", row), + ) + else: + stored = row_type( + row.as_table( + columns={"source": True, "meta": True, "system_tags": True} + ), + data_context=upstream_node.data_context, + ) + retained[id(row)] = (row, stored) + return stored + + for tag, data in buf: + stored_rows.append((stored_view(tag, Tag), stored_view(data, Data))) + upstream_node._sync_row_views = retained + stream._cached_elements = stored_rows + return stream @staticmethod def _gc_buffers( diff --git a/superpowers/plans/2026-11-13-itl-1063-cache-reuse.md b/superpowers/plans/2026-11-13-itl-1063-cache-reuse.md index 78c4112ae..73cc44166 100644 --- a/superpowers/plans/2026-11-13-itl-1063-cache-reuse.md +++ b/superpowers/plans/2026-11-13-itl-1063-cache-reuse.md @@ -12,35 +12,39 @@ Files: `src/orcapod/pipeline/sync_orchestrator.py`, `async_orchestrator.py`, focused internal cache helper if needed, `src/orcapod/core/nodes/{function_node,operator_node,source_node}.py`, `tests/test_pipeline/test_execution_cache_reuse.py`. -- [ ] Add regression cases for repeated flat/grouped jobs, sync/async/cross-orchestrator execution. Count `StarfixArrowHasher.hash_table`, `GroupBy.unary_static_process`, and user function calls after the first successful run. Assert unchanged row values, provenance and summary counts. -- [ ] Run `uv run pytest tests/test_pipeline/test_execution_cache_reuse.py -q` and confirm redundant computation failures. -- [ ] Preserve `(Tag, Data)` references when wrapping execution buffers, including data context; do not transfer hashes across transformations or drop source/system columns. -- [ ] Add a completed-execution reuse path with conservative upstream validity checks. Reuse empty outputs too; do not use timestamps or source identity hashes alone as proof for mutable/dynamic sources. Preserve lifecycle hooks and side-effect stale-store checks. -- [ ] Avoid unconditional async DB loads when the same valid completed result is retained. Do not change streaming behavior for first runs or force a barrier onto dynamic paths. -- [ ] Test source rebinding, explicit clear/invalidation, partial and failed runs, observer counts, side effects, and materialize_results=False. Preserve per-row recovery for incomplete runs. -- [ ] Run new tests plus `tests/test_pipeline`, `tests/test_core/nodes`, `tests/test_side_effects` and channel tests. Fix failures in scope. +- [x] Add regression cases for repeated flat/grouped jobs, sync/async/cross-orchestrator execution. Count `StarfixArrowHasher.hash_table`, `GroupBy.unary_static_process`, and user function calls after the first successful run. Assert unchanged row values, provenance and summary counts. +- [x] Run `uv run pytest tests/test_pipeline/test_execution_cache_reuse.py -q` and confirm redundant computation failures. +- [x] Preserve `(Tag, Data)` references when wrapping execution buffers, including data context; do not transfer hashes across transformations or drop source/system columns. +- [x] Add a completed-execution reuse path with conservative upstream validity checks. Reuse empty outputs too; do not use timestamps or source identity hashes alone as proof for mutable/dynamic sources. Preserve lifecycle hooks and side-effect stale-store checks. +- [x] Avoid unconditional async DB loads when the same valid completed result is retained. Do not change streaming behavior for first runs or force a barrier onto dynamic paths. +- [x] Test source rebinding, explicit clear/invalidation, partial and failed runs, observer counts, side effects, and materialize_results=False. Preserve per-row recovery for incomplete runs. +- [x] Run new tests plus `tests/test_pipeline`, `tests/test_core/nodes`, `tests/test_side_effects` and channel tests. Fix failures in scope. ## 2. Pinned Spiral snapshot retention Files: `src/orcapod/core/sources/spiraldb_table_source.py`, `tests/test_core/sources/test_spiraldb_table_source_streaming.py`, `docs/concepts/sources.md`. -- [ ] Add fake-connector scan-count regressions: lazy construction, complete repeat reads, mixed table/row/async reads, failed/partial reads, empty pinned snapshot and concurrent readers. -- [ ] Run focused tests to demonstrate repeated scans. -- [ ] Retain successfully completed pinned-source read buffers with reusable row objects or Arrow streams. Publish only on completion; concurrent readers must not expose partial cache state. Unpinned/unknown table versions must stay conservative. No hidden repinning or hash changes. -- [ ] Keep head bounded and lazy before a full read; allow it to use retained rows afterward. Document retention memory cost and constructing a new source to select a new snapshot. -- [ ] Run Spiral source tests and the existing live-test collection without requiring live credentials. +- [x] Add fake-connector scan-count regressions: lazy construction, complete repeat reads, mixed table/row/async reads, failed/partial reads, empty pinned snapshot and concurrent readers. +- [x] Run focused tests to demonstrate repeated scans. +- [x] Retain successfully completed pinned-source read buffers with reusable row objects or Arrow streams. Publish only on completion; concurrent readers must not expose partial cache state. Unpinned/unknown table versions must stay conservative. No hidden repinning or hash changes. +- [x] Keep head bounded and lazy before a full read; allow it to use retained rows afterward. Document retention memory cost and constructing a new source to select a new snapshot. +- [x] Run Spiral source tests and the existing live-test collection without requiring live credentials. ## 3. Integration, documentation and delivery Files: `design_issues/ITL-1063-CR1.md`, `changelog.d/ITL-1063.md`, approved design and this plan. -- [ ] Review interactions between source snapshots, node invalidation, sync/async switching and existing row readback caches. -- [ ] Record implementation decisions and limitations in the approved design; add the authorized design-issue entry and changelog. -- [ ] Benchmark flat and grouped repeated jobs with function/operator/hash counters and compare to baseline. Use operation-count assertions, not flaky timing thresholds, in tests. -- [ ] Run `uv run ruff check .`, `uv run ruff format --check .`, `uv run basedpyright`, and `uv run pytest tests/`; inspect exit codes and logs. -- [ ] Request independent review; resolve correctness findings and rerun affected checks. +- [x] Review interactions between source snapshots, node invalidation, sync/async switching and existing row readback caches. +- [x] Record implementation decisions and limitations in the approved design; add the authorized design-issue entry and changelog. +- [x] Benchmark flat and grouped repeated jobs with function/operator/hash counters and compare to baseline. Use operation-count assertions, not flaky timing thresholds, in tests. +- [x] Run `uv run ruff check .`, `uv run ruff format --check .`, `uv run basedpyright`, and `uv run pytest tests/`; inspect exit codes and logs. +- [x] Request independent review; resolve correctness findings and rerun affected checks. - [ ] Commit with conventional message, push issue branch, create PR against main linking ITL-1063, and inspect CI. Include measurements and decisions; do not merge. ## Authorization and decisions Developer approved the design, issue creation and implementation through PR. Existing isolated cowfish workspace is used. No additional confirmation for routine implementation decisions; record them for the developer's return. + +Remaining performance investigation is captured in ITL-1064; it does not block this correctness and reuse fix. + +Final verification: 11,625 tests passed, 119 skipped, 19 expected failures; basedpyright zero errors; Ruff lint/format and repository documentation checks pass. diff --git a/superpowers/specs/2026-11-13-pipeline-cache-reuse-design.md b/superpowers/specs/2026-11-13-pipeline-cache-reuse-design.md index bfcc36a35..62bc34c48 100644 --- a/superpowers/specs/2026-11-13-pipeline-cache-reuse-design.md +++ b/superpowers/specs/2026-11-13-pipeline-cache-reuse-design.md @@ -1,6 +1,6 @@ # Reuse completed work across pipeline runs -Status: proposed for developer approval. Implementation and PR requested November 13, 2026. +Status: approved November 13, 2026; implementation tracked as ITL-1063. ## Problem and evidence @@ -73,7 +73,59 @@ the snapshot-memory tradeoff in the PR description. ## Tracking and delivery -No matching open Linear issue was found in the cache/rerun searches. Create a dedicated -issue and design-issue entry once approved, use the issue's branch name, write the detailed -implementation plan, add failing regressions, implement, review and open a PR against main. -No deployment, release or merge is included. +ITL-1063 tracks the approved implementation. The branch is +`eywalker/itl-1063-reuse-completed-pipeline-work-and-row`; the implementation plan is +`superpowers/plans/2026-11-13-itl-1063-cache-reuse.md`. No deployment, release or merge is +included. + +## Decisions made during implementation + +- Keep concurrent initial Spiral reads independent rather than holding a lock across + consumer yields. The first completed read publishes its snapshot atomically; incomplete + or cancelled reads never publish. Later readers reuse its row objects. This avoids + deadlocks when iterators overlap or a sync consumer runs on the async event-loop thread. +- Retain batch streams rather than eagerly constructing rows for a table-only read. + Row iteration initializes each retained batch's row memo under a short lock. Unknown + table versions are not cached. A retained head uses the snapshot without another scan. +- Scope complete-execution reuse to trusted orchestrator graph execution. Direct node + execution may receive inputs different from its declared upstreams, so declared graph + state alone cannot certify those calls. +- Use conservative known snapshot support. Custom and polling sources must not be + considered unchanged merely because their identity hash or construction timestamp is + unchanged. Preserve ordinary execution for them. +- Continue per-row function observer callbacks and summary accounting instead of skipping + the entire job. Warm runs may still perform linear iteration; the regression criterion + is no repeated user computation, operator computation, or row hashing for unchanged + supported snapshots. +- The developer explicitly authorized judgment calls while away; review findings are + resolved within this scope and decisions are recorded here and in the PR. +- Freeze graph definitions before callbacks and require a one-use orchestrator invocation + ticket. Source rebinding during callbacks and nested direct calls cannot certify stale + buffered rows as current work. +- Warm async operators drain their input channels and wait for upstream task success + before replaying retained output. Channel closure alone also occurs on failure and is + insufficient evidence. Cold and unknown-source execution retain the streaming path. +- Preserve actual object references in identity comparisons and entry-ID memoization; + do not rely on reusable integer object addresses or transfer hashes to new values. +- Preserve the sync path's historical Arrow-decoded values with a retained storage view + for each Tag/Data object. Keep the original object untouched, so this fix does not alter + async/standalone subclass behavior tracked by ITL-978. The view shares the original + exact Arrow representation and hashes, and identity memoization recognizes that origin. + Explicit invalidation clears the retained views; replaced buffers discard old entries. + +## Measured result + +The 1,000-row benchmark compared base `89160c47` with this implementation using the +same environment and in-memory result database. First warm runs (milliseconds): + +| Pipeline | Before | After | Arrow hash calls before / after | GroupBy calls before / after | +| --- | ---: | ---: | ---: | ---: | +| Flat sync | 275.82 | 19.58 | 2000 / 0 | 0 / 0 | +| Flat async | 233.72 | 5.40 | 1000 / 0 | 0 / 0 | +| Grouped sync | 29.22 | 16.86 | 20 / 0 | 1 / 0 | +| Grouped async | 31.24 | 2.38 | 20 / 0 | 1 / 0 | + +Warm user-function calls were zero both before and after. These are local illustrative +timings, not performance guarantees; operation counts are regression-tested. Sync still +materializes Arrow tables at node boundaries, and both paths retain traversal/callback +cost. No constant-time, cross-process or newly constructed job guarantee is claimed. diff --git a/tests/test_core/operators/test_async_sync_equivalence.py b/tests/test_core/operators/test_async_sync_equivalence.py index 5aabda955..0237cc7c2 100644 --- a/tests/test_core/operators/test_async_sync_equivalence.py +++ b/tests/test_core/operators/test_async_sync_equivalence.py @@ -777,8 +777,6 @@ def make_sources() -> tuple[StreamProtocol, ...]: "group_by": _MATERIALIZE_DROPS_META, "polars_filter": _MATERIALIZE_DROPS_META, "select_data_columns": _SYNC_DROPS_META, - "index": _SYNC_DROPS_META, - "pick": _SYNC_DROPS_META, } META_CASES = [ pytest.param( diff --git a/tests/test_core/sources/test_spiraldb_table_source_streaming.py b/tests/test_core/sources/test_spiraldb_table_source_streaming.py index 7f911b5b5..763880f32 100644 --- a/tests/test_core/sources/test_spiraldb_table_source_streaming.py +++ b/tests/test_core/sources/test_spiraldb_table_source_streaming.py @@ -866,7 +866,7 @@ def scan(connector, *args, **kwargs): async def cancel(): rows = source.async_iter_data() - task = asyncio.create_task(anext(rows)) + task = asyncio.ensure_future(anext(rows)) assert await asyncio.to_thread(entered.wait, 5) task.cancel() with pytest.raises(asyncio.CancelledError): @@ -889,3 +889,27 @@ def test_head_reuses_completed_snapshot(self, project): assert source.head(2).total_rows == _ROWS assert source.head(2).total_is_estimate is False assert len(fake_pyspiral.scan_log) == 2 + + def test_first_completed_scan_remains_retained_when_other_reader_finishes( + self, project + ): + """A slower initial scan cannot replace the first completed snapshot.""" + source = SpiralDBTableSource(_PROJECT, "event") + slower = source.iter_data() + first_row = next(slower) + winner = list(source.iter_data()) + loser = [first_row, *slower] + retained = list(source.iter_data()) + + assert len(winner) == len(loser) == len(retained) == _ROWS + assert [tag.as_dict() for tag, _ in loser] == [ + tag.as_dict() for tag, _ in winner + ] + assert [data.as_dict() for _, data in loser] == [ + data.as_dict() for _, data in winner + ] + assert all( + tag is retained_tag and data is retained_data + for (tag, data), (retained_tag, retained_data) in zip(winner, retained) + ) + assert len(fake_pyspiral.scan_log) == 2 diff --git a/tests/test_pipeline/test_execution_cache_reuse.py b/tests/test_pipeline/test_execution_cache_reuse.py new file mode 100644 index 000000000..d33d81ac7 --- /dev/null +++ b/tests/test_pipeline/test_execution_cache_reuse.py @@ -0,0 +1,355 @@ +"""Completed jobs retain operator work and row hashes across orchestrators.""" + +from unittest.mock import patch +from dataclasses import dataclass +from typing import cast + +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.core.operators import GroupBy +from orcapod.databases import InMemoryArrowDatabase +from orcapod.hashing.arrow_hashers import StarfixArrowHasher +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.sources import DictSource + + +@function_pod(output_keys="total") +def total(x: list[int]) -> int: + """Sum a group.""" + return sum(x) + + +@pytest.mark.parametrize("first", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator]) +@pytest.mark.parametrize( + "second", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator] +) +def test_warm_grouped_job_reuses_work(first, second) -> None: + """A warm run does no grouping or Arrow hashing and preserves every output.""" + job = PipelineJob(name="reuse", store=InMemoryArrowDatabase()) + with job: + grouped = GroupBy(by=["g"])( + DictSource([{"g": 1, "x": 2}, {"g": 1, "x": 3}], tag_columns=["g"]), + label="grouped", + ) + total.pod(grouped, label="total") + job.run(orchestrator=first()) + before = job.nodes["total"].as_table(all_info=True) + with ( + patch.object( + GroupBy, "unary_static_process", side_effect=AssertionError("regrouped") + ), + patch.object( + StarfixArrowHasher, "hash_table", side_effect=AssertionError("rehashed") + ), + ): + job.run(orchestrator=second()) + assert job.nodes["total"].as_table(all_info=True).equals(before) + + +@function_pod(output_keys="x") +def doubled(x: int) -> int: + """Double a source row before grouping.""" + return 2 * x + + +@function_pod(output_keys="x") +def omitted(x: int) -> int | None: + """Record a successful computation that produces no row.""" + return None + + +fail_row = False +CALLS: list[int] = [] + + +@function_pod(output_keys="x") +def fallible(x: int) -> int: + """Fail one row on demand to exercise partial-run recovery.""" + CALLS.append(x) + if fail_row and x == 3: + raise ValueError("retry this row") + return x + + +def make_chain(pod=doubled): + """Build a function/operator/function chain with a reusable source.""" + job = PipelineJob(name="chain_reuse", store=InMemoryArrowDatabase()) + with job: + rows = pod.pod( + DictSource( + [{"g": 1, "x": 2}, {"g": 1, "x": 3}], + tag_columns=["g"], + source_id="same", + ), + label="rows", + ) + grouped = GroupBy(by=["g"])(rows, label="grouped") + if pod is not omitted: + total.pod(grouped, label="total") + return job + + +@pytest.mark.parametrize("first", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator]) +@pytest.mark.parametrize( + "second", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator] +) +@pytest.mark.parametrize("materialize", [True, False]) +def test_warm_chain_reuses_work(first, second, materialize) -> None: + """Function cache hits preserve the completion proof used by downstream operators.""" + job = make_chain() + job.run(orchestrator=first(), materialize_results=materialize) + before = job.nodes["total"].as_table(all_info=True) + with ( + patch.object( + GroupBy, "unary_static_process", side_effect=AssertionError("regrouped") + ), + patch.object( + StarfixArrowHasher, "hash_table", side_effect=AssertionError("rehashed") + ), + ): + summary = job.run(orchestrator=second(), materialize_results=materialize) + assert (summary.nodes["rows"].computed, summary.nodes["rows"].reused) == (0, 2) + assert (summary.nodes["total"].computed, summary.nodes["total"].reused) == (0, 1) + assert job.nodes["total"].as_table(all_info=True).equals(before) + + +@pytest.mark.parametrize( + "orchestrator", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator] +) +def test_rebinding_same_source_id_invalidates_completion(orchestrator) -> None: + """Source identity does not make a replacement source's changed rows reusable.""" + from orcapod.core.nodes.source_node import SourceJobNode + + job = make_chain() + job.run(orchestrator=orchestrator()) + source = next( + node for node in job.nodes.values() if isinstance(node, SourceJobNode) + ) + source.bound_source = DictSource( + [{"g": 1, "x": 9}], tag_columns=["g"], source_id="same" + ) + job.run(orchestrator=orchestrator()) + assert job.nodes["grouped"].as_table().column("x").to_pylist() == [[18]] + + +@pytest.mark.parametrize( + "orchestrator", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator] +) +def test_explicit_operator_clear_recomputes(orchestrator) -> None: + """Explicit cache clearing invalidates completion even for unchanged inputs.""" + job = make_chain() + job.run(orchestrator=orchestrator()) + job.nodes["grouped"].clear_cache() + original = GroupBy.unary_static_process + calls = [] + + def grouped(self, stream): + calls.append(True) + return original(self, stream) + + with patch.object(GroupBy, "unary_static_process", grouped): + job.run(orchestrator=orchestrator()) + assert calls == [True] + + +@pytest.mark.parametrize( + "orchestrator", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator] +) +def test_partial_failure_retries_and_regroups(orchestrator) -> None: + """Continue-policy failures never publish a reusable completed result.""" + global fail_row + fail_row = True + CALLS.clear() + job = make_chain(fallible) + try: + with pytest.warns(Warning, match="failed"): + job.run(orchestrator=orchestrator()) + finally: + fail_row = False + CALLS.clear() + job.run(orchestrator=orchestrator()) + assert CALLS == [3] + assert sorted(job.nodes["grouped"].as_table().column("x").to_pylist()[0]) == [2, 3] + + +@pytest.mark.parametrize("first", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator]) +@pytest.mark.parametrize( + "second", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator] +) +def test_empty_results_are_completed(first, second) -> None: + """Successful None outputs and the empty downstream result are reusable.""" + job = make_chain() + job.nodes["rows"]._function_pod.data_function.set_active(False) + job.run(orchestrator=first()) + with patch.object( + StarfixArrowHasher, "hash_table", side_effect=AssertionError("rehashed") + ): + job.run(orchestrator=second()) + assert job.nodes["grouped"].as_table().num_rows == 0 + job.nodes["rows"]._function_pod.data_function.set_active(True) + + +@pytest.mark.parametrize( + "orchestrator", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator] +) +def test_source_callback_rebinding_does_not_certify_old_rows(orchestrator) -> None: + """A source callback cannot relabel an old buffer as a new source snapshot.""" + from orcapod.core.nodes.source_node import SourceJobNode + from orcapod.pipeline.observer import NoOpObserver + + job = make_chain() + source = next( + node for node in job.nodes.values() if isinstance(node, SourceJobNode) + ) + + class Rebind(NoOpObserver): + def on_node_end(self, node_label, node_hash): + if node_label == source.label: + source.bound_source = DictSource( + [{"g": 1, "x": 9}], tag_columns=["g"], source_id="same" + ) + + job.run(orchestrator=orchestrator()) + job.run(orchestrator=orchestrator(), observer=Rebind()) + job.run(orchestrator=orchestrator()) + assert job.nodes["grouped"].as_table().column("x").to_pylist() == [[18]] + + +@pytest.mark.parametrize("asynchronous", [False, True]) +def test_direct_operator_call_consumes_actual_inputs(asynchronous) -> None: + """Declared graph inputs never authorize reuse for a direct node invocation.""" + import asyncio + + from orcapod.channels import Channel + + job = make_chain() + job.run(orchestrator=SyncPipelineOrchestrator()) + node = job.nodes["grouped"] + replacement = DictSource([{"g": 1, "x": 99}], tag_columns=["g"]) + if asynchronous: + + async def run(): + inputs = Channel(buffer_size=1) + output = Channel(buffer_size=1) + + async def feed(): + for row in replacement.iter_data(): + await inputs.writer.send(row) + await inputs.writer.close() + + async with asyncio.TaskGroup() as group: + group.create_task(feed()) + group.create_task(node.async_execute([inputs.reader], output.writer)) + collected = group.create_task(output.reader.collect()) + return collected.result() + + rows = asyncio.run(run()) + else: + rows = node.execute(replacement) + assert rows[0][1].as_dict()["x"] == [99] + + +@pytest.mark.parametrize( + "orchestrator", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator] +) +def test_unknown_source_never_reuses_operator_completion(orchestrator) -> None: + """A custom source is not assumed immutable from its class ancestry or name.""" + from orcapod.core.nodes.source_node import SourceJobNode + + class ChangingSource(DictSource): + value = 7 + + def iter_data(self): + yield from DictSource( + [{"g": 1, "x": self.value}], tag_columns=["g"], source_id="same" + ).iter_data() + + async def async_iter_data(self): + for row in self.iter_data(): + yield row + + job = make_chain() + source = next( + node for node in job.nodes.values() if isinstance(node, SourceJobNode) + ) + changing = ChangingSource([{"g": 1, "x": 7}], tag_columns=["g"], source_id="same") + source.bound_source = changing + job.run(orchestrator=orchestrator()) + changing.value = 8 + job.run(orchestrator=orchestrator()) + assert job.nodes["grouped"].as_table().column("x").to_pylist() == [[16]] + assert job.nodes["rows"]._entry_id_rows == {} + + +@pytest.mark.parametrize("failure_at", ["source", "rows"]) +def test_warm_async_upstream_failure_emits_no_cached_rows(failure_at) -> None: + """Failed producers close channels, but that never authorizes downstream replay.""" + from orcapod.core.nodes.source_node import SourceJobNode + from orcapod.pipeline.observer import NoOpObserver + + job = make_chain() + job.run(orchestrator=AsyncPipelineOrchestrator()) + source = next( + node for node in job.nodes.values() if isinstance(node, SourceJobNode) + ) + failed_label = source.label if failure_at == "source" else "rows" + downstream = [] + + class FailingObserver(NoOpObserver): + def on_node_start(self, node_label, node_hash, tag_schema=None): + if node_label == failed_label: + raise ValueError("upstream never produced") + + def on_data_start(self, node_label, tag, data): + if node_label == "total": + downstream.append(data) + + with pytest.raises(ExceptionGroup): + job.run(orchestrator=AsyncPipelineOrchestrator(), observer=FailingObserver()) + assert downstream == [] + + +@dataclass +class Base: + value: int + + +@dataclass +class Child(Base): + extra: int + + +def test_sync_storage_views_are_retained_without_mutating_originals() -> None: + """The sync boundary decodes the declared type once while preserving live originals.""" + from types import SimpleNamespace + + from orcapod.core.datagrams import Data, Tag + + from orcapod.types import DataValue + + # DataValue's static union does not enumerate registered user logical types. + original = Data( + {"x": cast(DataValue, cast(object, Child(1, 99)))}, python_schema={"x": Base} + ) + tag = Tag( + {"k": cast(DataValue, cast(object, Child(2, 88)))}, python_schema={"k": Base} + ) + upstream = SimpleNamespace( + producer=None, upstreams=(), data_context=original.data_context + ) + first = SyncPipelineOrchestrator._materialize_as_stream([(tag, original)], upstream) + second = SyncPipelineOrchestrator._materialize_as_stream( + [(tag, original)], upstream + ) + first_tag, first_data = next(first.iter_data()) + second_tag, second_data = next(second.iter_data()) + assert first_tag is second_tag + assert type(first_tag.as_dict()["k"]) is Base + assert type(tag.as_dict()["k"]) is Child + assert first_tag.content_hash() == tag.content_hash() + assert first_data is second_data + assert type(first_data.as_dict()["x"]) is Base + assert type(original.as_dict()["x"]) is Child + assert cast(Child, cast(object, original.as_dict()["x"])).extra == 99 + assert first_data.content_hash() == original.content_hash() diff --git a/tests/test_pipeline/test_spiral_execution_cache_reuse.py b/tests/test_pipeline/test_spiral_execution_cache_reuse.py new file mode 100644 index 000000000..0d1b3eeaf --- /dev/null +++ b/tests/test_pipeline/test_spiral_execution_cache_reuse.py @@ -0,0 +1,67 @@ +"""Pinned source snapshots and execution caches cooperate across orchestrators.""" + +from pathlib import Path +from unittest.mock import patch + +import pyarrow as pa +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.core.operators import GroupBy +from orcapod.databases import InMemoryArrowDatabase +from orcapod.hashing.arrow_hashers import StarfixArrowHasher +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.sources import SpiralDBTableSource +from tests.test_databases import fake_pyspiral + + +@function_pod(output_keys="total") +def total(x: list[int | None]) -> int: + """Reduce one group from the nullable fake table.""" + return sum(value for value in x if value is not None) + + +@pytest.mark.parametrize("first", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator]) +@pytest.mark.parametrize( + "second", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator] +) +def test_pinned_grouped_job_reuses_scan_rows_and_hashes( + tmp_path: Path, first, second +) -> None: + """A warm run neither scans nor groups nor hashes, even with channel backpressure.""" + with fake_pyspiral.install(tmp_path / "spiral"): + project = fake_pyspiral.FakeSpiral().project("reuse-integration") + data = pa.table( + { + "g": [i // 20 for i in range(100)], + "k": list(range(100)), + "x": list(range(100)), + } + ) + project.create_table( + "default.events", + key_schema=pa.schema([data.schema.field("g"), data.schema.field("k")]), + ).write(data) + fake_pyspiral.scan_log.clear() + source = SpiralDBTableSource("reuse-integration", "events", batch_rows=13) + with PipelineJob(name="spiral_reuse", store=InMemoryArrowDatabase()) as job: + grouped = source.group_by(["g"]).drop_tag_columns(["k"]) + total.pod(grouped, label="total") + assert fake_pyspiral.scan_log == [] + assert job.run(orchestrator=first()).ok + before = job.nodes["total"].as_table(all_info=True) + assert len(fake_pyspiral.scan_log) == 1 + with ( + patch.object( + GroupBy, "unary_static_process", side_effect=AssertionError("regrouped") + ), + patch.object( + StarfixArrowHasher, "hash_table", side_effect=AssertionError("rehashed") + ), + ): + summary = job.run(orchestrator=second()) + assert summary.ok + assert summary.nodes["total"].computed == 0 + assert summary.nodes["total"].reused == 5 + assert len(fake_pyspiral.scan_log) == 1 + assert job.nodes["total"].as_table(all_info=True).equals(before) diff --git a/tests/test_pipeline/test_system_tag_run_stability.py b/tests/test_pipeline/test_system_tag_run_stability.py index e4b67a735..ae10e6ae1 100644 --- a/tests/test_pipeline/test_system_tag_run_stability.py +++ b/tests/test_pipeline/test_system_tag_run_stability.py @@ -146,7 +146,9 @@ def test_operator_materialize_to_stream(self) -> None: def test_orchestrator_materialize_as_stream(self) -> None: """The orchestrator materializer emits rows in the first row's column order.""" - upstream = SimpleNamespace(producer=None, upstreams=()) + upstream = SimpleNamespace( + producer=None, upstreams=(), data_context=Data({"v": 10}).data_context + ) stream = SyncPipelineOrchestrator._materialize_as_stream( _rows_with_swapped_tag_order(), upstream ) diff --git a/tests/test_pipeline/test_tag_table_source_info.py b/tests/test_pipeline/test_tag_table_source_info.py index 0bd3844a7..58af5c7c4 100644 --- a/tests/test_pipeline/test_tag_table_source_info.py +++ b/tests/test_pipeline/test_tag_table_source_info.py @@ -174,6 +174,18 @@ def broadcasting( monkeypatch.setattr( StaticOutputOperatorPod, "_rows_to_table", staticmethod(broadcasting) ) + # The old materializer rebuilt row objects from this corrupted table. Modern + # materializers preserve the original rows, so recreate that historical read + # path too; otherwise this fixture would no longer write an old-format store. + from orcapod.core.streams.arrow_table_stream import ArrowTableStream + + original_iter = ArrowTableStream.iter_data + + def historical_rows(self: ArrowTableStream): + self._cached_elements = None + yield from original_iter(self) + + monkeypatch.setattr(ArrowTableStream, "iter_data", historical_rows) @pytest.mark.parametrize("orchestrator", ["sync", "async"]) From 5382d5a37cf02dc4e54b481481cf5d027cab9fd2 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 02:41:04 -0800 Subject: [PATCH 4/7] docs(cache): record verification and PR handoff --- superpowers/plans/2026-11-13-itl-1063-cache-reuse.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/superpowers/plans/2026-11-13-itl-1063-cache-reuse.md b/superpowers/plans/2026-11-13-itl-1063-cache-reuse.md index 73cc44166..4e2ad2211 100644 --- a/superpowers/plans/2026-11-13-itl-1063-cache-reuse.md +++ b/superpowers/plans/2026-11-13-itl-1063-cache-reuse.md @@ -39,7 +39,7 @@ Files: `design_issues/ITL-1063-CR1.md`, `changelog.d/ITL-1063.md`, approved desi - [x] Benchmark flat and grouped repeated jobs with function/operator/hash counters and compare to baseline. Use operation-count assertions, not flaky timing thresholds, in tests. - [x] Run `uv run ruff check .`, `uv run ruff format --check .`, `uv run basedpyright`, and `uv run pytest tests/`; inspect exit codes and logs. - [x] Request independent review; resolve correctness findings and rerun affected checks. -- [ ] Commit with conventional message, push issue branch, create PR against main linking ITL-1063, and inspect CI. Include measurements and decisions; do not merge. +- [x] Commit with conventional message, push issue branch, create PR against main linking ITL-1063, and inspect CI. Include measurements and decisions; do not merge. ## Authorization and decisions @@ -48,3 +48,5 @@ Developer approved the design, issue creation and implementation through PR. Exi Remaining performance investigation is captured in ITL-1064; it does not block this correctness and reuse fix. Final verification: 11,625 tests passed, 119 skipped, 19 expected failures; basedpyright zero errors; Ruff lint/format and repository documentation checks pass. + +Delivery: PR https://github.com/metamorphic-labs/orcapod-python/pull/359 opened against main. Initial CI inspected: tests pending; no merge performed. From 23e1273f8d3c09192d1758dac338be5d29005fe9 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 08:30:09 -0800 Subject: [PATCH 5/7] fix(cache): bound DAG validation and reject reentrant cache replacement --- src/orcapod/core/nodes/_execution_cache.py | 115 +++++++++++++--- .../2026-11-13-pipeline-cache-reuse-design.md | 7 + .../test_execution_cache_reuse.py | 25 ++++ .../test_execution_cache_review.py | 128 ++++++++++++++++++ 4 files changed, 253 insertions(+), 22 deletions(-) create mode 100644 tests/test_pipeline/test_execution_cache_review.py diff --git a/src/orcapod/core/nodes/_execution_cache.py b/src/orcapod/core/nodes/_execution_cache.py index 29e4aec10..15b1cfcb4 100644 --- a/src/orcapod/core/nodes/_execution_cache.py +++ b/src/orcapod/core/nodes/_execution_cache.py @@ -26,6 +26,32 @@ def __eq__(self, other: object) -> bool: return isinstance(other, _Identity) and self.value is other.value +class _Definition: + """A shared snapshot DAG with comparison bounded by distinct ancestor pairs.""" + + def __init__( + self, local: tuple[Any, ...], upstreams: tuple[_Definition, ...] = () + ) -> None: + self.local = local + self.upstreams = upstreams + + def __eq__(self, other: object) -> bool: + if not isinstance(other, _Definition): + return False + pending: list[tuple[_Definition, _Definition]] = [(self, other)] + seen: set[tuple[int, int]] = set() + while pending: + left, right = pending.pop() + pair = (id(left), id(right)) + if left is right or pair in seen: + continue + seen.add(pair) + if left.local != right.local or len(left.upstreams) != len(right.upstreams): + return False + pending.extend(zip(left.upstreams, right.upstreams, strict=True)) + return True + + class _Run: """Freeze source bindings, configuration and graph edges before any callbacks.""" @@ -47,53 +73,93 @@ def __init__(self, graph: Any, kinds: dict[str, tuple[type, ...]]) -> None: self.aligned[id(node)] = tuple(id(p) for p in predecessors) == tuple( id(p) for p in upstreams ) - self.initial = {key: self.definition(node) for key, node in self.nodes.items()} - - def definition(self, stream: Any) -> object | None: + definitions: dict[int, _Definition | None] = {} + self.initial = { + key: self.definition(node, definitions) for key, node in self.nodes.items() + } + + def definition( + self, stream: Any, memo: dict[int, _Definition | None] | None = None + ) -> _Definition | None: """Describe the graph independently of whether its work has completed.""" + if memo is None: + memo = {} + key = id(stream) + if key not in memo: + memo[key] = self._definition(stream, memo) + return memo[key] + + def _definition( + self, stream: Any, memo: dict[int, _Definition | None] + ) -> _Definition | None: + """Build one node once per traversal, without retaining a stale traversal.""" kind = type(stream) if kind in self.kinds["source_node"]: - bound = self.definition(stream.bound_source) - return None if bound is None else (_Identity(stream), bound) + bound = self.definition(stream.bound_source, memo) + return ( + None if bound is None else _Definition((_Identity(stream),), (bound,)) + ) if kind in self.kinds["arrow"]: - return (_Identity(stream), stream.last_modified, stream.data_context_key) + return _Definition( + (_Identity(stream), stream.last_modified, stream.data_context_key) + ) if kind in self.kinds["memory_source"]: - underlying = self.definition(stream._stream) - return None if underlying is None else (_Identity(stream), underlying) + underlying = self.definition(stream._stream, memo) + return ( + None + if underlying is None + else _Definition((_Identity(stream),), (underlying,)) + ) if kind in self.kinds["spiral"] and stream._asof is not None: - return (_Identity(stream), stream._asof, stream.data_context_key) + return _Definition( + (_Identity(stream), stream._asof, stream.data_context_key) + ) if kind not in self.kinds["computed"]: return None - upstreams = tuple(self.definition(s) for s in stream.upstreams) + upstreams = tuple(self.definition(s, memo) for s in stream.upstreams) if not upstreams or any(s is None for s in upstreams): return None producer = stream.producer - return ( - upstreams, - getattr(stream, "_execution_generation", 0), - _Identity(producer), - producer.last_modified if producer is not None else None, - _Identity(getattr(stream, "_node_config", None)), - _Identity(getattr(stream, "_pipeline_database", None)), - _Identity(getattr(stream, "_ephemeral_cached_pod", None)), - stream.data_context_key, + return _Definition( + ( + getattr(stream, "_execution_generation", 0), + _Identity(producer), + producer.last_modified if producer is not None else None, + _Identity(getattr(stream, "_node_config", None)), + _Identity(getattr(stream, "_pipeline_database", None)), + _Identity(getattr(stream, "_ephemeral_cached_pod", None)), + stream.data_context_key, + ), + cast(tuple[_Definition, ...], upstreams), ) - def proof(self, node: Any) -> object | None: + def proof(self, node: Any) -> _Definition | None: """Require unchanged run-start inputs and successfully completed predecessors.""" + return self._proof(node, {}, {}) + + def _proof( + self, + node: Any, + definitions: dict[int, _Definition | None], + proofs: dict[int, _Definition | None], + ) -> _Definition | None: key = id(node) + if key in proofs: + return proofs[key] + proofs[key] = None if key not in self.nodes or not self.aligned[key]: return None - definition = self.definition(node) + definition = self.definition(node, definitions) if definition is None or definition != self.initial[key]: return None for upstream in node.upstreams: if type(upstream) in self.kinds["computed"]: - proof = self.proof(upstream) + proof = self._proof(upstream, definitions, proofs) if proof is None or proof != getattr( upstream, "_completed_execution", None ): return None + proofs[key] = definition return definition @@ -247,6 +313,11 @@ def track_completion(method: F) -> F: """Track completion without changing per-row execution, persistence or hooks.""" def setup(node: Any, kwargs: dict[str, Any]) -> _TrackingObserver: + if _ACTIVE.get() is not node: + # An untrusted call may replace this node's output while an outer + # trusted call is suspended in a callback or another coroutine. + # The outer run-start definition must no longer certify that cache. + invalidate(node) if not is_current(node): node._entry_id_rows = {} observer = _TrackingObserver(kwargs.get("observer"), []) diff --git a/superpowers/specs/2026-11-13-pipeline-cache-reuse-design.md b/superpowers/specs/2026-11-13-pipeline-cache-reuse-design.md index 62bc34c48..8d393a99d 100644 --- a/superpowers/specs/2026-11-13-pipeline-cache-reuse-design.md +++ b/superpowers/specs/2026-11-13-pipeline-cache-reuse-design.md @@ -115,6 +115,13 @@ included. ## Measured result +CodeRabbit review follow-up: snapshot construction now memoizes each node within one +traversal, and definition comparison visits each shared ancestor pair once. Proof +validation also memoizes upstream checks within that call. Fresh traversals still detect +callback changes. A nested untrusted node execution advances the invalidation generation +so an outer trusted invocation cannot certify a replacement cache. Regression coverage +includes twenty split/join stages and both sync and async nested execution. + The 1,000-row benchmark compared base `89160c47` with this implementation using the same environment and in-memory result database. First warm runs (milliseconds): diff --git a/tests/test_pipeline/test_execution_cache_reuse.py b/tests/test_pipeline/test_execution_cache_reuse.py index d33d81ac7..73781a7e6 100644 --- a/tests/test_pipeline/test_execution_cache_reuse.py +++ b/tests/test_pipeline/test_execution_cache_reuse.py @@ -216,6 +216,31 @@ def on_node_end(self, node_label, node_hash): assert job.nodes["grouped"].as_table().column("x").to_pylist() == [[18]] +@pytest.mark.parametrize( + "orchestrator", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator] +) +def test_callback_replacing_operator_cache_invalidates_outer_completion( + orchestrator, +) -> None: + """A direct call from on_node_end cannot poison the next unchanged job run.""" + from orcapod.pipeline.observer import NoOpObserver + + job = make_chain() + node = job.nodes["grouped"] + job.run(orchestrator=orchestrator()) + expected = node.as_table() + + class ReplaceCache(NoOpObserver): + def on_node_end(self, node_label, node_hash): + if node_label == "grouped": + node.execute(DictSource([{"g": 1, "x": 99}], tag_columns=["g"])) + + job.run(orchestrator=orchestrator(), observer=ReplaceCache()) + assert node._completed_execution is None + job.run(orchestrator=orchestrator()) + assert node.as_table().equals(expected) + + @pytest.mark.parametrize("asynchronous", [False, True]) def test_direct_operator_call_consumes_actual_inputs(asynchronous) -> None: """Declared graph inputs never authorize reuse for a direct node invocation.""" diff --git a/tests/test_pipeline/test_execution_cache_review.py b/tests/test_pipeline/test_execution_cache_review.py new file mode 100644 index 000000000..13d817f14 --- /dev/null +++ b/tests/test_pipeline/test_execution_cache_review.py @@ -0,0 +1,128 @@ +"""Completion bookkeeping stays bounded and rejects replaced cache ownership.""" + +import asyncio +from types import SimpleNamespace +from typing import Any +from unittest.mock import patch + +import pytest + +from orcapod.core.nodes import _execution_cache as cache + + +class Snapshot: + """A stable snapshot for exercising graph bookkeeping without row costs.""" + + upstreams: tuple[()] = () + last_modified = 0 + data_context_key = "test" + + +class Node: + """A completion-aware node with observable definition reads.""" + + data_context_key = "test" + + def __init__(self, *upstreams: Any) -> None: + self.upstreams = upstreams + self.reads = 0 + self._producer = SimpleNamespace(last_modified=0) + self.output = "" + self._completed_execution: object | None = None + + @property + def producer(self) -> Any: + self.reads += 1 + assert self.reads <= 100, "shared upstream was expanded repeatedly" + return self._producer + + @cache.track_completion + def execute( + self, value: str, *, nested: bool = False, observer: Any = None + ) -> None: + self.output = value + if nested: + self.execute("replacement") + + @cache.track_completion + async def async_execute( + self, value: str, *, nested: bool = False, observer: Any = None + ) -> None: + self.output = value + if nested: + await self.async_execute("replacement") + + +class Graph: + """Minimal graph surface used by the orchestrator's execution bookkeeping.""" + + def __init__(self, nodes: list[Any]) -> None: + self.nodes = nodes + + def topological_sort(self) -> list[Any]: + return self.nodes + + def predecessors(self, node: Any) -> tuple[Any, ...]: + return node.upstreams + + +KINDS = { + "source_node": (), + "arrow": (Snapshot,), + "memory_source": (), + "spiral": (), + "computed": (Node,), +} + + +def test_shared_dag_snapshot_and_proof_visit_each_definition_once() -> None: + """Twenty diamonds must not expand into a million upstream paths.""" + nodes: list[Any] = [Snapshot()] + for _ in range(20): + left, right = Node(nodes[-1]), Node(nodes[-1]) + nodes.extend([left, right, Node(left, right)]) + run = cache._Run(Graph(nodes), KINDS) + comparisons = 0 + original_equal = cache._Identity.__eq__ + + def counted_equal(left: cache._Identity, right: object) -> bool: + nonlocal comparisons + comparisons += 1 + assert comparisons <= 8 * len(nodes), "shared definitions compared repeatedly" + return original_equal(left, right) + + with patch.object(cache._Identity, "__eq__", counted_equal): + assert run.definition(nodes[-1]) == run.initial[id(nodes[-1])] + for node in nodes[1:]: + assert node.reads == 2 + node._completed_execution = run.initial[id(node)] + node.reads = 0 + assert run.proof(nodes[-1]) == run.initial[id(nodes[-1])] + assert all(node.reads == 1 for node in nodes[1:]) + # A fresh validity traversal must see changes below a shared branch. + nodes[1]._execution_generation = 1 + assert run.proof(nodes[-1]) is None + + +@pytest.mark.parametrize("asynchronous", [False, True]) +def test_nested_direct_execution_cannot_be_certified_by_outer_call( + asynchronous: bool, +) -> None: + """A nested call replaces the output but cannot inherit the outer graph proof.""" + source = Snapshot() + node = Node(source) + run = cache._Run(Graph([source, node]), KINDS) + token = cache._RUN.set(run) + try: + if asynchronous: + asyncio.run( + cache.async_execute_node( + node, node.async_execute, "original", nested=True + ) + ) + else: + cache.execute_node(node, node.execute, "original", nested=True) + finally: + cache._RUN.reset(token) + assert node.output == "replacement" + assert node._completed_execution is None From 0cbc892b74a179354a71419166162c47de8b8a33 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 09:26:51 -0800 Subject: [PATCH 6/7] fix(cache): preserve freshness and expose retention lifetime controls --- changelog.d/ITL-1063.md | 6 + docs/concepts/sources.md | 12 ++ docs/concepts/streams.md | 39 ++++ src/orcapod/core/datagrams/datagram.py | 30 +++ src/orcapod/core/nodes/_execution_cache.py | 38 +++- src/orcapod/core/nodes/_retained_rows.py | 42 ++++ src/orcapod/core/nodes/function_node.py | 22 ++- src/orcapod/core/nodes/operator_node.py | 7 +- src/orcapod/core/nodes/source_node.py | 28 ++- .../core/operators/static_output_pod.py | 38 +--- src/orcapod/core/sources/base.py | 6 + .../core/sources/spiraldb_table_source.py | 40 +++- .../core/streams/arrow_table_stream.py | 66 ++++++- src/orcapod/pipeline/job.py | 20 ++ src/orcapod/pipeline/sync_orchestrator.py | 76 ++------ .../plans/2026-11-13-itl-1063-cache-reuse.md | 12 ++ .../2026-11-13-pipeline-cache-reuse-design.md | 36 ++++ .../operators/test_async_sync_equivalence.py | 2 + .../operators/test_materialize_source_info.py | 10 + .../test_spiraldb_table_source_streaming.py | 66 +++++++ .../test_execution_cache_freshness.py | 180 ++++++++++++++++++ .../test_execution_cache_reuse.py | 9 +- .../test_execution_cache_review.py | 1 + .../test_execution_cache_review_failures.py | 160 ++++++++++++++++ .../test_retained_storage_views.py | 123 ++++++++++++ .../test_spiral_execution_cache_reuse.py | 36 ++++ .../test_tag_table_source_info.py | 16 +- 27 files changed, 990 insertions(+), 131 deletions(-) create mode 100644 src/orcapod/core/nodes/_retained_rows.py create mode 100644 tests/test_pipeline/test_execution_cache_freshness.py create mode 100644 tests/test_pipeline/test_execution_cache_review_failures.py create mode 100644 tests/test_pipeline/test_retained_storage_views.py diff --git a/changelog.d/ITL-1063.md b/changelog.d/ITL-1063.md index 8280d32de..659882f29 100644 --- a/changelog.d/ITL-1063.md +++ b/changelog.d/ITL-1063.md @@ -10,3 +10,9 @@ reads do not publish a snapshot. Retention uses memory proportional to the snapshot; `batch_rows` bounds processing slices, not total retained memory. Construct a new source to read a newer table version. +- Unchanged job reruns preserve node modification times so external lazy readers + remain current. Source nodes follow their bound source's freshness. Node-owned + stored row views preserve declared types without changing live Python objects. +- `PipelineJob.clear_cache()` releases retained working sets without deleting stored + results. Spiral sources also expose `clear_cache()` and a `retain_snapshot=False` + streaming opt-out; clearing during a scan prevents that scan republishing a cache. diff --git a/docs/concepts/sources.md b/docs/concepts/sources.md index 82980a844..e1027a5bd 100644 --- a/docs/concepts/sources.md +++ b/docs/concepts/sources.md @@ -132,6 +132,18 @@ publish a snapshot. Concurrent initial readers may scan independently; the first complete read atomically publishes the retained snapshot. When no table version can be pinned, each read scans again rather than reusing potentially stale rows. +For streaming-only reads, pass `retain_snapshot=False`. Each iteration then scans again +and releases consumed batch streams, retaining only the current SpiralDB batch and its +processing slice. SpiralDB controls the underlying batch size; `batch_rows` bounds the +slice size. Whole-table methods such as `as_table()` still materialize their returned +rows, and pipeline jobs can retain their own outputs. The option is saved in source +configuration and defaults to `True`. + +Call `source.clear_cache()` to release a retained snapshot. This does not change the +pinned `asof` version. Existing readers can finish with the data they already hold, but +a scan started before clearing cannot republish its snapshot. A later full read can +retain the same version again. Construct a new source to select a newer table version. + To read part of a table, say so when you build the source. SpiralDB applies both restrictions before any row is transferred: diff --git a/docs/concepts/streams.md b/docs/concepts/streams.md index d209ccb64..275df36ab 100644 --- a/docs/concepts/streams.md +++ b/docs/concepts/streams.md @@ -299,6 +299,45 @@ print(table.to_pandas()) # 2 mouse_03 15 27.8 ``` +## Reusing completed pipeline work + +An unchanged `PipelineJob` reuses completed work when its nodes are not stale and +their inputs are supported retained snapshots. The completion record proves that a +whole execution succeeded with those graph inputs; it does not replace `is_stale`. +A reused result keeps its modification time, so lazy streams over that node remain +current. Rebinding a source, clearing a cache, changing the producer, or observing a +newer upstream invalidates reuse. Unknown freshness stays conservative. + +The supported concrete source classes are `ArrowTableStream`, `ArrowTableSource`, +`DictSource`, `ListSource`, and `SpiralDBTableSource` with a pinned version and +`retain_snapshot=True`. Subclasses and other sources, including `CSVSource`, +`DataFrameSource`, `DeltaTableSource`, `CachedSource`, `DerivedSource` and polling +sources, do not currently get completed-work reuse. Their ordinary per-row result +cache still works. One info-level log per run lists source bindings that disable the +fast path. A Delta table's version is not yet an execution snapshot capability; it +must be integrated explicitly before it can safely qualify. + +Reuse has a memory cost beyond the source snapshot. Nodes retain entry IDs and row +views for their latest buffers, in addition to output caches. A boundary may retain +both live Python values and an Arrow-decoded view of the declared stored type; Arrow +buffers are shared. The row-view mapping can hold two entries per input pair, and a +function node can retain an entry ID per pair. Memory therefore grows with processed +rows and graph width; `materialize_results=False` only releases orchestrator result +buffers, not these reusable node caches. + +Call `job.clear_cache()` between executions to release node caches and clear bound +sources that support cache release. Persistent database results, original source data +and pinned versions are preserved. `job.clear_cache(include_sources=False)` keeps the +source snapshots. Clearing a shared source also releases that source's retained rows +for its other consumers. A later run can recover persisted function results but must +rebuild cleared rows and operator outputs. + +For bounded source iteration, a Spiral source can use `retain_snapshot=False`; this +also disables completed-work reuse downstream. See [Sources](sources.md) for the +snapshot lifetime and streaming limits. Warm job execution still traverses rows and +callbacks, and sync execution still materializes tables; reuse is not a constant-time +whole-job shortcut or a cache of new jobs across processes. + ## How it connects to other concepts - [Sources](sources.md) produce streams from external data diff --git a/src/orcapod/core/datagrams/datagram.py b/src/orcapod/core/datagrams/datagram.py index bcc3ee9fe..e06dac069 100644 --- a/src/orcapod/core/datagrams/datagram.py +++ b/src/orcapod/core/datagrams/datagram.py @@ -64,6 +64,8 @@ class Datagram(ContentIdentifiableBase): # Construction # ------------------------------------------------------------------ + _stored_row_origin: Datagram | None = None + def __init__( self, data: Mapping[str, DataValue] | pa.Table | pa.RecordBatch, @@ -1026,6 +1028,34 @@ def with_columns( # 8. Utility Operations # ------------------------------------------------------------------ + @property + def storage_origin(self) -> Datagram: + """Original row whose exact stored representation this view shares.""" + return self if self._stored_row_origin is None else self._stored_row_origin + + def storage_view(self, *, include_meta: bool = True) -> Self: + """Expose Arrow-decoded values without modifying the live Python values. + + A declared base type can store fewer fields than its live subclass. This + view uses that stored type, shares immutable Arrow buffers and hashes, + and preserves provenance. Callers own retention of the returned view. + Metadata may be omitted to match a materialization boundary's columns. + """ + self._ensure_data_table() + if (self._data_dict is None or self.storage_origin is not self) and ( + include_meta or not self._meta + ): + return self + view = self.copy(preserve_id=True) + view._data_dict = None + view._data_python_schema = None + view._stored_row_origin = self.storage_origin + if not include_meta: + view._meta = {} + view._meta_python_schema = Schema() + view._meta_table = None + return view + def copy(self, include_cache: bool = True, preserve_id: bool = True) -> Self: """Return a shallow copy of this datagram. diff --git a/src/orcapod/core/nodes/_execution_cache.py b/src/orcapod/core/nodes/_execution_cache.py index 15b1cfcb4..40b1ca71c 100644 --- a/src/orcapod/core/nodes/_execution_cache.py +++ b/src/orcapod/core/nodes/_execution_cache.py @@ -9,11 +9,13 @@ import asyncio import functools import inspect +import logging from collections.abc import Callable from contextvars import ContextVar from typing import Any, TypeVar, cast F = TypeVar("F", bound=Callable[..., Any]) +logger = logging.getLogger(__name__) class _Identity: @@ -77,6 +79,17 @@ def __init__(self, graph: Any, kinds: dict[str, tuple[type, ...]]) -> None: self.initial = { key: self.definition(node, definitions) for key, node in self.nodes.items() } + disabled = [ + f"{getattr(node, 'label', type(node).__name__)} ({type(node.bound_source).__name__})" + for key, node in self.nodes.items() + if type(node) in self.kinds["source_node"] and self.initial[key] is None + ] + if disabled: + logger.info( + "Completed-work reuse disabled for sources without a supported retained " + "snapshot: %s", + ", ".join(disabled), + ) def definition( self, stream: Any, memo: dict[int, _Definition | None] | None = None @@ -97,7 +110,12 @@ def _definition( if kind in self.kinds["source_node"]: bound = self.definition(stream.bound_source, memo) return ( - None if bound is None else _Definition((_Identity(stream),), (bound,)) + None + if bound is None + else _Definition( + (_Identity(stream), getattr(stream, "_execution_generation", 0)), + (bound,), + ) ) if kind in self.kinds["arrow"]: return _Definition( @@ -110,9 +128,18 @@ def _definition( if underlying is None else _Definition((_Identity(stream),), (underlying,)) ) - if kind in self.kinds["spiral"] and stream._asof is not None: + if ( + kind in self.kinds["spiral"] + and stream._asof is not None + and stream._retain_snapshot + ): return _Definition( - (_Identity(stream), stream._asof, stream.data_context_key) + ( + _Identity(stream), + stream._asof, + stream._snapshot_generation, + stream.data_context_key, + ) ) if kind not in self.kinds["computed"]: return None @@ -250,6 +277,7 @@ def is_current(node: Any) -> bool: and run is not None and previous is not None and previous == run.proof(node) + and not node.is_stale ) @@ -266,7 +294,9 @@ def invalidate(node: Any) -> None: """Forget completion and row memoization after an explicit cache clear.""" node._completed_execution = None node._execution_generation = getattr(node, "_execution_generation", 0) + 1 - node._sync_row_views = {} + clear_rows = getattr(node, "clear_retained_rows", None) + if clear_rows is not None: + clear_rows() node._entry_id_rows = {} diff --git a/src/orcapod/core/nodes/_retained_rows.py b/src/orcapod/core/nodes/_retained_rows.py new file mode 100644 index 000000000..19e1b55e5 --- /dev/null +++ b/src/orcapod/core/nodes/_retained_rows.py @@ -0,0 +1,42 @@ +"""Node-owned storage views, retained only for the most recent input buffer.""" + +from __future__ import annotations + +from collections.abc import Callable, Sequence +from typing import Any, TypeVar + +TagT = TypeVar("TagT") +DataT = TypeVar("DataT") + + +class RetainedRowsMixin: + """Own reusable row views and expose explicit release to job orchestration.""" + + _sync_row_views: dict[int, tuple[Any, Any]] | None = None + + def clear_retained_rows(self) -> None: + """Release row views without removing any persisted results.""" + self._sync_row_views = {} + + def retain_storage_rows( + self, + rows: Sequence[tuple[TagT, DataT]], + *, + tag_view: Callable[[TagT], TagT], + data_view: Callable[[DataT], DataT], + ) -> list[tuple[TagT, DataT]]: + """Return canonical stored rows, reusing views for identical live rows.""" + previous = self._sync_row_views or {} + retained: dict[int, tuple[Any, Any]] = {} + + def view(row: Any, convert: Any) -> Any: + cached = previous.get(id(row)) + value = ( + cached[1] if cached is not None and cached[0] is row else convert(row) + ) + retained[id(row)] = (row, value) + return value + + result = [(view(tag, tag_view), view(data, data_view)) for tag, data in rows] + self._sync_row_views = retained + return result diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 5260a6817..cc8fcc437 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -31,6 +31,7 @@ is_current, track_completion, ) +from orcapod.core.nodes._retained_rows import RetainedRowsMixin from orcapod import contexts from orcapod.channels import Channel, ReadableChannel, WritableChannel from orcapod.config import OrcapodConfig @@ -222,7 +223,7 @@ def _bare_digest(hash_string: str) -> str: _TAG_NODE_INPUT_REF = "_tag_node_input_ref" -def _with_datagram_id(data: Data, datagram_id: uuid.UUID) -> Data: +def _with_datagram_id(data: DataProtocol, datagram_id: uuid.UUID) -> Data: """Return ``data`` rebuilt with ``datagram_id`` as its datagram id. A stored row read back is built by a stream, which gives every row a fresh id. @@ -1038,7 +1039,7 @@ class _JoinedRecords(NamedTuple): empty_taginfo_rows: dict[bytes, dict] -class FunctionJobNode(FunctionNodeBase): +class FunctionJobNode(RetainedRowsMixin, FunctionNodeBase): """DB-backed execution node for function pod invocations. Created by ``PipelineJob`` at run time; never recorded inside a plain @@ -1905,6 +1906,8 @@ def execute( if self._function_pod is None: raise self._cannot_compute_error() + was_current = is_current(self) + missed_cache = False self._begin_run() # A whole-input condition, not a row failure: the error policy does not apply. require_same_data_context( @@ -1948,6 +1951,7 @@ def execute( output.append((tag_out, result)) continue + missed_cache = True pkt_logger = ctx_obs.create_data_logger(tag, data) try: tag_out, result = self._process_data_internal( @@ -2007,7 +2011,8 @@ def execute( ctx_obs.on_node_end(node_label, node_hash) # Mark this node as freshly computed so subsequent iter_data() calls # skip the is_stale check and serve results directly from the in-memory cache. - self._update_modified_time() + if not (was_current and not missed_cache and is_current(self)): + self._update_modified_time() return output def _process_data_internal( @@ -2339,8 +2344,8 @@ def compute_base_entry_id( rows = self._entry_id_rows = {} # A sync storage view shares its origin's exact Arrow table and hashes. # Recognize that representation-only boundary across orchestrators too. - tag_identity = getattr(tag, "_stored_row_origin", tag) - row_identity = getattr(input_data, "_stored_row_origin", input_data) + tag_identity = getattr(tag, "storage_origin", tag) + row_identity = getattr(input_data, "storage_origin", input_data) key = (id(tag_identity), id(row_identity), self.data_context_key) cached = rows.get(key) if ( @@ -3348,6 +3353,8 @@ async def async_execute( """ from orcapod.pipeline.serialization import LoadStatus + was_current = is_current(self) + missed_cache = self._cached_function_pod is None try: self._begin_run() except StoreError: @@ -3430,6 +3437,7 @@ async def async_execute( async def route_inputs() -> None: """Stage 1: send cache hits to output; stamp misses for computation.""" + nonlocal missed_cache try: async for tag, data in input_channel: base_entry_id = self.compute_base_entry_id(tag, data) @@ -3452,6 +3460,7 @@ async def route_inputs() -> None: if cached_data is not None: await output.send((cached_tag, cached_data)) else: + missed_cache = True correlation_key = uuid.uuid4().bytes input_store[correlation_key] = ( tag, @@ -3627,7 +3636,8 @@ def create_data_logger(self, tag: Any, data: Any) -> Any: # The run is complete: stamp the node as the sync execute() does, before # the finally closes the output, so every downstream node, which stamps # after it has seen that close, is newer than this one (ITL-1042). - self._update_modified_time() + if not (was_current and not missed_cache and is_current(self)): + self._update_modified_time() publish_completion(self, observer) except Exception as run_error: self._commit_after_failure(run_error) diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index 34bf20670..ded870ffe 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -29,6 +29,7 @@ track_completion, upstreams_succeeded, ) +from orcapod.core.nodes._retained_rows import RetainedRowsMixin from orcapod import contexts from orcapod.channels import Channel, ReadableChannel, WritableChannel from orcapod.config import OrcapodConfig @@ -616,7 +617,7 @@ def as_node(self) -> OperatorNode: # --------------------------------------------------------------------------- -class OperatorJobNode(OperatorNodeBase): +class OperatorJobNode(RetainedRowsMixin, OperatorNodeBase): """DB-backed execution node for operator pod invocations. Stream node representing an operator invocation with optional DB persistence. @@ -1316,7 +1317,7 @@ def execute( if is_current(self) and self._cached_output_stream is not None: output = list(self._cached_output_stream.iter_data()) - self._update_modified_time() + # Replaying unchanged output is not a content modification. ctx_obs.on_node_end(node_label, node_hash) return output @@ -1723,7 +1724,7 @@ async def async_execute( if is_current(self): for row in self._cached_output_stream.iter_data(): await output.send(row) - self._update_modified_time() + # Keep the completed result's timestamp on a proven replay. ctx_obs.on_node_end(node_label, node_hash) return diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index e5546f46b..985e0a63c 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -9,10 +9,13 @@ import functools import logging +from datetime import datetime from abc import ABC, abstractmethod from collections.abc import AsyncIterator, Iterator from typing import TYPE_CHECKING, Any +from orcapod.core.nodes._execution_cache import invalidate +from orcapod.core.nodes._retained_rows import RetainedRowsMixin from orcapod import contexts from orcapod.config import OrcapodConfig from orcapod.core.base import TraceableBase @@ -502,7 +505,7 @@ def from_stream( ) -class SourceJobNode(SourceNodeBase): +class SourceJobNode(RetainedRowsMixin, SourceNodeBase): """Execution-ready source node wrapping an optional concrete stream. Used inside ``PipelineJob._persistent_node_map``. The ``_concrete`` @@ -578,10 +581,31 @@ def bound_source(self) -> StreamProtocol | None: @bound_source.setter def bound_source(self, value: StreamProtocol | None) -> None: - """Bind *value* as the concrete source and invalidate both hash caches.""" + """Bind a source, making an actual rebinding visible to freshness readers.""" + if value is self._bound_source: + return self._bound_source = value + self.clear_cache() + + @property + def last_modified(self) -> datetime | None: + """Follow the binding and its source; unknown source freshness stays unknown.""" + bound = getattr(self, "_bound_source", None) + if bound is None or bound.last_modified is None: + return None + own_time = super().last_modified + return ( + bound.last_modified + if own_time is None + else max(own_time, bound.last_modified) + ) + + def clear_cache(self) -> None: + """Invalidate retained rows, identity caches and this binding's freshness.""" + invalidate(self) self._invalidate_content_hash_cache() self._invalidate_pipeline_hash_cache() + self._update_modified_time() # ------------------------------------------------------------------ # Identity — delegate to bound source when set diff --git a/src/orcapod/core/operators/static_output_pod.py b/src/orcapod/core/operators/static_output_pod.py index 3038198b0..90577a5c7 100644 --- a/src/orcapod/core/operators/static_output_pod.py +++ b/src/orcapod/core/operators/static_output_pod.py @@ -226,30 +226,9 @@ def _rows_to_table( Raises: ValueError: If ``rows`` is empty. """ - from orcapod.system_constants import constants - from orcapod.utils import arrow_utils + from orcapod.core.streams.arrow_table_stream import rows_to_table - if not rows: - raise ValueError("Cannot materialize an empty list of rows into a stream") - - tag_tables = [tag.as_table(columns={"system_tags": True}) for tag, _ in rows] - data_tables = [data.as_table(columns={"source": True}) for _, data in rows] - - # Rows may list the same columns in a different order (cached vs - # freshly computed), so match columns by name. - combined_tags = arrow_utils.concat_tables_by_name(tag_tables) - combined_data = arrow_utils.concat_tables_by_name(data_tables) - - data_schema = combined_data.schema - for i, field in enumerate(data_schema): - if field.name.startswith(constants.SOURCE_PREFIX): - nullable = combined_data.column(i).null_count > 0 - data_schema = data_schema.set(i, field.with_nullable(nullable)) # pyright: ignore[reportUnknownMemberType] # stubs leave Field unparametrized - # Same arrays, new field flags: no cast, so no value or type can change. - combined_data = pa.Table.from_arrays(combined_data.columns, schema=data_schema) - - full_table = arrow_utils.hstack_tables(combined_tags, combined_data) - return full_table, tuple(rows[0][0].keys()) + return rows_to_table(rows) @staticmethod def _materialize_to_stream( @@ -273,18 +252,7 @@ def _materialize_to_stream( Raises: ValueError: If ``rows`` is empty. """ - table, tag_keys = StaticOutputOperatorPod._rows_to_table(rows) - # No ``source_info`` argument: the table's own ``_source_*`` columns - # pass through, one value per row. - stream = ArrowTableStream( - table, - tag_columns=tag_keys, - data_context=getattr( - rows[0][1], "data_context", rows[0][1].data_context_key - ), - ) - stream._cached_elements = list(rows) - return stream + return ArrowTableStream.from_rows(rows) @staticmethod def _materialize_input_stream( diff --git a/src/orcapod/core/sources/base.py b/src/orcapod/core/sources/base.py index 180c182f7..1f3002e8d 100644 --- a/src/orcapod/core/sources/base.py +++ b/src/orcapod/core/sources/base.py @@ -178,6 +178,12 @@ def keys( """Delegate to the underlying stream's keys.""" return self._stream.keys(columns=columns, all_info=all_info) + def clear_cache(self) -> None: + """Release retained rows in a delegated stream, keeping source data intact.""" + clear = getattr(getattr(self, "_stream", None), "clear_cache", None) + if clear is not None: + clear() + def iter_data(self): """Delegate to the underlying stream's iter_data.""" return self._stream.iter_data() diff --git a/src/orcapod/core/sources/spiraldb_table_source.py b/src/orcapod/core/sources/spiraldb_table_source.py index eaebf5c8c..201733013 100644 --- a/src/orcapod/core/sources/spiraldb_table_source.py +++ b/src/orcapod/core/sources/spiraldb_table_source.py @@ -166,7 +166,9 @@ class SpiralDBTableSource(RootSource): later reads, using memory proportional to the snapshot. Failed or partial reads are not retained. Concurrent initial readers may scan independently; the first complete read publishes the retained snapshot. Sources without a pinned version - scan on every read because their freshness cannot be established. + scan on every read because their freshness cannot be established. Set + ``retain_snapshot=False`` for streaming reads that release consumed batches. + ``clear_cache()`` releases retained data without changing the pinned version. Display. ``head(n)`` and the notebook display (its ``head()``) read only the first ``n`` rows (``PREVIEW_ROWS``, 10, by default), in one scan to which ``max_rows`` does @@ -215,6 +217,10 @@ class SpiralDBTableSource(RootSource): memory, about 5 KB per row and node. batch_rows: The most rows handed on at once while iterating (default 8,192). A SpiralDB batch larger than this is held while its slices are handed on. + retain_snapshot: Retain completed pinned reads for reuse (default True). + False keeps streaming reads to the current SpiralDB batch and processing + slice, scanning again on each read. Whole-table methods still materialize + their result. ``clear_cache()`` releases retained rows without repinning. source_id: Canonical source name for the registry and provenance tokens. Defaults to ``"/.@"``. label: Human-readable label for this source node. @@ -258,6 +264,7 @@ def __init__( config: "OrcapodConfig | None" = None, overrides: dict[str, str] | None = None, tags_as_data: Mapping[str, str] | None = None, + retain_snapshot: bool = True, ) -> None: for name, value in (("limit", limit), ("max_rows", max_rows)): if value is not None and not _is_positive_int(value): @@ -280,6 +287,8 @@ def __init__( self._limit = limit self._max_rows = max_rows self._batch_rows = batch_rows + self._retain_snapshot: bool = retain_snapshot + self._snapshot_generation: int = 0 self._retained_streams: tuple[ArrowTableStream, ...] | None = None self._snapshot_lock: threading.Lock = threading.Lock() self._given_asof = asof @@ -490,6 +499,17 @@ def _key_tokens(self, table: "pa.Table") -> list[str]: for i in range(keys.num_rows) ] + def clear_cache(self) -> None: + """Release retained rows without changing the pinned table version. + + Existing readers may finish using their own references, but reads started + before this call cannot republish the cleared snapshot. A later complete + read can retain the same pinned version again when retention is enabled. + """ + with self._snapshot_lock: + self._retained_streams = None + self._snapshot_generation += 1 + def _batch_streams( self, cancelled: threading.Event | None = None ) -> Generator["ArrowTableStream", None, None]: @@ -501,6 +521,8 @@ def _batch_streams( """ with self._snapshot_lock: retained = self._retained_streams + generation = self._snapshot_generation + retain = self._retain_snapshot and self._asof is not None if retained is not None: yield from retained return @@ -509,13 +531,15 @@ def _batch_streams( self._read_batches(limit=self._limit, max_rows=self._max_rows) ) as batches: for stream in batches: - if self._asof is not None: + if retain: streams.append(stream) yield stream - if self._asof is not None: + if retain: with self._snapshot_lock: - if self._retained_streams is None and ( - cancelled is None or not cancelled.is_set() + if ( + self._retained_streams is None + and generation == self._snapshot_generation + and (cancelled is None or not cancelled.is_set()) ): self._retained_streams = tuple(streams) @@ -730,7 +754,9 @@ def _head_note(self, head: "_Head", sorted_by_tags: bool = False) -> str: if not head.total_is_estimate: if shown < head.total_rows: if sorted_by_tags: - return f"{shown:,} rows read, sorted by tags, of {head.total_rows:,}" + return ( + f"{shown:,} rows read, sorted by tags, of {head.total_rows:,}" + ) return f"first {shown:,} rows of {head.total_rows:,}" all_rows = f"all {shown:,} row{'' if shown == 1 else 's'}" return f"{all_rows}, sorted by tags" if sorted_by_tags else all_rows @@ -821,6 +847,7 @@ def to_config(self, db_registry=None) -> dict[str, Any]: "asof": self._given_asof, "max_rows": self._max_rows, "batch_rows": self._batch_rows, + "retain_snapshot": self._retain_snapshot, "tags_as_data": dict(self._tags_as_data) or None, "source_id": self.source_id, **self._identity_config(), @@ -854,6 +881,7 @@ def from_config( asof=config.get("asof"), max_rows=config.get("max_rows", DEFAULT_MAX_ROWS), batch_rows=config.get("batch_rows", DEFAULT_BATCH_ROWS), + retain_snapshot=config.get("retain_snapshot", True), tags_as_data=config.get("tags_as_data"), source_id=config.get("source_id"), label=config.get("label"), diff --git a/src/orcapod/core/streams/arrow_table_stream.py b/src/orcapod/core/streams/arrow_table_stream.py index 153e9038f..4d13ef38d 100644 --- a/src/orcapod/core/streams/arrow_table_stream.py +++ b/src/orcapod/core/streams/arrow_table_stream.py @@ -2,8 +2,8 @@ import itertools import logging -from collections.abc import Collection, Iterator -from typing import TYPE_CHECKING, Any +from collections.abc import Collection, Iterator, Sequence +from typing import TYPE_CHECKING, Any, Self from orcapod import contexts from orcapod.core.datagrams import Data, Tag @@ -33,6 +33,46 @@ logger = logging.getLogger(__name__) +def stored_tag(tag: TagProtocol) -> TagProtocol: + """Retain the exact stored tag representation, excluding metadata.""" + if type(tag) is Tag: + return tag.storage_view(include_meta=False) + return Tag( + tag.as_table(columns={"system_tags": True}), data_context=tag.data_context_key + ) + + +def stored_data(data: DataProtocol) -> DataProtocol: + """Retain the exact stored data representation, excluding metadata.""" + if type(data) is Data: + return data.storage_view(include_meta=False) + return Data( + data.as_table(columns={"source": True}), data_context=data.data_context_key + ) + + +def rows_to_table( + rows: Sequence[tuple[TagProtocol, DataProtocol]], +) -> tuple["pa.Table", tuple[str, ...]]: + """Combine rows in order, preserving per-row provenance and field flags.""" + import pyarrow as pa + + if not rows: + raise ValueError("Cannot materialize an empty list of rows into a stream") + tags = arrow_utils.concat_tables_by_name( + [tag.as_table(columns={"system_tags": True}) for tag, _ in rows] + ) + data = arrow_utils.concat_tables_by_name( + [packet.as_table(columns={"source": True}) for _, packet in rows] + ) + schema = data.schema + for i, field in enumerate(schema): + if field.name.startswith(constants.SOURCE_PREFIX): + schema = schema.set(i, field.with_nullable(data.column(i).null_count > 0)) + data = pa.Table.from_arrays(data.columns, schema=schema) + return arrow_utils.hstack_tables(tags, data), tuple(rows[0][0].keys()) + + class ArrowTableStream(StreamBase): """ An immutable stream based on a PyArrow Table. @@ -147,6 +187,28 @@ def __init__( self._cached_elements: list[tuple[TagProtocol, DataProtocol]] | None = None self._update_modified_time() # set modified time to now + @classmethod + def from_rows( + cls, rows: Sequence[tuple[TagProtocol, DataProtocol]], **kwargs: Any + ) -> Self: + """Build a stream and retain the same canonical rows used for its table. + + Like existing row materialization, this includes source/system columns + and excludes metadata. Live subclass values are decoded as their stored + declared types. Source-field nullability is normalized across the combined + table; individual rows retain their own source schema. Payload values and + hashes are unchanged. Empty inputs require an explicit schema via __init__. + """ + stored = [(stored_tag(tag), stored_data(data)) for tag, data in rows] + table, tags = rows_to_table(stored) + kwargs.setdefault( + "data_context", + getattr(stored[0][1], "data_context", stored[0][1].data_context_key), + ) + stream = cls(table, tag_columns=tags, **kwargs) + stream._cached_elements = stored + return stream + def identity_structure(self) -> Any: if self._producer is not None: return super().identity_structure() diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index 7f9cd1b7f..470418df6 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -860,6 +860,26 @@ def is_runnable(self, node_label: str) -> bool: # run() # ------------------------------------------------------------------ + def clear_cache(self, *, include_sources: bool = True) -> None: + """Release in-memory node caches without deleting persisted results. + + When include_sources is true, also release bound sources that expose + clear_cache(). Shared source objects are cleared once; other jobs using + those objects will also lose that retained working set. Original source + data and pinned versions are preserved. Call between job executions. + """ + seen: set[int] = set() + for node in self._nodes.values(): + clear = getattr(node, "clear_cache", None) + if clear is not None: + clear() + source = getattr(node, "bound_source", None) + if include_sources and source is not None and id(source) not in seen: + seen.add(id(source)) + clear_source = getattr(source, "clear_cache", None) + if clear_source is not None: + clear_source() + def run( self, orchestrator: "SyncPipelineOrchestrator | AsyncPipelineOrchestrator | None" = None, diff --git a/src/orcapod/pipeline/sync_orchestrator.py b/src/orcapod/pipeline/sync_orchestrator.py index 436f7628e..85c1ff69a 100644 --- a/src/orcapod/pipeline/sync_orchestrator.py +++ b/src/orcapod/pipeline/sync_orchestrator.py @@ -225,10 +225,16 @@ def _materialize_as_stream(buf: list[tuple[Any, Any]], upstream_node: Any) -> An Returns: An ArrowTableStream. """ - from orcapod.core.streams.arrow_table_stream import ArrowTableStream + from orcapod.core.streams.arrow_table_stream import ( + ArrowTableStream, + stored_data, + stored_tag, + ) if not buf: - upstream_node._sync_row_views = {} + clear_rows = getattr(upstream_node, "clear_retained_rows", None) + if clear_rows is not None: + clear_rows() # Build an empty stream preserving declared field nullability. tag_schema, data_schema = upstream_node.output_schema( columns={"system_tags": True, "source": True} @@ -246,67 +252,15 @@ def _materialize_as_stream(buf: list[tuple[Any, Any]], upstream_node: Any) -> An data_context=upstream_node.data_context, ) - from orcapod.core.operators.static_output_pod import StaticOutputOperatorPod - - # Each row keeps its own system tags and source info. - full_table, user_tag_keys = StaticOutputOperatorPod._rows_to_table(buf) - - # Pass the upstream node's producer and upstreams so the - # materialized stream inherits the correct identity_structure - # and pipeline_identity_structure (via StreamBase delegation). - # This ensures downstream operators produce correct system tag - # column names (which embed pipeline hashes of their inputs). - producer = upstream_node.producer - upstreams = upstream_node.upstreams - - stream = ArrowTableStream( - full_table, - tag_columns=user_tag_keys, - producer=producer, - upstreams=upstreams, + retain_rows = getattr(upstream_node, "retain_storage_rows", None) + return ArrowTableStream.from_rows( + retain_rows(buf, tag_view=stored_tag, data_view=stored_data) + if retain_rows is not None + else buf, + producer=upstream_node.producer, + upstreams=upstream_node.upstreams, data_context=upstream_node.data_context, ) - # Sync boundaries have always exposed the Arrow-stored representation. - # A dict-born Data can still hold a subclass with fields omitted by its - # declared storage type. Keep a separate, memoized storage view instead of - # mutating the live row (async and standalone paths still see that row). - from orcapod.core.datagrams import Data, Tag - - previous = getattr(upstream_node, "_sync_row_views", {}) - retained = {} - stored_rows = [] - - def stored_view(row: Any, row_type: type) -> Any: - cached = previous.get(id(row)) - if cached is not None and cached[0] is row: - stored = cached[1] - elif type(row) is row_type: - # _rows_to_table above has materialized this exact Arrow table. - # Copying preserves source/meta/context and its cached hashes; - # only the view's Python representation is decoded anew. - stored = row.copy() - stored._data_dict = None - stored._data_python_schema = None - setattr( - stored, - "_stored_row_origin", - getattr(row, "_stored_row_origin", row), - ) - else: - stored = row_type( - row.as_table( - columns={"source": True, "meta": True, "system_tags": True} - ), - data_context=upstream_node.data_context, - ) - retained[id(row)] = (row, stored) - return stored - - for tag, data in buf: - stored_rows.append((stored_view(tag, Tag), stored_view(data, Data))) - upstream_node._sync_row_views = retained - stream._cached_elements = stored_rows - return stream @staticmethod def _gc_buffers( diff --git a/superpowers/plans/2026-11-13-itl-1063-cache-reuse.md b/superpowers/plans/2026-11-13-itl-1063-cache-reuse.md index 4e2ad2211..bc03040bd 100644 --- a/superpowers/plans/2026-11-13-itl-1063-cache-reuse.md +++ b/superpowers/plans/2026-11-13-itl-1063-cache-reuse.md @@ -50,3 +50,15 @@ Remaining performance investigation is captured in ITL-1064; it does not block t Final verification: 11,625 tests passed, 119 skipped, 19 expected failures; basedpyright zero errors; Ruff lint/format and repository documentation checks pass. Delivery: PR https://github.com/metamorphic-labs/orcapod-python/pull/359 opened against main. Initial CI inspected: tests pending; no merge performed. + +## Review revision plan (authorized autonomous follow-through) + +- [x] Integrate current main while preserving published review history. +- [x] Reconcile completed proofs with existing freshness and preserve no-op timestamps. +- [x] Move retained row ownership/conversion to node/datagram/stream APIs. +- [x] Add job/source release and explicit Spiral retention opt-out. +- [x] Document source eligibility and conservative fallback diagnostics. +- [x] Verify failed writes, dangling refs, None datagrams and metadata compatibility. +- [ ] Finish latest-main full suite/type/lint checks, publish and reply to all threads. +- [ ] Repeat CodeRabbit review until the latest revision has no actionable additions. +- [ ] Update the explanatory Page with final decisions and validation. diff --git a/superpowers/specs/2026-11-13-pipeline-cache-reuse-design.md b/superpowers/specs/2026-11-13-pipeline-cache-reuse-design.md index 8d393a99d..527815219 100644 --- a/superpowers/specs/2026-11-13-pipeline-cache-reuse-design.md +++ b/superpowers/specs/2026-11-13-pipeline-cache-reuse-design.md @@ -136,3 +136,39 @@ Warm user-function calls were zero both before and after. These are local illust timings, not performance guarantees; operation counts are regression-tested. Sync still materializes Arrow tables at node boundaries, and both paths retain traversal/callback cost. No constant-time, cross-process or newly constructed job guarantee is claimed. + +## Claude/eywalker review and latest-main integration + +Merged main `5c7a058f` (including stored data-ID readback, stale table-cache, +sorted previews, interleaved stream reads, crash-recovery isolation and Join fixes) +without rewriting the published branch. + +- Completed-work reuse now also requires the existing `not node.is_stale` gate. + Proven no-op function/operator executions preserve timestamps; actual work, failures, + unknown sources and changed bindings keep normal freshness updates. Source nodes + follow their bound source freshness and explicitly invalidate on rebinding/clearing. +- `Datagram.storage_view()` owns conversion to declared stored values. Nodes own + memoized views through a release method; `ArrowTableStream.from_rows()` owns its + row buffer. Orchestration no longer mutates datagram internals or stream buffers. + Source-field nullability still normalizes across a combined table, whereas each + retained row keeps its own source schema. Regression coverage verifies identical + source values, payload hashes and materialized stored schemas. +- Retention remains the default, matching the requested reusable streams. It is O(N), + not bounded streaming memory. `job.clear_cache()` releases node and bound-source + memory without deleting persistent results; `include_sources=False` keeps shared + source snapshots. Spiral `retain_snapshot=False` opts out; source `clear_cache()` + does not repin and generation checks prevent an in-flight read from republishing. +- Keep the exact supported-source allowlist, document it and log one INFO explanation + per run for unsupported sources. A general source snapshot capability remains a + separately designed follow-up in ITL-1064; timestamps alone are not sufficient. +- Restore Index/Pick metadata expected-failure markers and historical metadata + exclusion at materialization boundaries; no stored hash/schema migration is added. +- Added store-write failure, dangling-reference repair/failure and true None-datagram + regressions for both orchestrators. A function returning a Python None value is + distinct from producing no datagram. All decorated execute methods take observer + keyword-only, so positional-observer wrapping is not a supported missing path. + +The original benchmark above remains a historical comparison against `89160c47`. +A new run after integrating main again records zero warm Arrow hashes, user function +calls and GroupBy calls in all four cases. Timing collected alongside validation is +not a replacement for isolated measurements on the actual ediop dataset. diff --git a/tests/test_core/operators/test_async_sync_equivalence.py b/tests/test_core/operators/test_async_sync_equivalence.py index da915fa3c..420a5b72f 100644 --- a/tests/test_core/operators/test_async_sync_equivalence.py +++ b/tests/test_core/operators/test_async_sync_equivalence.py @@ -777,6 +777,8 @@ def make_sources() -> tuple[StreamProtocol, ...]: "group_by": _MATERIALIZE_DROPS_META, "polars_filter": _MATERIALIZE_DROPS_META, "select_data_columns": _SYNC_DROPS_META, + "index": _SYNC_DROPS_META, + "pick": _SYNC_DROPS_META, } META_CASES = [ pytest.param( diff --git a/tests/test_core/operators/test_materialize_source_info.py b/tests/test_core/operators/test_materialize_source_info.py index 59a7c9f8b..d5491364d 100644 --- a/tests/test_core/operators/test_materialize_source_info.py +++ b/tests/test_core/operators/test_materialize_source_info.py @@ -156,6 +156,16 @@ def broadcasting( monkeypatch.setattr( StaticOutputOperatorPod, "_rows_to_table", staticmethod(broadcasting) ) + # Recreate the historical table round trip at the new materialization API too. + # Patching only the old helper no longer constructs an old-format store. + from orcapod.core.streams.arrow_table_stream import ArrowTableStream + + def historical_stream(cls, rows, **kwargs): + table, tag_keys = broadcasting(rows) + kwargs.setdefault("data_context", rows[0][1].data_context_key) + return cls(table, tag_columns=tag_keys, **kwargs) + + monkeypatch.setattr(ArrowTableStream, "from_rows", classmethod(historical_stream)) def _stored_source(node: OperatorJobNode, column: str = "x") -> list[Any]: diff --git a/tests/test_core/sources/test_spiraldb_table_source_streaming.py b/tests/test_core/sources/test_spiraldb_table_source_streaming.py index 467dbb2ca..bb281ae00 100644 --- a/tests/test_core/sources/test_spiraldb_table_source_streaming.py +++ b/tests/test_core/sources/test_spiraldb_table_source_streaming.py @@ -939,3 +939,69 @@ def test_first_completed_scan_remains_retained_when_other_reader_finishes( for (tag, data), (retained_tag, retained_data) in zip(winner, retained) ) assert len(fake_pyspiral.scan_log) == 2 + + +class TestSnapshotRetentionControls: + """Retention can be disabled or released without changing the pinned version.""" + + def test_opt_out_reads_again_and_round_trips(self, project): + source = SpiralDBTableSource(_PROJECT, "event", retain_snapshot=False) + config = source.to_config() + assert config["retain_snapshot"] is False + restored = SpiralDBTableSource.from_config(config) + assert fake_pyspiral.scan_log == [] + first = list(restored.iter_data()) + second = list(restored.iter_data()) + assert len(first) == len(second) == _ROWS + assert len(fake_pyspiral.scan_log) == 2 + assert all(a is not b for (_, a), (_, b) in zip(first, second)) + + def test_clear_releases_snapshot_without_repinning(self, project, monkeypatch): + from orcapod.databases.spiraldb_connector import SpiralDBConnector + + source = SpiralDBTableSource(_PROJECT, "event") + first = list(source.iter_data()) + pinned_version = fake_pyspiral.scan_log[0].kwargs["asof"] + monkeypatch.setattr(SpiralDBConnector, "get_table_version", lambda *args: 99) + source.clear_cache() + second = list(source.iter_data()) + assert len(second) == _ROWS + assert [scan.kwargs["asof"] for scan in fake_pyspiral.scan_log] == [ + pinned_version, + pinned_version, + ] + assert all(a is not b for (_, a), (_, b) in zip(first, second)) + assert len(list(source.iter_data())) == _ROWS + assert len(fake_pyspiral.scan_log) == 2 + + def test_read_started_before_clear_cannot_republish(self, project): + source = SpiralDBTableSource(_PROJECT, "event") + older = source.iter_data() + next(older) + source.clear_cache() + assert len(list(older)) == _ROWS - 1 + assert len(list(source.iter_data())) == _ROWS + assert len(fake_pyspiral.scan_log) == 2 + + def test_streaming_opt_out_releases_consumed_batches(self, project, monkeypatch): + import weakref + + from orcapod.core.sources.stream_builder import SourceStreamBuilder + + built = [] + original = SourceStreamBuilder.attach_provenance + + def attach(builder, *args, **kwargs): + stream = original(builder, *args, **kwargs) + built.append(weakref.ref(stream)) + return stream + + source = SpiralDBTableSource(_PROJECT, "event", retain_snapshot=False) + monkeypatch.setattr(SourceStreamBuilder, "attach_provenance", attach) + rows = source.iter_data() + for _ in range(5): + next(rows) + assert len(built) == 3 + assert built[0]() is None + assert built[1]() is None + rows.close() diff --git a/tests/test_pipeline/test_execution_cache_freshness.py b/tests/test_pipeline/test_execution_cache_freshness.py new file mode 100644 index 000000000..320ed3f1a --- /dev/null +++ b/tests/test_pipeline/test_execution_cache_freshness.py @@ -0,0 +1,180 @@ +"""A completed no-op job rerun does not invalidate external lazy readers.""" + +from __future__ import annotations + +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.core.nodes import SourceJobNode +from orcapod.core.function_pod import FunctionPodStream +from orcapod.core.data_function import PythonDataFunction +from orcapod.core.operators import GroupBy +from orcapod.databases import InMemoryArrowDatabase +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.sources import DictSource + +READS: list[int] = [] + + +@function_pod(output_keys="x") +def twice(x: int) -> int: + """Transform source values before grouping.""" + return 2 * x + + +@function_pod(output_keys="total") +def total(x: list[int]) -> int: + """Reduce a grouped row.""" + return sum(x) + + +@function_pod(output_keys="read") +def read_total(total: int) -> int: + """Record actual lazy-reader computations.""" + READS.append(total) + return total + + +def make_job(source=None, function=twice): + """Build a source/function/operator/function chain.""" + source = source or DictSource( + [{"g": 1, "x": 2}, {"g": 1, "x": 3}], tag_columns=["g"], source_id="rows" + ) + job = PipelineJob(name="freshness", store=InMemoryArrowDatabase()) + with job: + doubled = function.pod(source, label="doubled") + grouped = GroupBy(by=["g"])(doubled, label="grouped") + total.pod(grouped, label="total") + return job + + +@pytest.mark.parametrize("first", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator]) +@pytest.mark.parametrize( + "second", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator] +) +def test_noop_rerun_preserves_node_times_and_lazy_reader(first, second) -> None: + """Proven cache hits retain completed timestamps across orchestrator changes.""" + job = make_job() + before_first = {name: node.last_modified for name, node in job.nodes.items()} + job.run(orchestrator=first()) + after_first = {name: node.last_modified for name, node in job.nodes.items()} + assert after_first["total"] > before_first["total"] + reader = read_total.pod(job.nodes["total"]) + assert isinstance(reader, FunctionPodStream) + assert [data.as_dict()["read"] for _, data in reader.iter_data()] == [10] + assert not reader.is_stale + READS.clear() + + summary = job.run(orchestrator=second()) + + assert {name: node.last_modified for name, node in job.nodes.items()} == after_first + assert not reader.is_stale + assert [data.as_dict()["read"] for _, data in reader.iter_data()] == [10] + assert READS == [] + assert summary.nodes["doubled"].reused == 2 + assert summary.nodes["total"].reused == 1 + + +@pytest.mark.parametrize( + "orchestrator", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator] +) +def test_changed_source_advances_time_and_invalidates_lazy_reader(orchestrator) -> None: + """A new binding changes freshness even if its source id is unchanged.""" + job = make_job() + job.run(orchestrator=orchestrator()) + reader = read_total.pod(job.nodes["total"]) + assert isinstance(reader, FunctionPodStream) + list(reader.iter_data()) + before = job.nodes["total"].last_modified + source = next( + node for node in job.nodes.values() if isinstance(node, SourceJobNode) + ) + source_time = source.last_modified + source.bound_source = DictSource( + [{"g": 1, "x": 9}], tag_columns=["g"], source_id="rows" + ) + rebound_time = source.last_modified + assert rebound_time is not None and source_time is not None + assert rebound_time > source_time + assert job.nodes["doubled"].is_stale + job.run(orchestrator=orchestrator()) + assert job.nodes["total"].last_modified > before + assert reader.is_stale + READS.clear() + list(reader.iter_data()) + assert 18 in READS + + +@pytest.mark.parametrize( + "orchestrator", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator] +) +def test_unknown_source_rerun_remains_conservatively_freshened(orchestrator) -> None: + """A stable-looking timestamp never proves a custom source is unchanged.""" + + class UnknownSource(DictSource): + pass + + job = make_job(UnknownSource([{"g": 1, "x": 2}], tag_columns=["g"])) + job.run(orchestrator=orchestrator()) + before = job.nodes["total"].last_modified + job.run(orchestrator=orchestrator()) + assert job.nodes["total"].last_modified > before + + +def test_source_node_follows_bound_source_timestamp() -> None: + """Unknown bound freshness remains unknown, including an unbound source slot.""" + source = DictSource([{"x": 1}], source_id="rows") + node = SourceJobNode(name="rows", bound_source=source) + source._set_modified_time(None) + assert node.last_modified is None + node.bound_source = None + assert node.last_modified is None + + +@pytest.mark.parametrize( + "orchestrator", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator] +) +def test_failed_warm_run_cannot_keep_noop_freshness(orchestrator) -> None: + """An interrupted invocation loses completion, even when its rows were cache hits.""" + from orcapod.pipeline.observer import NoOpObserver + + job = make_job() + job.run(orchestrator=orchestrator()) + before = job.nodes["doubled"].last_modified + + class Interrupt(NoOpObserver): + def on_data_start(self, node_label, tag, data): + if node_label == "doubled": + raise ValueError("interrupted cached read") + + with pytest.raises((ValueError, ExceptionGroup)): + job.run(orchestrator=orchestrator(), observer=Interrupt()) + assert job.nodes["doubled"]._completed_execution is None + job.run(orchestrator=orchestrator()) + assert job.nodes["doubled"].last_modified > before + + +@function_pod(output_keys="x") +def disabled(x: int) -> int: + """A pod deactivated in the empty-output regression.""" + return x + + +@pytest.mark.parametrize("first", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator]) +@pytest.mark.parametrize( + "second", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator] +) +def test_completed_empty_outputs_keep_timestamp(first, second) -> None: + """A successfully empty result is complete, rather than perpetually modified.""" + data_function = disabled.pod.data_function + assert isinstance(data_function, PythonDataFunction) + data_function.set_active(False) + try: + job = make_job(function=disabled) + job.run(orchestrator=first()) + before = {name: node.last_modified for name, node in job.nodes.items()} + job.run(orchestrator=second()) + assert job.nodes["total"].as_table().num_rows == 0 + assert {name: node.last_modified for name, node in job.nodes.items()} == before + finally: + data_function.set_active(True) diff --git a/tests/test_pipeline/test_execution_cache_reuse.py b/tests/test_pipeline/test_execution_cache_reuse.py index 73781a7e6..b03c8eef9 100644 --- a/tests/test_pipeline/test_execution_cache_reuse.py +++ b/tests/test_pipeline/test_execution_cache_reuse.py @@ -348,6 +348,7 @@ class Child(Base): def test_sync_storage_views_are_retained_without_mutating_originals() -> None: """The sync boundary decodes the declared type once while preserving live originals.""" from types import SimpleNamespace + from orcapod.core.nodes._retained_rows import RetainedRowsMixin from orcapod.core.datagrams import Data, Tag @@ -360,9 +361,11 @@ def test_sync_storage_views_are_retained_without_mutating_originals() -> None: tag = Tag( {"k": cast(DataValue, cast(object, Child(2, 88)))}, python_schema={"k": Base} ) - upstream = SimpleNamespace( - producer=None, upstreams=(), data_context=original.data_context - ) + + class Upstream(RetainedRowsMixin, SimpleNamespace): + pass + + upstream = Upstream(producer=None, upstreams=(), data_context=original.data_context) first = SyncPipelineOrchestrator._materialize_as_stream([(tag, original)], upstream) second = SyncPipelineOrchestrator._materialize_as_stream( [(tag, original)], upstream diff --git a/tests/test_pipeline/test_execution_cache_review.py b/tests/test_pipeline/test_execution_cache_review.py index 13d817f14..8012b29a1 100644 --- a/tests/test_pipeline/test_execution_cache_review.py +++ b/tests/test_pipeline/test_execution_cache_review.py @@ -22,6 +22,7 @@ class Node: """A completion-aware node with observable definition reads.""" data_context_key = "test" + is_stale = False def __init__(self, *upstreams: Any) -> None: self.upstreams = upstreams diff --git a/tests/test_pipeline/test_execution_cache_review_failures.py b/tests/test_pipeline/test_execution_cache_review_failures.py new file mode 100644 index 000000000..bfcb8302e --- /dev/null +++ b/tests/test_pipeline/test_execution_cache_review_failures.py @@ -0,0 +1,160 @@ +"""Completion accounting for failures and successful filtered rows (ITL-1063).""" + +from __future__ import annotations + +import logging +from collections.abc import Iterator +from typing import cast + +from orcapod.core.data_function import PythonDataFunction +from orcapod.core.function_pod import FunctionPod +from orcapod.protocols.database_protocols import ArrowDatabaseProtocol + +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.databases import InMemoryArrowDatabase +from orcapod.errors import StoreWriteError +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.sources import DictSource +from tests.test_pipeline.test_store_write_failure import ( + RESULT_TABLE, + ROWS, + CrashingObserver, + Faults, + FaultyStore, + build_job, +) + +ORCHESTRATORS = [SyncPipelineOrchestrator, AsyncPipelineOrchestrator] +CALLS: list[int] = [] +fail_repair = False + + +@function_pod(output_keys="y") +def recover(x: int) -> int: + """Count every recovery attempt and optionally fail it.""" + CALLS.append(x) + if fail_repair: + raise ValueError("repair failed") + return x * 2 + + +@function_pod(output_keys="y") +def filter_even(x: int) -> int | None: + """Return no row for even inputs, counting successful filtering too.""" + CALLS.append(x) + return x if x % 2 else None + + +def make_job( + store: ArrowDatabaseProtocol, pod: FunctionPod = cast(FunctionPod, recover.pod) +) -> PipelineJob: + """Build the same stable two-row job over the supplied store.""" + job = PipelineJob(name="review_recovery", store=store) + with job: + _ = pod( + DictSource([{"k": 1, "x": 1}, {"k": 2, "x": 2}], tag_columns=["k"]), + label="work", + ) + return job + + +@pytest.fixture(autouse=True) +def reset() -> Iterator[None]: + """Keep deliberate row failures and call counts local to each test.""" + global fail_repair + fail_repair = False + CALLS.clear() + assert isinstance(recover.pod.data_function, PythonDataFunction) + recover.pod.data_function.set_active(True) + yield + fail_repair = False + CALLS.clear() + assert isinstance(recover.pod.data_function, PythonDataFunction) + recover.pod.data_function.set_active(True) + + +@pytest.mark.parametrize("orchestrator", ORCHESTRATORS) +def test_kept_store_error_never_certifies_completion( + orchestrator: type[SyncPipelineOrchestrator] | type[AsyncPipelineOrchestrator], +) -> None: + """A failing observer cannot turn a propagating write error into completion.""" + faults = Faults(write_marker=RESULT_TABLE) + job = build_job(FaultyStore(InMemoryArrowDatabase(), faults), ROWS) + with pytest.raises(StoreWriteError) as info: + job.run(orchestrator=orchestrator(), observer=CrashingObserver()) + assert any("on_data_crash" in note for note in info.value.__notes__) + assert job.nodes["dbl"]._completed_execution is None + + +@pytest.mark.parametrize("orchestrator", ORCHESTRATORS) +@pytest.mark.parametrize("repair_fails", [False, True]) +def test_dangling_result_is_repaired_before_completion( + orchestrator: type[SyncPipelineOrchestrator] | type[AsyncPipelineOrchestrator], + repair_fails: bool, + caplog: pytest.LogCaptureFixture, +) -> None: + """Missing results are misses; only successful recovery certifies the new job.""" + global fail_repair + store = InMemoryArrowDatabase() + make_job(store).run(orchestrator=orchestrator()) + # Retain pipeline references but remove their result tables, as external loss can. + for tables in (store._tables, store._pending_batches): + for key in list(tables): + if RESULT_TABLE in key: + del tables[key] + CALLS.clear() + fail_repair = repair_fails + job = make_job(store) + with caplog.at_level(logging.WARNING, logger="orcapod.core.nodes.function_node"): + summary = job.run(orchestrator=orchestrator()) + assert any("have no match in persistent result DB" in m for m in caplog.messages) + assert sorted(CALLS) == [1, 2] + assert summary.nodes["work"].failed == (2 if repair_fails else 0) + assert (job.nodes["work"]._completed_execution is None) == repair_fails + assert job.nodes["work"].as_table().num_rows == (0 if repair_fails else 2) + if repair_fails: + fail_repair = False + CALLS.clear() + assert job.run(orchestrator=orchestrator()).ok + assert sorted(CALLS) == [1, 2] + assert job.nodes["work"]._completed_execution is not None + + +@pytest.mark.parametrize("first", ORCHESTRATORS) +@pytest.mark.parametrize("second", ORCHESTRATORS) +def test_none_results_are_successful_cached_filters( + first: type[SyncPipelineOrchestrator] | type[AsyncPipelineOrchestrator], + second: type[SyncPipelineOrchestrator] | type[AsyncPipelineOrchestrator], +) -> None: + """Both paths count filtered rows as success and never forward a None datagram.""" + job = make_job(InMemoryArrowDatabase()) + assert isinstance(recover.pod.data_function, PythonDataFunction) + recover.pod.data_function.set_active(False) + cold = job.run(orchestrator=first()) + assert CALLS == [] + assert cold.nodes["work"].computed == 2 + assert job.nodes["work"].as_table().num_rows == 0 + assert all( + data is None for _, data in job.nodes["work"]._cached_output_datas.values() + ) + CALLS.clear() + warm = job.run(orchestrator=second()) + assert CALLS == [] + assert warm.nodes["work"].computed == warm.nodes["work"].failed == 0 + assert warm.nodes["work"].reused == 2 + assert job.nodes["work"].as_table().num_rows == 0 + + +@pytest.mark.parametrize("orchestrator", ORCHESTRATORS) +def test_nullable_output_is_a_row_not_a_filtered_result( + orchestrator: type[SyncPipelineOrchestrator] | type[AsyncPipelineOrchestrator], +) -> None: + """A Python function returning a nullable value produces Data containing None.""" + job = make_job(InMemoryArrowDatabase(), cast(FunctionPod, filter_even.pod)) + job.run(orchestrator=orchestrator()) + assert job.nodes["work"].as_table().column("y").to_pylist() == [1, None] + assert all( + data is not None for _, data in job.nodes["work"]._cached_output_datas.values() + ) diff --git a/tests/test_pipeline/test_retained_storage_views.py b/tests/test_pipeline/test_retained_storage_views.py new file mode 100644 index 000000000..cece700e3 --- /dev/null +++ b/tests/test_pipeline/test_retained_storage_views.py @@ -0,0 +1,123 @@ +"""Retained row views have one stored representation and an explicit lifetime.""" + +from dataclasses import dataclass +from typing import Any, cast + +from orcapod.core.datagrams import Data, Tag +from orcapod.core.operators.static_output_pod import StaticOutputOperatorPod +from orcapod.core.streams.arrow_table_stream import ArrowTableStream + + +@dataclass +class StoredBase: + value: int + + +@dataclass +class LiveChild(StoredBase): + extra: int + + +def rows(): + """Live values whose stored declared class excludes the extra field.""" + return [ + ( + Tag({"k": 1}), + Data( + cast(Any, {"x": LiveChild(2, 9)}), + python_schema={"x": StoredBase}, + meta_info={"note": "private"}, + ), + ) + ] + + +def test_datagram_storage_view_preserves_original_and_hash() -> None: + """Canonicalization belongs to the datagram and never mutates its live values.""" + original = rows()[0][1] + stored = original.storage_view() + assert type(stored.as_dict()["x"]) is StoredBase + assert type(original.as_dict()["x"]) is LiveChild + assert stored.storage_origin is original + assert stored.content_hash() == original.content_hash() + assert stored.meta_columns == original.meta_columns + assert not original.storage_view(include_meta=False).meta_columns + + +def test_materialized_stream_rows_agree_with_stored_values() -> None: + """Static materialization must not expose subclass-only Python fields.""" + stream = StaticOutputOperatorPod._materialize_to_stream(rows()) + [(_, data)] = list(stream.iter_data()) + assert type(data.as_dict()["x"]) is StoredBase + assert not data.meta_columns + decoded = Data(stream.as_table().select(["x"]), data_context=data.data_context_key) + assert data.as_dict() == decoded.as_dict() + + +def test_arrow_row_factory_retains_canonical_objects() -> None: + """The stream owns its row buffer through a factory that builds the same table.""" + stream = ArrowTableStream.from_rows(rows()) + first, second = list(stream.iter_data()), list(stream.iter_data()) + assert first[0][0] is second[0][0] + assert first[0][1] is second[0][1] + assert type(first[0][1].as_dict()["x"]) is StoredBase + + +def test_job_clear_cache_releases_rows_and_keeps_persistent_results() -> None: + """A job can drop its in-memory working set and recover from its result store.""" + from tests.test_pipeline.test_execution_cache_reuse import make_chain + + job = make_chain() + assert job.run().ok + expected = job.nodes["total"].as_table(all_info=True) + assert any(getattr(node, "_sync_row_views", {}) for node in job.nodes.values()) + job.clear_cache() + for node in job.nodes.values(): + assert not getattr(node, "_sync_row_views", {}) + assert not getattr(node, "_entry_id_rows", {}) + assert getattr(node, "_completed_execution", None) is None + assert job.run().ok + assert job.nodes["total"].as_table(all_info=True).equals(expected) + + +def test_unsupported_source_emits_one_explanation_per_run(caplog) -> None: + """Users can tell why an otherwise stable custom source misses the fast path.""" + import logging + from orcapod import PipelineJob + from orcapod.databases import InMemoryArrowDatabase + from orcapod.sources import DictSource + from tests.test_pipeline.test_execution_cache_reuse import doubled + + class CustomSource(DictSource): + pass + + with PipelineJob(store=InMemoryArrowDatabase()) as job: + doubled.pod( + CustomSource([{"k": 1, "x": 2}], tag_columns=["k"], source_id="custom") + ) + with caplog.at_level(logging.INFO, logger="orcapod.core.nodes._execution_cache"): + assert job.run().ok + messages = [ + record.message + for record in caplog.records + if "Completed-work reuse disabled" in record.message + ] + assert len(messages) == 1 + assert "CustomSource" in messages[0] + + +def test_row_factory_preserves_source_values_with_table_nullability() -> None: + """Combined source fields normalize nullability without changing payload IDs.""" + original = Data({"x": 1}, source_info={"x": "token"}) + stream = ArrowTableStream.from_rows([(Tag({"k": 1}), original)]) + [(_, retained)] = list(stream.iter_data()) + row_table = retained.as_table(columns={"source": True}) + combined = stream.as_table(columns={"source": True}).select(row_table.column_names) + assert combined.to_pylist() == row_table.to_pylist() + assert not combined.schema.field("_source_x").nullable + assert retained.content_hash() == original.content_hash() + # Persisting retained rows and decoding the table use the same normalized schema. + decoded = Data(combined, data_context=original.data_context_key) + from_rows, _ = StaticOutputOperatorPod._rows_to_table([(Tag({"k": 1}), retained)]) + from_table, _ = StaticOutputOperatorPod._rows_to_table([(Tag({"k": 1}), decoded)]) + assert from_rows.equals(from_table, check_metadata=True) diff --git a/tests/test_pipeline/test_spiral_execution_cache_reuse.py b/tests/test_pipeline/test_spiral_execution_cache_reuse.py index 0d1b3eeaf..759306b3b 100644 --- a/tests/test_pipeline/test_spiral_execution_cache_reuse.py +++ b/tests/test_pipeline/test_spiral_execution_cache_reuse.py @@ -65,3 +65,39 @@ def test_pinned_grouped_job_reuses_scan_rows_and_hashes( assert summary.nodes["total"].reused == 5 assert len(fake_pyspiral.scan_log) == 1 assert job.nodes["total"].as_table(all_info=True).equals(before) + + +@pytest.mark.parametrize("retain_snapshot", [False, True]) +@pytest.mark.parametrize( + "orchestrator", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator] +) +def test_source_release_or_optout_prevents_completed_operator_reuse( + tmp_path: Path, retain_snapshot: bool, orchestrator +) -> None: + """A source-only clear invalidates downstream proofs; an opt-out never creates one.""" + with fake_pyspiral.install(tmp_path / "spiral"): + project = fake_pyspiral.FakeSpiral().project("release-integration") + project.create_table( + "default.events", key_schema=pa.schema([("g", pa.int64())]) + ).write(pa.table({"g": [1, 2], "x": [3, 4]})) + source = SpiralDBTableSource( + "release-integration", "events", retain_snapshot=retain_snapshot + ) + with PipelineJob(name="release", store=InMemoryArrowDatabase()) as job: + total.pod(source.group_by(["g"]), label="total") + fake_pyspiral.scan_log.clear() + assert job.run(orchestrator=orchestrator()).ok + if retain_snapshot: + source.clear_cache() + calls = 0 + original = GroupBy.unary_static_process + + def counted(self, *args, **kwargs): + nonlocal calls + calls += 1 + return original(self, *args, **kwargs) + + with patch.object(GroupBy, "unary_static_process", counted): + assert job.run(orchestrator=orchestrator()).ok + assert len(fake_pyspiral.scan_log) == 2 + assert calls == 1 diff --git a/tests/test_pipeline/test_tag_table_source_info.py b/tests/test_pipeline/test_tag_table_source_info.py index 58af5c7c4..1b2c14c15 100644 --- a/tests/test_pipeline/test_tag_table_source_info.py +++ b/tests/test_pipeline/test_tag_table_source_info.py @@ -174,18 +174,16 @@ def broadcasting( monkeypatch.setattr( StaticOutputOperatorPod, "_rows_to_table", staticmethod(broadcasting) ) - # The old materializer rebuilt row objects from this corrupted table. Modern - # materializers preserve the original rows, so recreate that historical read - # path too; otherwise this fixture would no longer write an old-format store. + # Recreate the historical table round trip at the new materialization API too. + # Patching only the old helper no longer constructs an old-format store. from orcapod.core.streams.arrow_table_stream import ArrowTableStream - original_iter = ArrowTableStream.iter_data + def historical_stream(cls, rows, **kwargs): + table, tag_keys = broadcasting(rows) + kwargs.setdefault("data_context", rows[0][1].data_context_key) + return cls(table, tag_columns=tag_keys, **kwargs) - def historical_rows(self: ArrowTableStream): - self._cached_elements = None - yield from original_iter(self) - - monkeypatch.setattr(ArrowTableStream, "iter_data", historical_rows) + monkeypatch.setattr(ArrowTableStream, "from_rows", classmethod(historical_stream)) @pytest.mark.parametrize("orchestrator", ["sync", "async"]) From 73db288147ae8514437c095cb905554587bba29f Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 09:47:11 -0800 Subject: [PATCH 7/7] fix(cache): invalidate reactivated filters and preserve stored row context --- src/orcapod/core/data_function.py | 16 +- src/orcapod/core/datagrams/datagram.py | 2 + src/orcapod/core/nodes/_execution_cache.py | 3 + src/orcapod/core/nodes/function_node.py | 19 +- .../core/sources/spiraldb_table_source.py | 3 +- .../core/streams/arrow_table_stream.py | 32 ++- .../2026-11-13-pipeline-cache-reuse-design.md | 15 ++ .../test_spiraldb_table_source_streaming.py | 67 +++++++ .../test_execution_cache_activation.py | 187 ++++++++++++++++++ .../test_retained_storage_views.py | 106 ++++++++++ 10 files changed, 442 insertions(+), 8 deletions(-) create mode 100644 tests/test_pipeline/test_execution_cache_activation.py diff --git a/src/orcapod/core/data_function.py b/src/orcapod/core/data_function.py index 3f085e0ef..8603e8dab 100644 --- a/src/orcapod/core/data_function.py +++ b/src/orcapod/core/data_function.py @@ -420,6 +420,7 @@ def __init__( executor = LocalPythonFunctionExecutor() self._function = function + self._activation_generation = 0 self._is_async = inspect.iscoroutinefunction(function) # Reject functions with variadic parameters -- PythonDataFunction maps # data keys to named parameters, so the full parameter set must be fixed. @@ -591,8 +592,16 @@ def is_active(self) -> bool: return self._active def set_active(self, active: bool = True) -> None: - """Set the active state. If False, ``call`` returns None for every data.""" + """Set the active state; disabled calls return None for every input. + + Actual changes tell job nodes to revisit their filtered completions on + the next run. Repeating the same state preserves current results. + """ + if active == self._active: + return self._active = active + self._activation_generation += 1 + self._update_modified_time() @property def is_async(self) -> bool: @@ -960,6 +969,11 @@ def __init__(self, data_function: DataFunctionProtocol, **kwargs) -> None: kwargs["data_context"] = getattr(data_function, "data_context", None) super().__init__(**kwargs) + @property + def _activation_generation(self) -> int: + """Propagate the wrapped function's activation changes through cache wrappers.""" + return getattr(self._data_function, "_activation_generation", 0) + def computed_label(self) -> str | None: return self._data_function.label diff --git a/src/orcapod/core/datagrams/datagram.py b/src/orcapod/core/datagrams/datagram.py index e06dac069..c8ed26d6e 100644 --- a/src/orcapod/core/datagrams/datagram.py +++ b/src/orcapod/core/datagrams/datagram.py @@ -1046,6 +1046,8 @@ def storage_view(self, *, include_meta: bool = True) -> Self: include_meta or not self._meta ): return self + # copy() carries the cached UUID; establish it before preserving identity. + _ = self.datagram_uuid view = self.copy(preserve_id=True) view._data_dict = None view._data_python_schema = None diff --git a/src/orcapod/core/nodes/_execution_cache.py b/src/orcapod/core/nodes/_execution_cache.py index 40b1ca71c..20940925e 100644 --- a/src/orcapod/core/nodes/_execution_cache.py +++ b/src/orcapod/core/nodes/_execution_cache.py @@ -150,6 +150,9 @@ def _definition( return _Definition( ( getattr(stream, "_execution_generation", 0), + getattr( + getattr(stream, "_data_function", None), "_activation_generation", 0 + ), _Identity(producer), producer.last_modified if producer is not None else None, _Identity(getattr(stream, "_node_config", None)), diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index cc8fcc437..d08aee064 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -1167,7 +1167,10 @@ def _delivers_side_effects(self) -> bool: ) def _begin_run(self) -> None: - """Make a node that delivers side effects look for older completions again. + """Refresh filtered activation state and recheck side-effect completions. + + Filtered inputs are revisited after an activation change; successful + stored results remain reusable under their existing keys. Tables written before v0.3 can appear in a store between two runs of one node object. Such a node must find them before it delivers anything, so it @@ -1178,6 +1181,20 @@ def _begin_run(self) -> None: SideEffectLogDataContextError: If this node delivers side effects and completions it cannot match exist; nothing has been delivered then. """ + # A None cache entry means the function filtered this input, not a + # persisted null-valued result. Reactivation must revisit those inputs. + # Keep successful results: activation does not change their stored keys. + activation = getattr(self._data_function, "_activation_generation", None) + if activation != getattr(self, "_cached_activation_generation", None): + self._cached_output_datas = { + key: row + for key, row in self._cached_output_datas.items() + if row[1] is not None + } + self._cached_output_table = None + self._cached_content_hash_column = None + self._completed_execution = None + self._cached_activation_generation = activation if self._delivers_side_effects: self._stale_tables_checked = False self._account_for_stale_tables() diff --git a/src/orcapod/core/sources/spiraldb_table_source.py b/src/orcapod/core/sources/spiraldb_table_source.py index 201733013..044091f3a 100644 --- a/src/orcapod/core/sources/spiraldb_table_source.py +++ b/src/orcapod/core/sources/spiraldb_table_source.py @@ -642,7 +642,8 @@ async def async_iter_data( stream = await asyncio.to_thread(next, streams, None) if stream is None: break - for pair in self._stream_rows(stream): + rows = await asyncio.to_thread(list, self._stream_rows(stream)) + for pair in rows: yield pair finally: cancelled.set() diff --git a/src/orcapod/core/streams/arrow_table_stream.py b/src/orcapod/core/streams/arrow_table_stream.py index 4d13ef38d..dfb3b897a 100644 --- a/src/orcapod/core/streams/arrow_table_stream.py +++ b/src/orcapod/core/streams/arrow_table_stream.py @@ -38,7 +38,8 @@ def stored_tag(tag: TagProtocol) -> TagProtocol: if type(tag) is Tag: return tag.storage_view(include_meta=False) return Tag( - tag.as_table(columns={"system_tags": True}), data_context=tag.data_context_key + tag.as_table(columns={"system_tags": True}), + data_context=getattr(tag, "data_context", tag.data_context_key), ) @@ -47,7 +48,8 @@ def stored_data(data: DataProtocol) -> DataProtocol: if type(data) is Data: return data.storage_view(include_meta=False) return Data( - data.as_table(columns={"source": True}), data_context=data.data_context_key + data.as_table(columns={"source": True}), + data_context=getattr(data, "data_context", data.data_context_key), ) @@ -70,7 +72,8 @@ def rows_to_table( if field.name.startswith(constants.SOURCE_PREFIX): schema = schema.set(i, field.with_nullable(data.column(i).null_count > 0)) data = pa.Table.from_arrays(data.columns, schema=schema) - return arrow_utils.hstack_tables(tags, data), tuple(rows[0][0].keys()) + table = arrow_utils.hstack_tables(tags, data) if tags.num_columns else data + return table, tuple(rows[0][0].keys()) class ArrowTableStream(StreamBase): @@ -197,14 +200,33 @@ def from_rows( and excludes metadata. Live subclass values are decoded as their stored declared types. Source-field nullability is normalized across the combined table; individual rows retain their own source schema. Payload values and - hashes are unchanged. Empty inputs require an explicit schema via __init__. + hashes are unchanged. All rows must share this stream's data context. + Source-info overrides and system-tag relabeling are rejected: transform + the rows first. Empty inputs require an explicit schema via __init__. """ + if kwargs.get("source_info") is not None: + raise ValueError("from_rows does not accept source_info overrides") + if kwargs.get("system_tag_columns"): + raise ValueError("from_rows does not accept system_tag_columns relabeling") stored = [(stored_tag(tag), stored_data(data)) for tag, data in rows] table, tags = rows_to_table(stored) kwargs.setdefault( "data_context", getattr(stored[0][1], "data_context", stored[0][1].data_context_key), ) + context = contexts.resolve_context(kwargs["data_context"]) + if any( + tag.data_context_key != context.context_key + or data.data_context_key != context.context_key + for tag, data in stored + ): + raise ValueError( + "from_rows requires rows and stream to share a data context" + ) + table = table.append_column( + pa.field(constants.CONTEXT_KEY, pa.large_string(), nullable=False), + pa.array([data.data_context_key for _, data in stored], pa.large_string()), + ) stream = cls(table, tag_columns=tags, **kwargs) stream._cached_elements = stored return stream @@ -442,7 +464,7 @@ def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: ) else: # A fresh empty tag per row, so rows never share one object. - tag = Tag({}) + tag = Tag({}, data_context=self.data_context) data = Data( data_batch.slice(i, 1), # type: ignore diff --git a/superpowers/specs/2026-11-13-pipeline-cache-reuse-design.md b/superpowers/specs/2026-11-13-pipeline-cache-reuse-design.md index 527815219..f915ccee7 100644 --- a/superpowers/specs/2026-11-13-pipeline-cache-reuse-design.md +++ b/superpowers/specs/2026-11-13-pipeline-cache-reuse-design.md @@ -172,3 +172,18 @@ The original benchmark above remains a historical comparison against `89160c47`. A new run after integrating main again records zero warm Arrow hashes, user function calls and GroupBy calls in all four cases. Timing collected alongside validation is not a replacement for isolated measurements on the actual ediop dataset. + +## Further CodeRabbit review + +- Activation changes advance a function generation and freshness timestamp. A job + removes retained absent-output entries when that generation changes, preserving + successful cached results. Completion proofs include activation state so downstream + empty outputs cannot hide a reactivated function. Repeated unchanged activation + preserves the warm cache. +- Async Spiral reads construct batch rows and acquire the snapshot lock in a worker + thread, keeping the event loop responsive, including during cancellation. +- The row factory exports the retained context explicitly and requires consistent row + and stream contexts. It rejects provenance overrides and system-tag relabeling that + would silently make table and row views disagree; callers transform rows first. + Empty tags inherit their stream context. Storage views initialize a lazy UUID before + copying so original and view retain the same identity. diff --git a/tests/test_core/sources/test_spiraldb_table_source_streaming.py b/tests/test_core/sources/test_spiraldb_table_source_streaming.py index bb281ae00..a4703c387 100644 --- a/tests/test_core/sources/test_spiraldb_table_source_streaming.py +++ b/tests/test_core/sources/test_spiraldb_table_source_streaming.py @@ -1005,3 +1005,70 @@ def attach(builder, *args, **kwargs): assert built[0]() is None assert built[1]() is None rows.close() + + +def test_async_row_materialization_runs_off_event_loop(project, monkeypatch): + """Batch row creation and its memo lock run in a worker, not on the loop.""" + import threading + + from orcapod.core.streams.arrow_table_stream import ArrowTableStream + + source = SpiralDBTableSource(_PROJECT, "event") + loop_thread = threading.get_ident() + materialization_threads = [] + original = ArrowTableStream.iter_data + + def rows(stream): + materialization_threads.append(threading.get_ident()) + assert threading.get_ident() != loop_thread + yield from original(stream) + + monkeypatch.setattr(ArrowTableStream, "iter_data", rows) + + async def collect(): + return [pair async for pair in source.async_iter_data()] + + first = asyncio.run(collect()) + second = asyncio.run(collect()) + assert len(first) == len(second) == _ROWS + assert materialization_threads + assert all(thread != loop_thread for thread in materialization_threads) + assert all(a is b for (_, a), (_, b) in zip(first, second)) + assert len(fake_pyspiral.scan_log) == 1 + + +def test_cancellation_during_async_row_materialization_does_not_retain( + project, monkeypatch +): + """Cancellation stays responsive while a worker owns the row-memo lock.""" + import threading + + from orcapod.core.streams.arrow_table_stream import ArrowTableStream + + source = SpiralDBTableSource(_PROJECT, "event") + entered = threading.Event() + release = threading.Event() + original = ArrowTableStream.iter_data + + def rows(stream): + entered.set() + assert release.wait(5) + yield from original(stream) + + monkeypatch.setattr(ArrowTableStream, "iter_data", rows) + + async def cancel(): + iterator = source.async_iter_data() + task = asyncio.ensure_future(anext(iterator)) + try: + assert await asyncio.to_thread(entered.wait, 5) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, timeout=1) + finally: + release.set() + + asyncio.run(cancel()) + assert len(list(source.iter_data())) == _ROWS + assert len(list(source.iter_data())) == _ROWS + assert len(fake_pyspiral.scan_log) == 2 diff --git a/tests/test_pipeline/test_execution_cache_activation.py b/tests/test_pipeline/test_execution_cache_activation.py new file mode 100644 index 000000000..d4d30276b --- /dev/null +++ b/tests/test_pipeline/test_execution_cache_activation.py @@ -0,0 +1,187 @@ +"""Reactivating a function discards filtered completions, including downstream empties.""" + +from __future__ import annotations + +from unittest.mock import patch + +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.core.data_function import PythonDataFunction +from orcapod.core.nodes import FunctionJobNode +from orcapod.core.operators import GroupBy +from orcapod.databases import InMemoryArrowDatabase +from orcapod.hashing.arrow_hashers import StarfixArrowHasher +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.sources import DictSource + +CALLS: list[int] = [] + + +@function_pod(output_keys="x") +def doubled(x: int) -> int: + """Record work performed after activation.""" + CALLS.append(x) + return x * 2 + + +@function_pod(output_keys="total") +def total(x: list[int]) -> int: + """Sum a group downstream of the toggled function.""" + return sum(x) + + +def make_job(pod=doubled.pod) -> PipelineJob: + """Build an activation-sensitive function/operator/function chain.""" + job = PipelineJob(name="activation", store=InMemoryArrowDatabase()) + with job: + rows = pod( + DictSource([{"g": 1, "x": 2}, {"g": 1, "x": 3}], tag_columns=["g"]), + label="doubled", + ) + grouped = GroupBy(by=["g"])(rows, label="grouped") + total.pod(grouped, label="total") + return job + + +@pytest.fixture +def data_function(): + """Leave the shared pod active after every test, including a failed assertion.""" + function = doubled.pod.data_function + assert isinstance(function, PythonDataFunction) + CALLS.clear() + try: + yield function + finally: + function.set_active(True) + + +@pytest.mark.parametrize("first", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator]) +@pytest.mark.parametrize( + "second", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator] +) +def test_reactivated_rows_and_downstream_empty_results_are_recomputed( + data_function, first, second +) -> None: + """A prior inactive invocation cannot suppress work after reactivation.""" + data_function.set_active(False) + job = make_job() + job.run(orchestrator=first()) + assert job.nodes["total"].as_table().num_rows == 0 + assert CALLS == [] + data_function.set_active(True) + + summary = job.run(orchestrator=second()) + + assert sorted(CALLS) == [2, 3] + assert summary.nodes["doubled"].computed == 2 + assert job.nodes["total"].as_table().column("total").to_pylist() == [10] + CALLS.clear() + with patch.object( + StarfixArrowHasher, "hash_table", side_effect=AssertionError("rehashed") + ): + job.run(orchestrator=second()) + assert CALLS == [] + + +@pytest.mark.parametrize( + "orchestrator", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator] +) +def test_unchanged_inactive_state_keeps_filtered_cache( + data_function, orchestrator +) -> None: + """An identical set_active call does not force DB reads or invalidate completed rows.""" + data_function.set_active(False) + job = make_job() + job.run(orchestrator=orchestrator()) + before = job.nodes["doubled"].last_modified + data_function.set_active(False) + with patch.object( + FunctionJobNode, "_load_cached_entries", side_effect=AssertionError("loaded") + ): + summary = job.run(orchestrator=orchestrator()) + assert summary.nodes["doubled"].reused == 2 + assert job.nodes["doubled"].last_modified == before + assert job.nodes["total"].as_table().num_rows == 0 + assert CALLS == [] + + +@pytest.mark.parametrize( + "orchestrator", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator] +) +def test_activation_during_completion_callback_cannot_certify_empty_output( + data_function, orchestrator +) -> None: + """The run-start proof notices activation changes before downstream completion.""" + from orcapod.pipeline.observer import NoOpObserver + + class Reactivate(NoOpObserver): + def on_node_end(self, node_label, node_hash): + if node_label == "doubled": + data_function.set_active(True) + + data_function.set_active(False) + job = make_job() + job.run(orchestrator=orchestrator(), observer=Reactivate()) + assert job.nodes["doubled"]._completed_execution is None + assert job.nodes["grouped"]._completed_execution is None + assert job.nodes["total"].as_table().num_rows == 0 + job.run(orchestrator=orchestrator()) + assert sorted(CALLS) == [2, 3] + assert job.nodes["total"].as_table().column("total").to_pylist() == [10] + + +@pytest.mark.parametrize( + "orchestrator", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator] +) +def test_activation_changes_preserve_successful_results( + data_function, orchestrator +) -> None: + """Only filtered entries are incompatible: positive stored results remain reusable.""" + job = make_job() + job.run(orchestrator=orchestrator()) + CALLS.clear() + data_function.set_active(False) + data_function.set_active(True) + summary = job.run(orchestrator=orchestrator()) + assert CALLS == [] + assert summary.nodes["doubled"].reused == 2 + assert job.nodes["total"].as_table().column("total").to_pylist() == [10] + + +def test_noop_activation_setter_keeps_modification_time(data_function) -> None: + """Repeatedly setting the same state is not a semantic modification.""" + before = data_function.last_modified + data_function.set_active(data_function.is_active()) + assert data_function.last_modified == before + data_function.set_active(False) + assert data_function.last_modified > before + + +@pytest.mark.parametrize( + "orchestrator", [SyncPipelineOrchestrator, AsyncPipelineOrchestrator] +) +@pytest.mark.parametrize("wrapper", ["pod", "data_function"]) +def test_wrapped_function_activation_reaches_node_cache( + data_function, orchestrator, wrapper +) -> None: + """Both supported cache wrappers retain the leaf function's activation revision.""" + from orcapod.core.cached_function_pod import CachedFunctionPod + from orcapod.core.data_function import CachedDataFunction + from orcapod.core.function_pod import FunctionPod + + store = InMemoryArrowDatabase() + original_pod = doubled.pod + assert isinstance(original_pod, FunctionPod) + if wrapper == "pod": + pod = CachedFunctionPod(original_pod, result_database=store) + else: + pod = FunctionPod(CachedDataFunction(data_function, result_database=store)) + data_function.set_active(False) + job = make_job(pod) + job.run(orchestrator=orchestrator()) + assert CALLS == [] + data_function.set_active(True) + job.run(orchestrator=orchestrator()) + assert sorted(CALLS) == [2, 3] + assert job.nodes["total"].as_table().column("total").to_pylist() == [10] diff --git a/tests/test_pipeline/test_retained_storage_views.py b/tests/test_pipeline/test_retained_storage_views.py index cece700e3..c238e0d23 100644 --- a/tests/test_pipeline/test_retained_storage_views.py +++ b/tests/test_pipeline/test_retained_storage_views.py @@ -121,3 +121,109 @@ def test_row_factory_preserves_source_values_with_table_nullability() -> None: from_rows, _ = StaticOutputOperatorPod._rows_to_table([(Tag({"k": 1}), retained)]) from_table, _ = StaticOutputOperatorPod._rows_to_table([(Tag({"k": 1}), decoded)]) assert from_rows.equals(from_table, check_metadata=True) + + +def test_row_factory_preserves_nondefault_context_in_export() -> None: + """Retained rows and context-inclusive exports identify the same decoder.""" + from dataclasses import replace + from orcapod import contexts + from orcapod.system_constants import constants + + context = replace(contexts.get_default_context(), context_key="test:retained:rows") + pairs = [ + (Tag({"k": 1}, data_context=context), Data({"x": 2}, data_context=context)) + ] + stream = ArrowTableStream.from_rows(pairs) + assert stream.as_table(columns={"context": True})[ + constants.CONTEXT_KEY + ].to_pylist() == [context.context_key] + stream.clear_cache() + assert list(stream.iter_data())[0][1].data_context_key == context.context_key + + +def test_row_factory_rejects_provenance_override() -> None: + """A row-preserving factory cannot override the backing table independently.""" + import pytest + + pairs = [(Tag({"k": 1}), Data({"x": 2}, source_info={"x": "original"}))] + with pytest.raises(ValueError, match="source_info"): + ArrowTableStream.from_rows(pairs, source_info={"x": "replacement"}) + + +def test_row_factory_rejects_context_override() -> None: + """A single-context stream cannot silently relabel retained rows.""" + from dataclasses import replace + from orcapod import contexts + import pytest + + context = replace(contexts.get_default_context(), context_key="test:retained:other") + with pytest.raises(ValueError, match="context"): + ArrowTableStream.from_rows(rows(), data_context=context) + + +def test_row_factory_rejects_system_tag_relabeling() -> None: + """Column relabeling belongs before constructing retained canonical rows.""" + import pytest + + with pytest.raises(ValueError, match="system_tag_columns"): + ArrowTableStream.from_rows(rows(), system_tag_columns=["k"]) + + +def test_row_factory_accepts_tagless_nondefault_stream_rows() -> None: + """Empty tags still belong to the row's context when a stream is retained.""" + from dataclasses import replace + import pyarrow as pa + from orcapod import contexts + + context = replace( + contexts.get_default_context(), context_key="test:retained:tagless" + ) + source = ArrowTableStream(pa.table({"x": [1]}), data_context=context) + stream = ArrowTableStream.from_rows(list(source.iter_data())) + assert list(stream.iter_data())[0][0].data_context_key == context.context_key + + +def test_storage_view_preserves_lazily_assigned_uuid() -> None: + """The view shares identity even when no caller has requested the UUID yet.""" + original = Data({"x": 1}) + view = original.storage_view() + assert view.datagram_uuid == original.datagram_uuid + + +def test_row_factory_preserves_unregistered_subclass_context() -> None: + """Canonicalizing a subclass must keep its context object, not resolve its key.""" + from dataclasses import replace + from orcapod import contexts + from orcapod.system_constants import constants + + class CustomTag(Tag): + pass + + class CustomData(Data): + pass + + context = replace( + contexts.get_default_context(), context_key="test:retained:subclass" + ) + for tag_type, data_type in [ + (CustomTag, Data), + (Tag, CustomData), + (CustomTag, CustomData), + ]: + stream = ArrowTableStream.from_rows( + [ + ( + tag_type({"k": 1}, data_context=context), + data_type({"x": 2}, data_context=context), + ) + ] + ) + assert stream.data_context is context + assert stream.as_table(columns={"context": True})[ + constants.CONTEXT_KEY + ].to_pylist() == [context.context_key] + for clear in (False, True): + if clear: + stream.clear_cache() + [(tag, data)] = list(stream.iter_data()) + assert tag.data_context_key == data.data_context_key == context.context_key