Skip to content
94 changes: 94 additions & 0 deletions bench/bench_pipeline_cached_rerun.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
"""Measure work retained by a job, independently of function execution cost.

Run with ``uv run python bench/bench_pipeline_cached_rerun.py --rows 1000``.
Counters, rather than timing thresholds, distinguish reuse from fast recomputation.
"""

from __future__ import annotations

import argparse
import json
import time
from unittest.mock import patch

from orcapod import PipelineJob, function_pod
from orcapod.core.operators.group_by import GroupBy
from orcapod.databases import InMemoryArrowDatabase
from orcapod.hashing.arrow_hashers import StarfixArrowHasher
from orcapod.pipeline import AsyncPipelineOrchestrator
from orcapod.sources import DictSource


def measure(rows: int, grouped: bool, asynchronous: bool) -> None:
"""Print cold and warm run times and counts for one independent job."""
counts = {"functions": 0, "arrow_hashes": 0, "group_by": 0}

@function_pod(output_keys="y")
def increment(x: int) -> int:
counts["functions"] += 1
return x + 1

@function_pod(output_keys="y")
def total(x: list[int]) -> int:
counts["functions"] += 1
return sum(x)

source = DictSource(
data=[{"g": i // 100, "k": i, "x": i} for i in range(rows)],
tag_columns=["g", "k"],
source_id="benchmark",
)
with PipelineJob(name="cache_benchmark", store=InMemoryArrowDatabase()) as job:
if grouped:
total.pod(source.group_by(["g"]).drop_tag_columns(["k"]), label="result")

Check warning on line 43 in bench/bench_pipeline_cached_rerun.py

View workflow job for this annotation

GitHub Actions / typecheck / typecheck

Result of call expression is of type "StreamProtocol" and is not used; assign to variable "_" if this is intentional (reportUnusedCallResult)
else:
increment.pod(source, label="result")

Check warning on line 45 in bench/bench_pipeline_cached_rerun.py

View workflow job for this annotation

GitHub Actions / typecheck / typecheck

Result of call expression is of type "StreamProtocol" and is not used; assign to variable "_" if this is intentional (reportUnusedCallResult)

original_hash = StarfixArrowHasher.hash_table
original_group = GroupBy.unary_static_process

def hash_table(self, *args, **kwargs):

Check warning on line 50 in bench/bench_pipeline_cached_rerun.py

View workflow job for this annotation

GitHub Actions / typecheck / typecheck

Type annotation is missing for parameter "kwargs" (reportMissingParameterType)

Check warning on line 50 in bench/bench_pipeline_cached_rerun.py

View workflow job for this annotation

GitHub Actions / typecheck / typecheck

Type of parameter "kwargs" is unknown (reportUnknownParameterType)

Check warning on line 50 in bench/bench_pipeline_cached_rerun.py

View workflow job for this annotation

GitHub Actions / typecheck / typecheck

Type annotation is missing for parameter "args" (reportMissingParameterType)

Check warning on line 50 in bench/bench_pipeline_cached_rerun.py

View workflow job for this annotation

GitHub Actions / typecheck / typecheck

Type of parameter "args" is unknown (reportUnknownParameterType)

Check warning on line 50 in bench/bench_pipeline_cached_rerun.py

View workflow job for this annotation

GitHub Actions / typecheck / typecheck

Type annotation is missing for parameter "self" (reportMissingParameterType)

Check warning on line 50 in bench/bench_pipeline_cached_rerun.py

View workflow job for this annotation

GitHub Actions / typecheck / typecheck

Type of parameter "self" is unknown (reportUnknownParameterType)
counts["arrow_hashes"] += 1
return original_hash(self, *args, **kwargs)

Check warning on line 52 in bench/bench_pipeline_cached_rerun.py

View workflow job for this annotation

GitHub Actions / typecheck / typecheck

Argument type is unknown   Argument corresponds to parameter "table" in function "hash_table" (reportUnknownArgumentType)

Check warning on line 52 in bench/bench_pipeline_cached_rerun.py

View workflow job for this annotation

GitHub Actions / typecheck / typecheck

Argument type is unknown   Argument corresponds to parameter "self" in function "hash_table" (reportUnknownArgumentType)

def group(self, *args, **kwargs):
counts["group_by"] += 1
return original_group(self, *args, **kwargs)

with (
patch.object(StarfixArrowHasher, "hash_table", hash_table),
patch.object(GroupBy, "unary_static_process", group),
):
for run in ("cold", "warm", "warm_again"):
counts.update(functions=0, arrow_hashes=0, group_by=0)
started = time.perf_counter()
summary = job.run(
orchestrator=AsyncPipelineOrchestrator() if asynchronous else None
)
elapsed = time.perf_counter() - started
if not summary.ok:
raise RuntimeError(str(summary))
print(
json.dumps(
{
"rows": rows,
"grouped": grouped,
"async": asynchronous,
"run": run,
"seconds": elapsed,
**counts,
}
),
flush=True,
)


if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--rows", type=int, default=1000)
options = parser.parse_args()
if options.rows < 1:
parser.error("--rows must be positive")
for is_grouped in (False, True):
for is_async in (False, True):
measure(options.rows, is_grouped, is_async)
18 changes: 18 additions & 0 deletions changelog.d/ITL-1063.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
### Fixed

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

F5. Everything here is under ### Fixed. The new API (PipelineJob.clear_cache(include_sources=...), SpiralDBTableSource(retain_snapshot=...), SpiralDBTableSource.clear_cache()) belongs under ### Added. Retaining pinned Spiral snapshots by default changes memory behaviour, so it belongs under ### Changed, which is where a user who relies on bounded streaming will look.

Posted by a Claude Code agent on behalf of @eywalker.


- Repeated pipeline runs reuse completed operator work and retained row hashes when
their input snapshots are unchanged. Sync execution preserves row objects across
node boundaries; async execution reuses valid in-memory function results. Unknown
or changing inputs, explicit invalidation and failed work remain subject to normal
execution and recovery.
- A SpiralDB source retains a successfully read pinned snapshot for later sync,
async and table reads. Construction remains lazy; partial, failed and cancelled
reads do not publish a snapshot. Retention uses memory proportional to the snapshot;
`batch_rows` bounds processing slices, not total retained memory. Construct a new
source to read a newer table version.
- Unchanged job reruns preserve node modification times so external lazy readers
remain current. Source nodes follow their bound source's freshness. Node-owned
stored row views preserve declared types without changing live Python objects.
- `PipelineJob.clear_cache()` releases retained working sets without deleting stored
results. Spiral sources also expose `clear_cache()` and a `retain_snapshot=False`
streaming opt-out; clearing during a scan prevents that scan republishing a cache.
31 changes: 31 additions & 0 deletions design_issues/ITL-1063-CR1.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
## `src/orcapod/pipeline/` — completed work on unchanged reruns

### CR1 — Job reruns rebuild rows and hashes despite retained function results
**Status:** resolved
**Severity:** high
**Issue:** ITL-1063

An unchanged job visits every node again. The sync orchestrator materializes retained
`(Tag, Data)` objects as a new Arrow stream, losing their memoized hashes. Operator
execution constructs a new stream despite holding a previous output. Async function
execution reloads stored results on every run. SpiralDB sources pin a table version but
rescan it and rebuild provenance on each read.

Measured at `89160c47`, a 1,000-row flat job's warm sync run performs 2,000 Arrow hashes
and its async run 1,000, with no user-function calls. Grouping the source into 10 rows still
reruns GroupBy and performs 20 hashes on both orchestrators. A retained standalone stream
read reuses its objects and does not repeat this work.

Approved design: `superpowers/specs/2026-11-13-pipeline-cache-reuse-design.md`.
Reproduction: `uv run python bench/bench_pipeline_cached_rerun.py --rows 1000`.

Completed graph execution now retains operator results and exact-row entry IDs against
known unchanged snapshots. Sync boundaries retain Arrow-decoded row views to preserve
declared-schema behavior; async functions reuse their valid in-memory results. Pinned
Spiral sources publish retained batch streams only after complete successful reads.
Unknown sources, direct node calls, changed bindings and failed work remain conservative.

Validated with 11,625 passing tests (119 skipped, 19 expected failures), zero type-check
errors, clean Ruff checks and independent review. Warm benchmark runs perform zero new
Arrow hashes and zero GroupBy computations. Remaining traversal/materialization costs
and potential further optimizations are tracked by ITL-1064.
35 changes: 29 additions & 6 deletions docs/concepts/sources.md
Original file line number Diff line number Diff line change
Expand Up @@ -115,14 +115,35 @@ Things to know about Spiral tables:
Building a `SpiralDBTableSource` reads no rows. It reads the table's schema, key columns and
table id, which takes about a second whatever the table's size. `output_schema()`, `keys()`,
`content_hash()` and `pipeline_hash()` read no rows either. Rows are read when you ask for
them, and every read scans the table again:
them. The first complete read of a pinned version is retained for later reads:

- `iter_data()` and a pipeline job hand the rows on in batches of at most `batch_rows`
rows (default 8,192). SpiralDB sends a scan in batches that follow its stored fragments,
whose sizes cannot be chosen; one of them is held while its rows are handed on.
whose sizes cannot be chosen. Completed batch streams are retained in memory, and
their row objects and hashes are reused by later sync and async reads.
- `as_table()`, `as_df()`, `as_polars_df()`, `as_pandas_df()` and `as_lazy_frame()` read
the rows into memory and return them all.

Retention uses memory proportional to the full snapshot, even with `iter_data()`;
`batch_rows` bounds processing slices, not retained memory. Row objects are built only
when iteration needs them, so a table-only read retains Arrow streams without eagerly
constructing all row objects. Failed, cancelled, or partially consumed scans do not
publish a snapshot. Concurrent initial readers may scan independently; the first
complete read atomically publishes the retained snapshot. When no table version can
be pinned, each read scans again rather than reusing potentially stale rows.

For streaming-only reads, pass `retain_snapshot=False`. Each iteration then scans again
and releases consumed batch streams, retaining only the current SpiralDB batch and its
processing slice. SpiralDB controls the underlying batch size; `batch_rows` bounds the
slice size. Whole-table methods such as `as_table()` still materialize their returned
rows, and pipeline jobs can retain their own outputs. The option is saved in source
configuration and defaults to `True`.

Call `source.clear_cache()` to release a retained snapshot. This does not change the
pinned `asof` version. Existing readers can finish with the data they already hold, but
a scan started before clearing cannot republish its snapshot. A later full read can
retain the same version again. Construct a new source to select a newer table version.

To read part of a table, say so when you build the source. SpiralDB applies both
restrictions before any row is transferred:

Expand All @@ -148,8 +169,9 @@ SpiralDB records for the table. That count can include deleted rows that are not
compacted, so a table near the cap can be refused although a scan would return fewer rows;
the error says so. The cap is there
because a pipeline job keeps every row it processes in memory, about 5 KB per row for each
node, so 100,000 rows cost about 0.5 GB per node. A table of millions of rows can be iterated
with `max_rows=None`, but it cannot yet be run through a job on a laptop.
node, so 100,000 rows cost about 0.5 GB per node. Removing the cap with `max_rows=None`
also permits a source to retain a snapshot of any size; it does not provide bounded-memory
iteration.

**Looking at a table in a notebook.** A Spiral source shown at the end of a notebook cell
(its `head()`, see [Looking at data](streams.md#looking-at-data)) reads only its first 10
Expand All @@ -163,6 +185,7 @@ by tags` when the source reads no more than that, or `10 matching rows read, sor
(where unit_id=106); the table has about 63,349 rows` with `where=`. The unsorted `head()`
says `first 10 rows of about 63,349`. Tag columns come first, and the line above the table
names them. To see more rows, use `as_df()` with a `limit=` or `where=` source.
After a complete snapshot has been retained, previews use that snapshot and report its exact row count without another scan.

**Restrictions are not part of identity.** `where`, `limit` and `asof` choose rows of the
same table, so two sources on one table have the same `source_id` and `content_hash`
Expand All @@ -172,8 +195,8 @@ job, and `as_table()` after an async run) returns every row that any job with th
pipeline has read from that table, not only the rows of the latest restriction. The rows
sent downstream during a run are only the rows that run read.

**Running again.** A second run scans the table again and computes nothing for the rows it
has already seen. A source object keeps reading the version its table had when it was
**Running again.** A second read reuses the completed pinned snapshot and its row
objects; a second run computes nothing for the rows it has already seen. A source object keeps reading the version its table had when it was
built, so rows added to the Spiral table afterwards are seen only by a new source: build
the source (and the job around it) again, and the added rows are new keys, the only rows
computed.
Expand Down
39 changes: 39 additions & 0 deletions docs/concepts/streams.md
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,45 @@ print(table.to_pandas())
# 2 mouse_03 15 27.8
```

## Reusing completed pipeline work

An unchanged `PipelineJob` reuses completed work when its nodes are not stale and
their inputs are supported retained snapshots. The completion record proves that a
whole execution succeeded with those graph inputs; it does not replace `is_stale`.
A reused result keeps its modification time, so lazy streams over that node remain
current. Rebinding a source, clearing a cache, changing the producer, or observing a
newer upstream invalidates reuse. Unknown freshness stays conservative.

The supported concrete source classes are `ArrowTableStream`, `ArrowTableSource`,
`DictSource`, `ListSource`, and `SpiralDBTableSource` with a pinned version and
`retain_snapshot=True`. Subclasses and other sources, including `CSVSource`,
`DataFrameSource`, `DeltaTableSource`, `CachedSource`, `DerivedSource` and polling
sources, do not currently get completed-work reuse. Their ordinary per-row result
cache still works. One info-level log per run lists source bindings that disable the
fast path. A Delta table's version is not yet an execution snapshot capability; it
must be integrated explicitly before it can safely qualify.

Reuse has a memory cost beyond the source snapshot. Nodes retain entry IDs and row
views for their latest buffers, in addition to output caches. A boundary may retain
both live Python values and an Arrow-decoded view of the declared stored type; Arrow
buffers are shared. The row-view mapping can hold two entries per input pair, and a
function node can retain an entry ID per pair. Memory therefore grows with processed
rows and graph width; `materialize_results=False` only releases orchestrator result
buffers, not these reusable node caches.

Call `job.clear_cache()` between executions to release node caches and clear bound
sources that support cache release. Persistent database results, original source data
and pinned versions are preserved. `job.clear_cache(include_sources=False)` keeps the
source snapshots. Clearing a shared source also releases that source's retained rows
for its other consumers. A later run can recover persisted function results but must
rebuild cleared rows and operator outputs.

For bounded source iteration, a Spiral source can use `retain_snapshot=False`; this
also disables completed-work reuse downstream. See [Sources](sources.md) for the
snapshot lifetime and streaming limits. Warm job execution still traverses rows and
callbacks, and sync execution still materializes tables; reuse is not a constant-time
whole-job shortcut or a cache of new jobs across processes.

## How it connects to other concepts

- [Sources](sources.md) produce streams from external data
Expand Down
16 changes: 15 additions & 1 deletion src/orcapod/core/data_function.py
Original file line number Diff line number Diff line change
Expand Up @@ -420,6 +420,7 @@ def __init__(
executor = LocalPythonFunctionExecutor()

self._function = function
self._activation_generation = 0
self._is_async = inspect.iscoroutinefunction(function)
# Reject functions with variadic parameters -- PythonDataFunction maps
# data keys to named parameters, so the full parameter set must be fixed.
Expand Down Expand Up @@ -591,8 +592,16 @@ def is_active(self) -> bool:
return self._active

def set_active(self, active: bool = True) -> None:
"""Set the active state. If False, ``call`` returns None for every data."""
"""Set the active state; disabled calls return None for every input.

Actual changes tell job nodes to revisit their filtered completions on
the next run. Repeating the same state preserves current results.
"""
if active == self._active:
return
self._active = active
self._activation_generation += 1
self._update_modified_time()

@property
def is_async(self) -> bool:
Expand Down Expand Up @@ -960,6 +969,11 @@ def __init__(self, data_function: DataFunctionProtocol, **kwargs) -> None:
kwargs["data_context"] = getattr(data_function, "data_context", None)
super().__init__(**kwargs)

@property
def _activation_generation(self) -> int:
"""Propagate the wrapped function's activation changes through cache wrappers."""
return getattr(self._data_function, "_activation_generation", 0)

def computed_label(self) -> str | None:
return self._data_function.label

Expand Down
32 changes: 32 additions & 0 deletions src/orcapod/core/datagrams/datagram.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,8 @@ class Datagram(ContentIdentifiableBase):
# Construction
# ------------------------------------------------------------------

_stored_row_origin: Datagram | None = None

def __init__(
self,
data: Mapping[str, DataValue] | pa.Table | pa.RecordBatch,
Expand Down Expand Up @@ -1026,6 +1028,36 @@ def with_columns(
# 8. Utility Operations
# ------------------------------------------------------------------

@property
def storage_origin(self) -> Datagram:
"""Original row whose exact stored representation this view shares."""
return self if self._stored_row_origin is None else self._stored_row_origin

def storage_view(self, *, include_meta: bool = True) -> Self:
"""Expose Arrow-decoded values without modifying the live Python values.

A declared base type can store fewer fields than its live subclass. This
view uses that stored type, shares immutable Arrow buffers and hashes,
and preserves provenance. Callers own retention of the returned view.
Metadata may be omitted to match a materialization boundary's columns.
"""
self._ensure_data_table()
if (self._data_dict is None or self.storage_origin is not self) and (
include_meta or not self._meta
):
return self
# copy() carries the cached UUID; establish it before preserving identity.
_ = self.datagram_uuid
view = self.copy(preserve_id=True)
view._data_dict = None
view._data_python_schema = None
view._stored_row_origin = self.storage_origin
if not include_meta:
view._meta = {}
view._meta_python_schema = Schema()
view._meta_table = None
return view

def copy(self, include_cache: bool = True, preserve_id: bool = True) -> Self:
"""Return a shallow copy of this datagram.

Expand Down
Loading
Loading