From 012c5a1ae3093202400855b7af2c448f7ace6c24 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 23:03:29 -0800 Subject: [PATCH 01/12] test(nodes): node staleness and head after sync and async runs (ITL-1042) WIP: the async cases fail until the fix. Co-Authored-By: Claude Opus 5.5 --- .../2026-11-12-itl-1042-head-after-async.md | 145 +++++++++ .../test_node_staleness_after_run.py | 291 ++++++++++++++++++ 2 files changed, 436 insertions(+) create mode 100644 superpowers/plans/2026-11-12-itl-1042-head-after-async.md create mode 100644 tests/test_pipeline/test_node_staleness_after_run.py diff --git a/superpowers/plans/2026-11-12-itl-1042-head-after-async.md b/superpowers/plans/2026-11-12-itl-1042-head-after-async.md new file mode 100644 index 000000000..ca8cd6bf5 --- /dev/null +++ b/superpowers/plans/2026-11-12-itl-1042-head-after-async.md @@ -0,0 +1,145 @@ +# ITL-1042: node head after an async run — implementation plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:executing-plans to implement this plan +> task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** After a completed run (sync or async), no job node is stale, and a node's `head()`, +`as_table()` and `iter_data()` agree that its results are there. A head with no rows has the +same columns as a head with rows, for every column configuration. + +**Architecture:** One rule for a node's modified time, followed by every job node on both +paths: a node stamps its modified time when its run completes, after it has consumed all its +inputs and **before** it signals completion downstream (closes its output). Then every node is +stamped no earlier than its upstreams, so nothing is stale after the run. + +**Tech stack:** Python, asyncio, PyArrow, pytest (`uv run`). + +--- + +## Evidence (established before any code change) + +Repro: the ITL-1034 suite (PR #346 head `ed333412`, copied to the scratchpad and run against +this branch) fails `test_every_row_of_a_node_view_is_aligned[unique-shuffled-async_run-natural-in_memory-joined.head]`: +`joined.head` has no rows. In isolation `joined.head` passes; it fails after the earlier views +on the same job (module-scoped run fixture). + +Modified times (microseconds) and staleness (`*`) after an async run of the #346 job: + +``` +after run B:135099 A:135112 SelectDataColumns:184056 scaled:136321* joined:202799 grouped:197372 +scaled.as_table B:135099 A:135112 SelectDataColumns:184056 scaled:276357 joined:202799* grouped:197372* +``` + +The same job after a sync run: no node is stale before or after any view. + +- `A` has a data column `scaled` does not take, so `pod(A)` adds an automatic + `SelectDataColumns` (ITL-807), an `OperatorJobNode` upstream of `scaled`. +- `OperatorJobNode.async_execute` stamps itself when it finishes + (`core/nodes/operator_node.py:1613`, `_update_modified_time()`). +- `FunctionJobNode.async_execute` never stamps (`core/nodes/function_node.py:3200-3487`); the + sync `execute` does (`function_node.py:1937`). So `scaled` keeps its construction time, + which is older than its upstream's: `scaled.is_stale` is True right after the run (lead (a), + confirmed). +- The first read of `scaled` (`iter_data`/`as_table`, `function_node.py:3151-3152`) sees it + stale, calls `clear_cache()`, which stamps the node (`function_node.py:512`) and re-reads + the store. Now `scaled` is newer than `joined` and `grouped`, which become stale, and + `OperatorJobNode.head` reports `NOT_COMPUTED` / "inputs changed" (`operator_node.py:1337-1341`). +- Lead (b) (the upstream bumped per forwarded row) is wrong: `record_and_forward` + (`function_node.py:3348-3390`) clears `_cached_output_table` but never stamps. + +Without the auto-select (a source with only the function's input) nothing is stale after an +async run, because the source node never stamps. So the defect needs a stamped upstream: an +operator node before a function node. + +## Cost measured today (before the fix) + +Script: count `scale` calls and `InMemoryArrowDatabase` reads (`get_all_records`, +`get_records_with_column_value`, `get_records_by_ids`, `get_record_by_id`). + +| After a run | sync | async | +|---|---|---| +| `scaled.head()` | 0 reads, ROWS | 2 `get_all_records` (whole pipeline + result tables), ROWS | +| `scaled.as_table()` | 0 reads | 2 `get_all_records`, in-memory results thrown away and reloaded | +| `joined.head()` / `grouped.head()` | ROWS | NOT_COMPUTED, 0 rows (after a `scaled` read) | +| `joined.as_table()` | 3 rows | 3 rows (no staleness check there) | +| function calls in any read / second run | 0 | 0 | + +A second async run reads the store (2 `get_all_records`) on both paths' async entry +(`async_execute` always loads cached entries); that is not caused by staleness. + +## Rule + +A job node's modified time is stamped once per completed run, after it has consumed all its +inputs and before it closes its output (signals completion downstream), on the sync and the +async path. A failed run does not stamp. Consequences: after a completed run no node is +stale; `head()`, `as_table()` and `iter_data()` serve the run's in-memory results without +reading the store. + +## Files + +- Modify `src/orcapod/core/nodes/function_node.py`: `FunctionJobNode.async_execute` stamps + after a completed run, before `output` is closed. The no-database branch hands the pod a + writer whose `close()` is deferred, so the stamp still precedes the close. +- Modify `src/orcapod/side_effects.py`: `SideEffectJobNode.execute` / `async_execute` stamp + likewise, so the rule holds for every job node (they keep no rows; staleness only). +- Create `tests/test_pipeline/test_node_staleness_after_run.py`. +- Item 4 (pending coordinator answer, option A recommended): rowless heads get the row-only + column groups (`_source_*`, `_context_key`, content hash) in each kind's own column order. +- Create `design_issues/ITL-1042-.md`, `changelog.d/ITL-1042.md`. + +## Tasks + +### Task 1: failing tests (public interface) + +- [ ] Write `tests/test_pipeline/test_node_staleness_after_run.py`: + job = source A (`id`, `g` tags; `w`, `x` data, rows not in tag order) → `scaled` + (function pod on `x`, auto-select drops `w`) → `joined` = `Join(scaled, B)` and + `grouped` = `GroupBy(by=["g"])(scaled)`. Parametrize orchestrator (sync, async) × store + (in_memory, delta). + - `test_no_node_is_stale_after_a_run`: every node with `is_stale` is not stale. + - `test_heads_show_rows_after_reading_upstream`: read `scaled.as_table()`, then every + node's `head(100)` is ROWS with as many rows as its `as_table()` and `iter_data()`. + - `test_reads_after_a_run_do_not_touch_the_store`: after the run, `head`, `as_table` and + `iter_data` of every node make zero store reads (patch the store class's read methods + with counting wrappers) and zero function calls. + - Async completion order forced without sleeps: an `async def` function pod gated by + `asyncio.Event`s so rows complete in reverse input order; same assertions. +- [ ] Run: `uv run pytest tests/test_pipeline/test_node_staleness_after_run.py -q`; expect the + async cases to FAIL and the sync cases to pass. +- [ ] Commit (WIP) `test(nodes): node staleness and head after sync and async runs (ITL-1042)`. + +### Task 2: stamp on the async path + +- [ ] In `FunctionJobNode.async_execute`, after `self._commit_at_node_end()` (DB branch) and + after the pod returns (no-DB branch), call `self._update_modified_time()` before + `on_node_end`; in the no-DB branch pass the pod a deferred-close writer so the stamp + precedes the close (the outer `finally` closes `output`). +- [ ] `SideEffectJobNode.execute` stamps after its loop; `async_execute` stamps after the + TaskGroup, before the `finally` close. +- [ ] Run the new tests: PASS. Run `tests/test_pipeline tests/test_core/function_pod tests/test_core/operators -q`. +- [ ] Run the #346 suite copy against the branch: `joined.head`/`grouped.head` pass. +- [ ] Commit `fix(nodes): stamp a node's modified time when its async run completes (ITL-1042)`. + +### Task 3: rowless head columns — coordinator decision: option C (Nov 12, 2026) + +Do not change rowless heads. Document in the `head()` docstring (`StreamBase.head`) and the +docs that a head with no rows shows the declared columns only (no `_source_*`, +`_context_key` or content-hash columns): their presence and, for `_source_*`, their type are +known only from rows. The coordinator records the finding (37 mismatches over 8 configs x 7 +stream kinds) on ITL-985 and ITL-1002. Save the measurement script as +`superpowers/specs/itl-1042/rowless_head_columns.py`. + +- [ ] Docstring + docs page; script saved; commit. + +### Task 4: finish + +- [ ] design_issues entry, changelog fragment, docs if a page states the old behaviour. +- [ ] ruff check/format, basedpyright exit 0, full suite on 3.11/3.12/3.13 (≤ 2 processes). +- [ ] Merge origin/main, push once, PR (Fixes ITL-1042; note #346 must drop its ITL-1042 + xfail), CodeRabbit, signed replies, worker_done. + +## Notes for a successor + +- ITL-1045 is a different defect (two passes in `as_table` with a genuinely stale upstream); + this one is a false staleness after an async run. +- Scratchpad scripts: `repro.py`, `cost.py`, `emptyhead.py`, `a346/drive.py`. diff --git a/tests/test_pipeline/test_node_staleness_after_run.py b/tests/test_pipeline/test_node_staleness_after_run.py new file mode 100644 index 000000000..2a9b97b8d --- /dev/null +++ b/tests/test_pipeline/test_node_staleness_after_run.py @@ -0,0 +1,291 @@ +"""After a completed run no job node is stale, on either orchestrator (ITL-1042). + +A job node stamps its modified time when its run completes, after it has read all its inputs +and before it tells its downstream nodes it is done. So after ``job.run()``, sync or async, +every node is at least as new as its upstreams: ``head()`` shows the rows ``as_table()`` and +``iter_data()`` return, and none of them reads the store again. + +The job: source ``A`` (tags ``id``, ``g``; data ``w``, ``x``; rows not in tag order) feeds the +function pod ``scaled``, which takes only ``x``, so calling it adds an automatic +``SelectDataColumns`` node before it (ITL-807). ``scaled`` feeds ``joined`` (``Join`` with +source ``B``) and ``grouped`` (``GroupBy`` on ``g``). +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Callable, Iterator +from pathlib import Path +from typing import Any +from unittest import mock + +import pyarrow as pa +import pytest + +from orcapod import function_pod +from orcapod.core.operators import GroupBy, Join +from orcapod.core.sources import ArrowTableSource +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator +from orcapod.pipeline.job import PipelineJob +from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator +from orcapod.protocols.database_protocols import ArrowDatabaseProtocol +from orcapod.streams import HeadStatus +from orcapod.types import CacheMode + +# The input order of A's rows (not in tag order), and the order the async rows complete +# in when the gate is armed: the reverse. +A_IDS = [3, 1, 4, 0, 2] +COMPLETION_ORDER = list(reversed(A_IDS)) + +ORCHESTRATORS = { + "sync": SyncPipelineOrchestrator, + "async": AsyncPipelineOrchestrator, +} + +# The read methods of the store protocol: a read after a run is counted through them. +STORE_READS = ( + "get_all_records", + "get_records_with_column_value", + "get_records_by_ids", + "get_record_by_id", +) + +CALLS: list[int] = [] + + +class _ReverseGate: + """Make the gated rows of an async run complete in ``COMPLETION_ORDER``, without sleeps. + + The row ``x`` returns only after the row before it in the order has returned. Disarmed, + rows return at once: a sync run calls them one by one, so an armed gate would wait forever. + """ + + def __init__(self) -> None: + """Build a disarmed gate.""" + self.armed = False + self.completed: list[int] = [] + self._events: dict[int, asyncio.Event] = {} + + def arm(self) -> None: + """Arm the gate for the next run and forget earlier completions.""" + self.armed = True + self.completed = [] + self._events = {} + + def disarm(self) -> None: + """Let rows return as the scheduler runs them.""" + self.armed = False + self._events = {} + + def _event(self, key: int) -> asyncio.Event: + """Return the event set when row ``key`` returns.""" + return self._events.setdefault(key, asyncio.Event()) + + async def wait_turn(self, key: int) -> None: + """Return once the row before ``key`` in the order has returned. + + The timeout is a deadlock guard only; a passing run never reaches it. + """ + if not self.armed: + return + position = COMPLETION_ORDER.index(key) + if position: + await asyncio.wait_for( + self._event(COMPLETION_ORDER[position - 1]).wait(), timeout=30 + ) + + def finish(self, key: int) -> None: + """Record that row ``key`` returned and release the next one.""" + self.completed.append(key) + if self.armed: + self._event(key).set() + + +GATE = _ReverseGate() + + +@function_pod(output_keys="y") +async def scale(id_: int, x: int) -> int: + """Return ``10 * x``, completing in the gate's order; counts every call.""" + await GATE.wait_turn(id_) + CALLS.append(x) + GATE.finish(id_) + return 10 * x + + +@pytest.fixture(autouse=True) +def _reset() -> Iterator[None]: + """Clear the call record and leave the gate disarmed around each test.""" + CALLS.clear() + GATE.disarm() + yield + GATE.disarm() + + +def _table(columns: dict[str, list[int]]) -> pa.Table: + """A table of non-nullable ``int64`` columns.""" + return pa.table( + columns, + schema=pa.schema([pa.field(name, pa.int64(), nullable=False) for name in columns]), + ) + + +def _store(kind: str, tmp_path: Path) -> ArrowDatabaseProtocol: + """A fresh store of the given kind.""" + if kind == "in_memory": + return InMemoryArrowDatabase() + return DeltaTableDatabase(base_path=tmp_path / "store") + + +def _job(store: ArrowDatabaseProtocol) -> PipelineJob: + """The job of the module docstring, ``joined`` cached in the store (LOG).""" + a = ArrowTableSource( + _table( + { + "id": A_IDS, + "g": [i % 2 for i in A_IDS], + "id_": A_IDS, + "w": [0] * len(A_IDS), + "x": [i + 1 for i in A_IDS], + } + ), + tag_columns=["id", "g"], + source_id="a", + label="A", + ) + b = ArrowTableSource( + _table({"id": [2, 0, 4, 1, 3], "z": [20, 0, 40, 10, 30]}), + tag_columns=["id"], + source_id="b", + label="B", + ) + job = PipelineJob(name="itl1042", store=store) + with job: + scaled = scale.pod(a, label="scaled") + Join()(scaled, b, label="joined") + GroupBy(by=["g"])(scaled, label="grouped") + # OperatorNode.attach_databases is the public way to store an operator's rows. + job.nodes["joined"].attach_databases( + pipeline_database=store.at("itl1042"), cache_mode=CacheMode.LOG + ) + return job + + +def _run(job: PipelineJob, orchestrator: str, *, gated: bool) -> None: + """Run ``job``; on the async orchestrator with ``gated``, rows complete in reverse.""" + if gated: + GATE.arm() + try: + job.run(orchestrator=ORCHESTRATORS[orchestrator]()) + finally: + GATE.disarm() + + +def _computed_nodes(job: PipelineJob) -> dict[str, Any]: + """The job's nodes that compute and keep results (every node with ``is_stale``).""" + return { + label: node for label, node in job.nodes.items() if hasattr(node, "is_stale") + } + + +def _count_store_reads(store: ArrowDatabaseProtocol) -> tuple[list[str], Any]: + """A record of store reads and the patcher that fills it (every view of the store).""" + reads: list[str] = [] + store_class = type(store) + + def counting(name: str) -> Callable[..., Any]: + original = getattr(store_class, name) + + def read(self: Any, *args: Any, **kwargs: Any) -> Any: + reads.append(name) + return original(self, *args, **kwargs) + + return read + + return reads, mock.patch.multiple( + store_class, **{name: counting(name) for name in STORE_READS} + ) + + +RUNS = [ + pytest.param(orchestrator, store_kind, gated, id=f"{orchestrator}-{store_kind}{tag}") + for store_kind in ("in_memory", "delta") + for orchestrator, gated, tag in ( + ("sync", False, ""), + ("async", False, ""), + ("async", True, "-reversed"), + ) +] + + +@pytest.mark.parametrize(("orchestrator", "store_kind", "gated"), RUNS) +def test_no_node_is_stale_after_a_run( + orchestrator: str, store_kind: str, gated: bool, tmp_path: Path +) -> None: + """Right after a run every node is at least as new as its upstreams.""" + job = _job(_store(store_kind, tmp_path)) + _run(job, orchestrator, gated=gated) + if gated: + assert GATE.completed == COMPLETION_ORDER + nodes = _computed_nodes(job) + assert {"SelectDataColumns", "scaled", "joined", "grouped"} <= set(nodes) + assert {label: node.is_stale for label, node in nodes.items()} == { + label: False for label in nodes + } + + +@pytest.mark.parametrize(("orchestrator", "store_kind", "gated"), RUNS) +def test_heads_show_the_rows_after_the_function_node_is_read( + orchestrator: str, store_kind: str, gated: bool, tmp_path: Path +) -> None: + """Reading ``scaled`` first does not make ``joined`` or ``grouped`` look stale.""" + job = _job(_store(store_kind, tmp_path)) + _run(job, orchestrator, gated=gated) + _ = job.nodes["scaled"].as_table() + _ = list(job.nodes["scaled"].iter_data()) + for label, node in _computed_nodes(job).items(): + head = node.head(100) + table = node.as_table() + assert head.status is HeadStatus.ROWS, (label, head.header()) + assert head.table.num_rows == table.num_rows == len(list(node.iter_data())) + assert head.table.equals(table), label + assert not node.is_stale, label + assert job.nodes["scaled"].as_table().num_rows == len(A_IDS) + assert job.nodes["joined"].as_table().num_rows == len(A_IDS) + assert job.nodes["grouped"].as_table().num_rows == 2 + + +@pytest.mark.parametrize(("orchestrator", "store_kind", "gated"), RUNS) +def test_reads_after_a_run_compute_nothing_and_read_no_store( + orchestrator: str, store_kind: str, gated: bool, tmp_path: Path +) -> None: + """After a run, every node's head, table and rows come from memory.""" + store = _store(store_kind, tmp_path) + job = _job(store) + _run(job, orchestrator, gated=gated) + CALLS.clear() + reads, patcher = _count_store_reads(store) + with patcher: + for node in _computed_nodes(job).values(): + _ = node.head(100) + _ = node.as_table(all_info=True) + _ = list(node.iter_data()) + assert reads == [] + assert CALLS == [] + + +@pytest.mark.parametrize(("orchestrator", "store_kind", "gated"), RUNS) +def test_a_second_run_leaves_no_node_stale( + orchestrator: str, store_kind: str, gated: bool, tmp_path: Path +) -> None: + """A second run of the same job computes nothing again and leaves nothing stale.""" + job = _job(_store(store_kind, tmp_path)) + _run(job, orchestrator, gated=gated) + _ = job.nodes["scaled"].as_table() + CALLS.clear() + _run(job, orchestrator, gated=False) + assert CALLS == [] + for label, node in _computed_nodes(job).items(): + assert not node.is_stale, label + assert node.head(100).status is HeadStatus.ROWS, label From 6dc70a5094fd4e6b65b6f66be581ae1988cb7e4c Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 23:09:42 -0800 Subject: [PATCH 02/12] fix(function_node): stamp a function node's modified time when its async run completes (ITL-1042) FunctionJobNode.async_execute never stamped the node, while the sync execute does. An operator node upstream (the automatic SelectDataColumns of a pod call) stamps itself when it finishes, so after an async run the function node looked stale: its first read threw away the in-memory results, re-read the store and stamped the node, after which every operator node downstream reported its head as not computed. The node now stamps once its run completes, before it closes its output, so each downstream node is newer than it. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/function_node.py | 34 ++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 8fda8cf78..122811f4f 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -223,6 +223,26 @@ def _executor_supports_concurrent( return executor is not None and executor.supports_concurrent_execution +class _CloseDeferredWriter: + """A writer that forwards rows to ``output`` and leaves closing it to its owner. + + A node hands it to a pod that closes the writer it is given, when the node must + still do something (stamp its modified time) before its downstream nodes see the + end. The node closes ``output`` itself afterwards. + """ + + def __init__(self, output: WritableChannel[tuple[TagProtocol, DataProtocol]]): + """Wrap ``output``.""" + self._output = output + + async def send(self, item: tuple[TagProtocol, DataProtocol]) -> None: + """Send ``item`` to the wrapped output.""" + await self._output.send(item) + + async def close(self) -> None: + """Do nothing: the owner closes the wrapped output.""" + + # --------------------------------------------------------------------------- # FunctionNodeBase — shared base (no DB) # --------------------------------------------------------------------------- @@ -3493,13 +3513,21 @@ def create_data_logger(self, tag: Any, data: Any) -> Any: # Commit the last rows inside a job (ITL-919); a no-op otherwise. self._commit_at_node_end() else: - # No-DB path: delegate dispatch entirely to the pod. - # The pod's async_execute() closes `output` in its own finally. + # No-DB path: delegate dispatch entirely to the pod. The pod closes + # the writer it is given; that close waits for the finally below, so + # the node is stamped before its downstream nodes see the end. await self._function_pod.async_execute( - [input_channel], output, observer=ctx_obs, run_id=run_id + [input_channel], + _CloseDeferredWriter(output), + observer=ctx_obs, + run_id=run_id, ) ctx_obs.on_node_end(node_label, node_hash) + # The run is complete: stamp the node as the sync execute() does, before + # the finally closes the output, so every downstream node, which stamps + # after it has seen that close, is newer than this one (ITL-1042). + self._update_modified_time() except Exception as run_error: self._commit_after_failure(run_error) raise From c7cfc3cc44791f9f761b02bc2c01d308a61ecc66 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 23:14:09 -0800 Subject: [PATCH 03/12] docs(streams): a rowless head shows the declared columns; nodes after an async run (ITL-1042) Adds the design_issues entry, the changelog fragment and the rowless-head column measurement kept for ITL-985 / ITL-1002, and makes the staleness tests basedpyright-clean. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1042.md | 12 ++ design_issues/ITL-1042-NS1.md | 35 ++++ docs/concepts/streams.md | 9 +- src/orcapod/core/streams/base.py | 5 +- .../specs/itl-1042/rowless_head_columns.py | 152 ++++++++++++++++++ .../test_node_staleness_after_run.py | 30 ++-- 6 files changed, 230 insertions(+), 13 deletions(-) create mode 100644 changelog.d/ITL-1042.md create mode 100644 design_issues/ITL-1042-NS1.md create mode 100644 superpowers/specs/itl-1042/rowless_head_columns.py diff --git a/changelog.d/ITL-1042.md b/changelog.d/ITL-1042.md new file mode 100644 index 000000000..11c707e5c --- /dev/null +++ b/changelog.d/ITL-1042.md @@ -0,0 +1,12 @@ +### Fixed + +#### A node's preview shows its rows after an async run (ITL-1042) + +After a job ran on `AsyncPipelineOrchestrator`, a function node downstream of an operator +node (for example the automatic column selection a pod call adds) was taken to be out of +date. Its first `as_table()`, `iter_data()` or `head()` read the store again instead of the +results in memory, and from then on every operator node downstream of it showed +`head()` as "not computed: inputs changed since the last computation", with no rows, although +`as_table()` and `iter_data()` returned every row. After a run on either orchestrator no node +is now out of date: `head()` shows the rows `as_table()` returns, and none of them reads the +store. Nothing was ever computed twice. (ITL-1042) diff --git a/design_issues/ITL-1042-NS1.md b/design_issues/ITL-1042-NS1.md new file mode 100644 index 000000000..ecbdf505b --- /dev/null +++ b/design_issues/ITL-1042-NS1.md @@ -0,0 +1,35 @@ +## `src/orcapod/core/nodes/function_node.py` + +### NS1 — After an async run a function node looked stale, so downstream heads said "not computed" +**Status:** resolved +**Severity:** high +**Issue:** ITL-1042 + +A node is stale when an upstream (or its pod) has a newer modified time than its own. The sync +`FunctionJobNode.execute` stamps the node when its run completes; `async_execute` never did. +An `OperatorJobNode` stamps itself when its async run finishes, so a function node downstream +of an operator node (most often the automatic `SelectDataColumns` that `pod(stream)` adds when +the stream has columns the function does not take, ITL-807) was stale right after an async +run. Its first `iter_data()` / `as_table()` then threw away the in-memory results, re-read the +node's whole stored table (two `get_all_records` calls) and stamped the node. That made every +operator node downstream of it stale, and their `head()` reported `not_computed` ("inputs +changed since the last computation") with no rows, although `as_table()` and `iter_data()` +still returned every row. `head()` of the function node itself read the store instead of +memory. No function was called again and no operator recomputed. + +**Rule:** a job node stamps its modified time once per completed run, after it has read all +its inputs and before it closes its output, on both paths; a failed run does not stamp. Every +node then ends a run at least as new as its upstreams, so nothing is stale after a completed +run, and `head()`, `as_table()` and `iter_data()` serve the run's results from memory. + +**Fix:** `async_execute` stamps the node after its run completes and before its `finally` +closes the output. On the no-database path the pod closes the writer it is given, so it gets a +writer whose close is left to the node. No hash, stored value or public API changes. +Regressions: `tests/test_pipeline/test_node_staleness_after_run.py` (sync and async, a forced +reverse completion order, in-memory and Delta stores; staleness, heads, store reads, function +calls, a second run). + +Not changed: a `SideEffectJobNode` never stamps, on either path, so it reads as stale after a +run; it keeps no rows and nothing reads its staleness. A head with no rows still shows the +declared columns only (no `_source_*`, `_context_key` or content-hash column; decision of +November 12, 2026, recorded on ITL-985 and ITL-1002). diff --git a/docs/concepts/streams.md b/docs/concepts/streams.md index ebcee55ad..910eedb41 100644 --- a/docs/concepts/streams.md +++ b/docs/concepts/streams.md @@ -165,7 +165,14 @@ What each kind of object shows: A head of an operator is never the operator applied to the heads of its inputs: a group over the first ten rows has the wrong members. So an uncomputed stream shows its columns and "not computed" until `as_table()`, `iter_data()` or a job run computes it. An empty result -shows status `rows` with zero rows, so it is never mistaken for "not computed". +shows status `rows` with zero rows, so it is never mistaken for "not computed". A head without +rows because nothing is computed or stored yet shows the stream's declared columns: the +source-info (`_source_*`), `_context_key` and content-hash columns, which are known only from +rows, can be missing from it even when `columns` or `all_info` asks for them. + +After `job.run()` on either orchestrator, sync or async, every node shows the rows of that +run: `head()`, `as_table()` and `iter_data()` all serve them from memory, without reading the +store. In the preview, large cells are summarised (an array as `ndarray float64 (2, 3)`, bytes as `bytes (300 B)`, long text and long lists shortened); `head().table` keeps the full values. diff --git a/src/orcapod/core/streams/base.py b/src/orcapod/core/streams/base.py index bb036a19c..a5176d318 100644 --- a/src/orcapod/core/streams/base.py +++ b/src/orcapod/core/streams/base.py @@ -520,7 +520,10 @@ def head( This default shows no rows: a stream class that can serve rows without computing overrides it. The zero-row table then has the stream's columns, typed from its declared (Python) schema, so a column's Arrow width can differ from the one the - computed stream will have (``int`` shows as ``int64``). + computed stream will have (``int`` shows as ``int64``). That zero-row table has + the declared columns only: never a ``_source_*``, ``_context_key`` or + content-hash column, whatever ``columns`` asks for, because whether such a column + exists and, for ``_source_*``, its type are known only from rows. Args: n: The largest number of rows to return. diff --git a/superpowers/specs/itl-1042/rowless_head_columns.py b/superpowers/specs/itl-1042/rowless_head_columns.py new file mode 100644 index 000000000..0bdb4a01e --- /dev/null +++ b/superpowers/specs/itl-1042/rowless_head_columns.py @@ -0,0 +1,152 @@ +"""Compare the columns of a head with no rows and a head with rows (ITL-1042, item 4). + +Measurement kept for ITL-985 / ITL-1002 (coordinator decision of November 12, 2026: rowless +heads keep the declared columns for now). For each stream kind and column configuration it +prints the columns a rowless head lacks, or has with another type, compared with the head of +the same stream once it has rows, and the count of mismatches. Run with +``uv run python superpowers/specs/itl-1042/rowless_head_columns.py``. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +import pyarrow as pa + +from orcapod import function_pod +from orcapod.core.operators import Batch, GroupBy, Join +from orcapod.core.sources import ArrowTableSource +from orcapod.databases import InMemoryArrowDatabase +from orcapod.pipeline.job import PipelineJob +from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator +from orcapod.streams import StreamHead +from orcapod.types import CacheMode + +CONFIGS: dict[str, dict[str, Any]] = { + "default": {}, + "content_hash": {"columns": {"content_hash": True}}, + "content_hash_named": {"columns": {"content_hash": "h"}}, + "source": {"columns": {"source": True}}, + "system_tags": {"columns": {"system_tags": True}}, + "context": {"columns": {"context": True}}, + "meta": {"columns": {"meta": True}}, + "all_info": {"all_info": True}, +} + + +@function_pod(output_keys="y") +def scale(x: int) -> int: + """Return ``2 * x``.""" + return 2 * x + + +def _table(columns: dict[str, list[int]]) -> pa.Table: + """A table of non-nullable ``int64`` columns.""" + return pa.Table.from_arrays( + [pa.array(values, type=pa.int64()) for values in columns.values()], + schema=pa.schema( + [pa.field(name, pa.int64(), nullable=False) for name in columns] + ), + ) + + +def _sources() -> tuple[ArrowTableSource, ArrowTableSource]: + """Sources ``a`` (tags ``id``, ``g``; data ``x``) and ``b`` (tag ``id``; data ``z``).""" + a = ArrowTableSource( + _table({"id": [3, 1, 2], "g": [0, 1, 0], "x": [30, 10, 20]}), + tag_columns=["id", "g"], + source_id="a", + label="a", + ) + b = ArrowTableSource( + _table({"id": [2, 3, 1], "z": [200, 300, 100]}), + tag_columns=["id"], + source_id="b", + label="b", + ) + return a, b + + +def _job(store: InMemoryArrowDatabase) -> PipelineJob: + """``scaled`` over ``a``, then ``joined`` (with ``b``, stored) and ``grouped``.""" + a, b = _sources() + job = PipelineJob(name="r", store=store) + with job: + scaled = scale.pod(a, label="scaled") + Join()(scaled, b, label="joined") + GroupBy(by=["g"])(scaled, label="grouped") + job.nodes["joined"].attach_databases( + pipeline_database=store.at("r"), cache_mode=CacheMode.LOG + ) + return job + + +def _schema(head: StreamHead) -> dict[str, str]: + """The head's columns and their Arrow types.""" + return {field.name: str(field.type) for field in head.table.schema} + + +def _report(kind: str, config: str, empty: StreamHead, full: StreamHead) -> bool: + """Print how ``empty``'s columns differ from ``full``'s; True when they differ.""" + empty_schema, full_schema = _schema(empty), _schema(full) + if empty_schema == full_schema: + return False + missing = [name for name in full_schema if name not in empty_schema] + extra = [name for name in empty_schema if name not in full_schema] + retyped = [ + (name, empty_schema[name], full_schema[name]) + for name in full_schema + if name in empty_schema and empty_schema[name] != full_schema[name] + ] + print( + f"{kind:18} {config:18} ({empty.status.value}) missing={missing} " + f"extra={extra} retyped={retyped}" + ) + return True + + +def main() -> None: + """Print every mismatch and their count.""" + mismatches = 0 + before = _job(InMemoryArrowDatabase()) + after = _job(InMemoryArrowDatabase()) + after.run(orchestrator=SyncPipelineOrchestrator()) + for label in ("scaled", "joined", "grouped"): + for config_name, config in CONFIGS.items(): + empty = before.nodes[label].head(10, **config) + full = after.nodes[label].head(10, **config) + mismatches += _report(f"node {label}", config_name, empty, full) + + a, b = _sources() + streams: dict[str, Callable[[], Any]] = { + "FunctionPodStream": lambda: scale.pod(a), + "Join stream": lambda: Join()(scale.pod(a), b), + "GroupBy stream": lambda: GroupBy(by=["g"])(scale.pod(a)), + "Batch stream": lambda: Batch(2)(a), + } + for kind, make in streams.items(): + for config_name, config in CONFIGS.items(): + empty = make().head(10, **config) + computed = make() + _ = computed.as_table() + full = computed.head(10, **config) + mismatches += _report(kind, config_name, empty, full) + print("mismatches", mismatches) + + print("--- source-info types: head with rows vs the zero-row as_table before a run") + for label in ("joined", "grouped"): + with_rows = after.nodes[label].head(10, all_info=True).table.schema + zero_rows = before.nodes[label].as_table(all_info=True).schema + print( + label, + [(f.name, str(f.type)) for f in with_rows if f.name.startswith("_source")], + ) + print( + label, + [(f.name, str(f.type)) for f in zero_rows if f.name.startswith("_source")], + ) + + +if __name__ == "__main__": + main() diff --git a/tests/test_pipeline/test_node_staleness_after_run.py b/tests/test_pipeline/test_node_staleness_after_run.py index 2a9b97b8d..3c9c971ef 100644 --- a/tests/test_pipeline/test_node_staleness_after_run.py +++ b/tests/test_pipeline/test_node_staleness_after_run.py @@ -14,6 +14,7 @@ from __future__ import annotations import asyncio +import contextlib from collections.abc import Callable, Iterator from pathlib import Path from typing import Any @@ -125,9 +126,11 @@ def _reset() -> Iterator[None]: def _table(columns: dict[str, list[int]]) -> pa.Table: """A table of non-nullable ``int64`` columns.""" - return pa.table( - columns, - schema=pa.schema([pa.field(name, pa.int64(), nullable=False) for name in columns]), + return pa.Table.from_arrays( + [pa.array(values, type=pa.int64()) for values in columns.values()], + schema=pa.schema( + [pa.field(name, pa.int64(), nullable=False) for name in columns] + ), ) @@ -189,8 +192,10 @@ def _computed_nodes(job: PipelineJob) -> dict[str, Any]: } -def _count_store_reads(store: ArrowDatabaseProtocol) -> tuple[list[str], Any]: - """A record of store reads and the patcher that fills it (every view of the store).""" +def _count_store_reads( + store: ArrowDatabaseProtocol, +) -> tuple[list[str], contextlib.ExitStack]: + """Start counting reads of ``store`` (every view of it); closing the stack stops it.""" reads: list[str] = [] store_class = type(store) @@ -203,13 +208,16 @@ def read(self: Any, *args: Any, **kwargs: Any) -> Any: return read - return reads, mock.patch.multiple( - store_class, **{name: counting(name) for name in STORE_READS} - ) + patches = contextlib.ExitStack() + for name in STORE_READS: + patches.enter_context(mock.patch.object(store_class, name, counting(name))) + return reads, patches RUNS = [ - pytest.param(orchestrator, store_kind, gated, id=f"{orchestrator}-{store_kind}{tag}") + pytest.param( + orchestrator, store_kind, gated, id=f"{orchestrator}-{store_kind}{tag}" + ) for store_kind in ("in_memory", "delta") for orchestrator, gated, tag in ( ("sync", False, ""), @@ -265,8 +273,8 @@ def test_reads_after_a_run_compute_nothing_and_read_no_store( job = _job(store) _run(job, orchestrator, gated=gated) CALLS.clear() - reads, patcher = _count_store_reads(store) - with patcher: + reads, counting = _count_store_reads(store) + with counting: for node in _computed_nodes(job).values(): _ = node.head(100) _ = node.as_table(all_info=True) From 0fb6c4ab4860d1c63915f3665a211f48f505892c Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 23:17:58 -0800 Subject: [PATCH 04/12] test(function_node): a function node is stamped before its async output closes (ITL-1042) Co-Authored-By: Claude Opus 5.5 --- .../test_node_staleness_after_run.py | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tests/test_pipeline/test_node_staleness_after_run.py b/tests/test_pipeline/test_node_staleness_after_run.py index 3c9c971ef..68bb5097e 100644 --- a/tests/test_pipeline/test_node_staleness_after_run.py +++ b/tests/test_pipeline/test_node_staleness_after_run.py @@ -24,6 +24,9 @@ import pytest from orcapod import function_pod +from orcapod.channels import Channel +from orcapod.core.function_pod import FunctionPod +from orcapod.core.nodes import FunctionJobNode from orcapod.core.operators import GroupBy, Join from orcapod.core.sources import ArrowTableSource from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase @@ -297,3 +300,57 @@ def test_a_second_run_leaves_no_node_stale( for label, node in _computed_nodes(job).items(): assert not node.is_stale, label assert node.head(100).status is HeadStatus.ROWS, label + + +@pytest.mark.parametrize("with_store", [False, True], ids=["no_store", "store"]) +def test_a_function_node_is_stamped_before_its_output_closes(with_store: bool) -> None: + """A reader that sees the node's output close sees the node already stamped. + + A downstream node stamps itself after it has seen that close, so it is then newer than + the function node. Without a store the pod closes the writer it is given; the node must + still stamp first. + """ + source = ArrowTableSource( + _table({"id": A_IDS, "id_": A_IDS, "x": [i + 1 for i in A_IDS]}), + tag_columns=["id"], + source_id="a", + label="A", + ) + store = InMemoryArrowDatabase() if with_store else None + pod = scale.pod + assert isinstance(pod, FunctionPod) + node = FunctionJobNode( + pod, + source, + pipeline_database=store, + result_database=store, + ) + constructed = node.last_modified + assert constructed is not None + seen_at_close: list[Any] = [] + + async def run() -> None: + """Feed the source to the node and record its stamp when the output closes.""" + inputs: Channel[Any] = Channel(buffer_size=16) + output: Channel[Any] = Channel(buffer_size=16) + + async def feed() -> None: + """Send every source row, then close the input.""" + for row in source.iter_data(): + await inputs.writer.send(row) + await inputs.writer.close() + + async def read() -> None: + """Read the output to its end, then note the node's modified time.""" + rows = await output.reader.collect() + assert len(rows) == len(A_IDS) + seen_at_close.append(node.last_modified) + + async with asyncio.TaskGroup() as group: + group.create_task(feed()) + group.create_task(node.async_execute(inputs.reader, output.writer)) + group.create_task(read()) + + asyncio.run(run()) + (stamp,) = seen_at_close + assert stamp is not None and stamp > constructed From 453576987e21d0bdd486b02c9323ddfbd7e080e0 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 23:36:45 -0800 Subject: [PATCH 05/12] test(function_pod): a chain of pod streams is not read again after a full read (ITL-1052) WIP: fails until the fix. Co-Authored-By: Claude Opus 5.5 --- .../function_pod/test_pod_stream_staleness.py | 270 ++++++++++++++++++ 1 file changed, 270 insertions(+) create mode 100644 tests/test_core/function_pod/test_pod_stream_staleness.py diff --git a/tests/test_core/function_pod/test_pod_stream_staleness.py b/tests/test_core/function_pod/test_pod_stream_staleness.py new file mode 100644 index 000000000..2cf01458f --- /dev/null +++ b/tests/test_core/function_pod/test_pod_stream_staleness.py @@ -0,0 +1,270 @@ +"""A chain of function pod streams is not stale after a full read (ITL-1052). + +A stream stamps its modified time when its results are complete (its input read to the end), +before it ends its own iterator, the same rule as a job node (ITL-1042). An upstream in a +chain finishes before its downstream sees the end, so after a full read every stream is at +least as new as its upstreams: a second read (``iter_data``, ``as_table`` or ``head``) calls +no function and reads no store. A real change upstream (``clear_cache()``) still makes the +downstream stale, and its next read recomputes. +""" + +from __future__ import annotations + +import contextlib +from collections import Counter +from collections.abc import Callable, Iterator +from typing import Any +from unittest import mock + +import pyarrow as pa +import pytest + +from orcapod.core.cached_function_pod import CachedFunctionPod +from orcapod.core.data_function import PythonDataFunction +from orcapod.core.executors.base import PythonFunctionExecutorBase +from orcapod.core.function_pod import FunctionPod +from orcapod.core.operators import Join +from orcapod.core.sources import ArrowTableSource +from orcapod.databases import InMemoryArrowDatabase +from orcapod.protocols.core_protocols import FunctionPodProtocol, StreamProtocol +from orcapod.streams import HeadStatus + +# Rows of the source, not in tag order. +IDS = [3, 1, 4, 0, 2] + +# The read methods of the store protocol, counted on every view of an in-memory store. +STORE_READS = ( + "get_all_records", + "get_records_with_column_value", + "get_records_by_ids", + "get_record_by_id", +) + +CALLS: Counter[str] = Counter() + + +def plus_one(x: int) -> int: + """Return ``x + 1``; counted.""" + CALLS["plus_one"] += 1 + return x + 1 + + +def double(y: int) -> int: + """Return ``2 * y``; counted.""" + CALLS["double"] += 1 + return 2 * y + + +def minus_three(z: int) -> int: + """Return ``z - 3``; counted.""" + CALLS["minus_three"] += 1 + return z - 3 + + +# (function, input name, output name) of each link of a chain, in order. +LINKS: list[tuple[Callable[[int], int], str, str]] = [ + (plus_one, "x", "y"), + (double, "y", "z"), + (minus_three, "z", "w"), +] + + +class _ConcurrentExecutor(PythonFunctionExecutorBase): + """An executor that asks for the concurrent read path; it runs functions inline.""" + + @property + def executor_type_id(self) -> str: + """The executor's id.""" + return "test-concurrent" + + def supported_function_type_ids(self) -> frozenset[str]: + """Every function type.""" + return frozenset() + + @property + def supports_concurrent_execution(self) -> bool: + """Take the concurrent read path.""" + return True + + def execute_callable( + self, fn: Callable[..., Any], kwargs: dict[str, Any], *args: Any, **kw: Any + ) -> Any: + """Call ``fn``.""" + return fn(**kwargs) + + async def async_execute_callable( + self, fn: Callable[..., Any], kwargs: dict[str, Any], *args: Any, **kw: Any + ) -> Any: + """Call ``fn``.""" + return fn(**kwargs) + + +@pytest.fixture(autouse=True) +def _reset_calls() -> Iterator[None]: + """Clear the call record around each test.""" + CALLS.clear() + yield + CALLS.clear() + + +def _source() -> ArrowTableSource: + """A source with tag ``id`` and data ``x``, rows not in tag order.""" + schema = pa.schema( + [ + pa.field("id", pa.int64(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] + ) + table = pa.Table.from_arrays( + [pa.array(IDS, type=pa.int64()), pa.array([10 * i for i in IDS], pa.int64())], + schema=schema, + ) + return ArrowTableSource(table, tag_columns=["id"], source_id="src") + + +def _pod( + link: int, executor: str, store: InMemoryArrowDatabase | None +) -> FunctionPodProtocol: + """The pod of link ``link``; with ``store``, a cached pod storing its results there.""" + function, _, output = LINKS[link] + data_function = PythonDataFunction( + function, + output_keys=output, + executor=_ConcurrentExecutor() if executor == "concurrent" else None, + ) + pod = FunctionPod(data_function) + if store is None: + return pod + return CachedFunctionPod(pod, result_database=store) + + +def _chain( + length: int, executor: str, store: InMemoryArrowDatabase | None = None +) -> list[StreamProtocol]: + """The streams of a chain of ``length`` pods over the source, upstream first.""" + streams: list[StreamProtocol] = [] + stream: StreamProtocol = _source() + for link in range(length): + stream = _pod(link, executor, store).process(stream) + streams.append(stream) + return streams + + +def _read(stream: StreamProtocol, how: str) -> int: + """Read ``stream`` with ``how`` and return the number of rows read.""" + if how == "iter_data": + return len(list(stream.iter_data())) + if how == "as_table": + return stream.as_table().num_rows + head = stream.head(100) + assert head.status is HeadStatus.ROWS, head.header() + return head.table.num_rows + + +@contextlib.contextmanager +def _counting_store_reads() -> Iterator[list[str]]: + """Count reads of every in-memory store while the block runs.""" + reads: list[str] = [] + + def counting(name: str) -> Callable[..., Any]: + original = getattr(InMemoryArrowDatabase, name) + + def read(self: Any, *args: Any, **kwargs: Any) -> Any: + reads.append(name) + return original(self, *args, **kwargs) + + return read + + with contextlib.ExitStack() as patches: + for name in STORE_READS: + patches.enter_context( + mock.patch.object(InMemoryArrowDatabase, name, counting(name)) + ) + yield reads + + +def _stale(streams: list[StreamProtocol]) -> list[bool]: + """Whether each stream is stale.""" + return [bool(getattr(stream, "is_stale")) for stream in streams] + + +@pytest.mark.parametrize("how", ["iter_data", "as_table", "head"]) +@pytest.mark.parametrize("executor", ["sequential", "concurrent"]) +@pytest.mark.parametrize("length", [2, 3]) +def test_a_second_read_of_a_chain_calls_no_function( + length: int, executor: str, how: str +) -> None: + """After a full read no stream of the chain is stale; reading again computes nothing.""" + streams = _chain(length, executor) + assert len(list(streams[-1].iter_data())) == len(IDS) + assert sum(CALLS.values()) == length * len(IDS) + assert _stale(streams) == [False] * length + CALLS.clear() + assert _read(streams[-1], how) == len(IDS) + assert CALLS == Counter() + assert _stale(streams) == [False] * length + + +@pytest.mark.parametrize("executor", ["sequential", "concurrent"]) +def test_a_chain_first_read_as_a_table_is_not_read_again(executor: str) -> None: + """A first read through ``as_table`` leaves the chain current too.""" + streams = _chain(3, executor) + first = streams[-1].as_table() + CALLS.clear() + assert streams[-1].as_table().equals(first) + assert len(list(streams[-1].iter_data())) == len(IDS) + assert streams[-1].head(100).table.num_rows == len(IDS) + assert CALLS == Counter() + + +@pytest.mark.parametrize("executor", ["sequential", "concurrent"]) +def test_a_chain_through_an_operator_is_not_read_again(executor: str) -> None: + """A pod after a join of a pod stream reads its input once.""" + first = _pod(0, executor, None).process(_source()) + joined = Join()(first, _source()) + last = _pod(1, executor, None).process(joined) + assert len(list(last.iter_data())) == len(IDS) + CALLS.clear() + for how in ("iter_data", "as_table", "head"): + assert _read(last, how) == len(IDS) + assert CALLS == Counter() + assert _stale([first, joined, last]) == [False, False, False] + + +@pytest.mark.parametrize("executor", ["sequential", "concurrent"]) +@pytest.mark.parametrize("length", [2, 3]) +def test_a_second_read_of_a_cached_chain_reads_no_store( + length: int, executor: str +) -> None: + """A chain of cached pods is read again from memory, without store lookups.""" + streams = _chain(length, executor, InMemoryArrowDatabase()) + assert len(list(streams[-1].iter_data())) == len(IDS) + assert sum(CALLS.values()) == length * len(IDS) + CALLS.clear() + with _counting_store_reads() as reads: + for how in ("iter_data", "as_table", "head"): + assert _read(streams[-1], how) == len(IDS) + assert reads == [] + assert CALLS == Counter() + assert _stale(streams) == [False] * length + + +@pytest.mark.parametrize("executor", ["sequential", "concurrent"]) +@pytest.mark.parametrize("length", [2, 3]) +def test_a_reset_upstream_still_recomputes_the_chain( + length: int, executor: str +) -> None: + """``clear_cache()`` on the first stream makes the rest stale; the next read recomputes. + + The read is ``iter_data``: ``as_table`` of a stale pod stream still returns its kept + table, which is ITL-1045. + """ + streams = _chain(length, executor) + expected = [data.as_dict() for _, data in streams[-1].iter_data()] + clear_cache = getattr(streams[0], "clear_cache") + clear_cache() + assert _stale(streams[1:]) == [True] * (length - 1) + CALLS.clear() + assert [data.as_dict() for _, data in streams[-1].iter_data()] == expected + assert sum(CALLS.values()) == length * len(IDS) + assert _stale(streams) == [False] * length From dc2cdb3afaf3346c8c1b5c284176b3edc6322f46 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 23:38:03 -0800 Subject: [PATCH 06/12] test(function_pod): type-clean executor in the pod stream staleness tests (ITL-1052) Co-Authored-By: Claude Opus 5.5 --- .../function_pod/test_pod_stream_staleness.py | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/tests/test_core/function_pod/test_pod_stream_staleness.py b/tests/test_core/function_pod/test_pod_stream_staleness.py index 2cf01458f..5c3d70694 100644 --- a/tests/test_core/function_pod/test_pod_stream_staleness.py +++ b/tests/test_core/function_pod/test_pod_stream_staleness.py @@ -13,20 +13,21 @@ import contextlib from collections import Counter from collections.abc import Callable, Iterator -from typing import Any +from typing import Any, Self from unittest import mock import pyarrow as pa import pytest +from orcapod import function_pod from orcapod.core.cached_function_pod import CachedFunctionPod -from orcapod.core.data_function import PythonDataFunction from orcapod.core.executors.base import PythonFunctionExecutorBase from orcapod.core.function_pod import FunctionPod from orcapod.core.operators import Join from orcapod.core.sources import ArrowTableSource from orcapod.databases import InMemoryArrowDatabase -from orcapod.protocols.core_protocols import FunctionPodProtocol, StreamProtocol +from orcapod.protocols.core_protocols import StreamProtocol +from orcapod.protocols.observability_protocols import DataExecutionLoggerProtocol from orcapod.streams import HeadStatus # Rows of the source, not in tag order. @@ -86,14 +87,28 @@ def supports_concurrent_execution(self) -> bool: """Take the concurrent read path.""" return True + def with_options(self, **opts: Any) -> Self: + """Return this executor: it has no options.""" + return self + def execute_callable( - self, fn: Callable[..., Any], kwargs: dict[str, Any], *args: Any, **kw: Any + self, + fn: Callable[..., Any], + kwargs: dict[str, Any], + executor_options: dict[str, Any] | None = None, + *, + logger: DataExecutionLoggerProtocol | None = None, ) -> Any: """Call ``fn``.""" return fn(**kwargs) async def async_execute_callable( - self, fn: Callable[..., Any], kwargs: dict[str, Any], *args: Any, **kw: Any + self, + fn: Callable[..., Any], + kwargs: dict[str, Any], + executor_options: dict[str, Any] | None = None, + *, + logger: DataExecutionLoggerProtocol | None = None, ) -> Any: """Call ``fn``.""" return fn(**kwargs) @@ -124,15 +139,14 @@ def _source() -> ArrowTableSource: def _pod( link: int, executor: str, store: InMemoryArrowDatabase | None -) -> FunctionPodProtocol: +) -> FunctionPod | CachedFunctionPod: """The pod of link ``link``; with ``store``, a cached pod storing its results there.""" function, _, output = LINKS[link] - data_function = PythonDataFunction( - function, + pod = function_pod( output_keys=output, executor=_ConcurrentExecutor() if executor == "concurrent" else None, - ) - pod = FunctionPod(data_function) + )(function).pod + assert isinstance(pod, FunctionPod) if store is None: return pod return CachedFunctionPod(pod, result_database=store) From dcb8ae8805b7dedca1f7d665ebd3114a6cd3764c Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 23:40:55 -0800 Subject: [PATCH 07/12] test(function_pod): an upstream reset during a read leaves the reader stale (ITL-1052) Co-Authored-By: Claude Opus 5.5 --- .../function_pod/test_pod_stream_staleness.py | 55 ++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/tests/test_core/function_pod/test_pod_stream_staleness.py b/tests/test_core/function_pod/test_pod_stream_staleness.py index 5c3d70694..4fdc64254 100644 --- a/tests/test_core/function_pod/test_pod_stream_staleness.py +++ b/tests/test_core/function_pod/test_pod_stream_staleness.py @@ -13,7 +13,7 @@ import contextlib from collections import Counter from collections.abc import Callable, Iterator -from typing import Any, Self +from typing import Any, Self, cast from unittest import mock import pyarrow as pa @@ -282,3 +282,56 @@ def test_a_reset_upstream_still_recomputes_the_chain( assert [data.as_dict() for _, data in streams[-1].iter_data()] == expected assert sum(CALLS.values()) == length * len(IDS) assert _stale(streams) == [False] * length + + +EPOCH = [0] + + +def _values(stream: StreamProtocol, column: str) -> dict[int, int]: + """Each row's ``column`` value, keyed by its ``id`` tag, read with ``iter_data``.""" + return { + cast(int, tag.as_dict()["id"]): cast(int, data.as_dict()[column]) + for tag, data in stream.iter_data() + } + + +def stamped(x: int) -> int: + """Return ``x`` tagged with the current epoch: a function whose values change.""" + CALLS["stamped"] += 1 + return 1000 * EPOCH[0] + x + + +@pytest.mark.parametrize("executor", ["sequential", "concurrent"]) +@pytest.mark.parametrize("length", [2, 3]) +def test_an_upstream_reset_during_a_read_leaves_the_reader_stale( + length: int, executor: str +) -> None: + """A reset of the first stream in the middle of a read is never reported as current. + + The read is finished after the reset; the last stream is then stale, and its next read + gives rows computed from one state of the first stream only. + """ + EPOCH[0] = 0 + first = function_pod( + output_keys="y", + executor=_ConcurrentExecutor() if executor == "concurrent" else None, + )(stamped).pod.process(_source()) + streams: list[StreamProtocol] = [first] + for link in range(1, length): + streams.append(_pod(link, executor, None).process(streams[-1])) + reader = iter(streams[-1].iter_data()) + _ = [next(reader) for _ in range(2)] + EPOCH[0] = 1 + clear_cache = getattr(first, "clear_cache") + clear_cache() + _ = list(reader) + assert getattr(streams[-1], "is_stale") + last = _values(streams[-1], "z" if length == 2 else "w") + ys = _values(first, "y") + assert sorted(last) == sorted(ys) == sorted(IDS) + # Every row of the first stream is of the state after the reset ... + assert {y // 1000 for y in ys.values()} == {1} + # ... and every row of the last stream was computed from that row. + for key, y in ys.items(): + assert last[key] == (2 * y if length == 2 else 2 * y - 3), key + assert _stale(streams) == [False] * length From c30950a01424171d5a5145777d23498a6036d014 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 23:42:06 -0800 Subject: [PATCH 08/12] docs(plan): ITL-1052 cause, rule and design (ITL-1042) Co-Authored-By: Claude Opus 5.5 --- .../2026-11-12-itl-1042-head-after-async.md | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/superpowers/plans/2026-11-12-itl-1042-head-after-async.md b/superpowers/plans/2026-11-12-itl-1042-head-after-async.md index ca8cd6bf5..c971c7a17 100644 --- a/superpowers/plans/2026-11-12-itl-1042-head-after-async.md +++ b/superpowers/plans/2026-11-12-itl-1042-head-after-async.md @@ -143,3 +143,39 @@ stream kinds) on ITL-985 and ITL-1002. Save the measurement script as - ITL-1045 is a different defect (two passes in `as_table` with a genuinely stale upstream); this one is a false staleness after an async run. - Scratchpad scripts: `repro.py`, `cost.py`, `emptyhead.py`, `a346/drive.py`. + +--- + +## ITL-1052 (added Nov 12, 2026 by the coordinator; same PR: Fixes ITL-1042, Fixes ITL-1052) + +**Cause.** `FunctionPodStream._ensure_iterator` stamps when the stream OPENS its input. The +upstream's `iter_data` is a lazy generator, so in `s2 = g.pod(f.pod(src))` the upstream opens +(and stamps) only after `s2` has stamped: `s2` ends its first full read older than `s1`, and its +second read clears and recomputes (6 calls, then 9). + +**Rule (same as ITL-1042).** A stream or node stamps its modified time when its results are +complete (its input read to the end), before it signals the end downstream; never at the start +of a read. `clear_cache()` stamps (a reset is a change). A failed or partial read does not stamp. + +**Mid-read reset (coordinator: must not become silent).** A reset of an upstream in the middle +of a downstream's read must leave the downstream stale. A modified-time snapshot alone cannot +tell a reset from the upstream's own completion stamp (which lands during the downstream's +read), so: +- `FunctionPodStream` keeps `_reset_generation`, bumped by `clear_cache()`. +- When a read takes its first item from the input (after the upstream's own stale refresh), it + snapshots each upstream: the generation for a `FunctionPodStream` upstream, the modified time + for any other. Kept across partial reads; cleared by `clear_cache()`. +- At the end of the input it stamps if the snapshot still holds and no upstream modified time is + `None`; otherwise it sets its own modified time to `None` (stale; downstream streams then see a + `None` upstream and are stale too, so it propagates). +- Concurrent path: same, at the end of the materialised compute. + +**Tests.** `tests/test_core/function_pod/test_pod_stream_staleness.py` (28 cases; all fail +before the fix on the staleness assertion): chains of 2 and 3, sequential and concurrent, second +read by `iter_data` / `as_table` / `head` calls nothing; first read by `as_table`; through a +`Join`; cached chain reads no store; `clear_cache()` upstream recomputes (read by `iter_data`: +`as_table` on a stale pod stream is ITL-1045); upstream reset mid-read leaves the reader stale +and the next read is of one state. + +**Waiting for** #348 (rewrites `_iter_data_sequential` / `_iter_data_concurrent`) to merge; then +merge origin/main and implement. From 0ea878c1b86040a942755e58a29f33000283e2e0 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 23:46:04 -0800 Subject: [PATCH 09/12] test(function_pod): mid-read upstream reset for cached chains and through an operator (ITL-1052) Co-Authored-By: Claude Opus 5.5 --- .../function_pod/test_pod_stream_staleness.py | 90 +++++++++++++++---- 1 file changed, 72 insertions(+), 18 deletions(-) diff --git a/tests/test_core/function_pod/test_pod_stream_staleness.py b/tests/test_core/function_pod/test_pod_stream_staleness.py index 4fdc64254..036ec7bc2 100644 --- a/tests/test_core/function_pod/test_pod_stream_staleness.py +++ b/tests/test_core/function_pod/test_pod_stream_staleness.py @@ -137,6 +137,22 @@ def _source() -> ArrowTableSource: return ArrowTableSource(table, tag_columns=["id"], source_id="src") +def _other() -> ArrowTableSource: + """A second source with tag ``id`` and data ``q``, in another row order.""" + schema = pa.schema( + [ + pa.field("id", pa.int64(), nullable=False), + pa.field("q", pa.int64(), nullable=False), + ] + ) + ids = sorted(IDS) + table = pa.Table.from_arrays( + [pa.array(ids, type=pa.int64()), pa.array([7 * i for i in ids], pa.int64())], + schema=schema, + ) + return ArrowTableSource(table, tag_columns=["id"], source_id="other") + + def _pod( link: int, executor: str, store: InMemoryArrowDatabase | None ) -> FunctionPod | CachedFunctionPod: @@ -235,8 +251,8 @@ def test_a_chain_first_read_as_a_table_is_not_read_again(executor: str) -> None: def test_a_chain_through_an_operator_is_not_read_again(executor: str) -> None: """A pod after a join of a pod stream reads its input once.""" first = _pod(0, executor, None).process(_source()) - joined = Join()(first, _source()) - last = _pod(1, executor, None).process(joined) + joined = Join()(first, _other()) + last = _pod(1, executor, None)(joined) assert len(list(last.iter_data())) == len(IDS) CALLS.clear() for how in ("iter_data", "as_table", "head"): @@ -301,37 +317,75 @@ def stamped(x: int) -> int: return 1000 * EPOCH[0] + x +def _stamped_stream( + executor: str, store: InMemoryArrowDatabase | None +) -> StreamProtocol: + """The stream of ``stamped`` over the source; cached in ``store`` when given.""" + pod = function_pod( + output_keys="y", + executor=_ConcurrentExecutor() if executor == "concurrent" else None, + )(stamped).pod + assert isinstance(pod, FunctionPod) + if store is None: + return pod.process(_source()) + return CachedFunctionPod(pod, result_database=store).process(_source()) + + +def _reset_during_a_read(first: StreamProtocol, last: StreamProtocol) -> None: + """Read two rows of ``last``, reset ``first`` (new epoch), then finish the read.""" + reader = iter(last.iter_data()) + _ = [next(reader) for _ in range(2)] + EPOCH[0] = 1 + clear_cache = getattr(first, "clear_cache") + clear_cache() + _ = list(reader) + + +@pytest.mark.parametrize("cached", [False, True], ids=["plain", "cached"]) @pytest.mark.parametrize("executor", ["sequential", "concurrent"]) @pytest.mark.parametrize("length", [2, 3]) def test_an_upstream_reset_during_a_read_leaves_the_reader_stale( - length: int, executor: str + length: int, executor: str, cached: bool ) -> None: """A reset of the first stream in the middle of a read is never reported as current. The read is finished after the reset; the last stream is then stale, and its next read - gives rows computed from one state of the first stream only. + gives rows computed from one state of the first stream only. A cached chain may serve + the first stream's stored results again, so there only the derivation is checked. """ EPOCH[0] = 0 - first = function_pod( - output_keys="y", - executor=_ConcurrentExecutor() if executor == "concurrent" else None, - )(stamped).pod.process(_source()) + store = InMemoryArrowDatabase() if cached else None + first = _stamped_stream(executor, store) streams: list[StreamProtocol] = [first] for link in range(1, length): - streams.append(_pod(link, executor, None).process(streams[-1])) - reader = iter(streams[-1].iter_data()) - _ = [next(reader) for _ in range(2)] - EPOCH[0] = 1 - clear_cache = getattr(first, "clear_cache") - clear_cache() - _ = list(reader) + streams.append(_pod(link, executor, store).process(streams[-1])) + _reset_during_a_read(first, streams[-1]) assert getattr(streams[-1], "is_stale") last = _values(streams[-1], "z" if length == 2 else "w") ys = _values(first, "y") assert sorted(last) == sorted(ys) == sorted(IDS) - # Every row of the first stream is of the state after the reset ... - assert {y // 1000 for y in ys.values()} == {1} - # ... and every row of the last stream was computed from that row. + if not cached: + # Every row of the first stream is of the state after the reset ... + assert {y // 1000 for y in ys.values()} == {1} + # ... and every row of the last stream was computed from that stream's row. for key, y in ys.items(): assert last[key] == (2 * y if length == 2 else 2 * y - 3), key assert _stale(streams) == [False] * length + + +@pytest.mark.parametrize("executor", ["sequential", "concurrent"]) +def test_an_upstream_reset_during_a_read_through_an_operator(executor: str) -> None: + """Pod, then a join, then a pod: a reset of the first pod mid-read leaves the last stale.""" + EPOCH[0] = 0 + first = _stamped_stream(executor, None) + joined = Join()(first, _other()) + last = _pod(1, executor, None)(joined) + _reset_during_a_read(first, last) + assert getattr(last, "is_stale") + zs = _values(last, "z") + ys = _values(first, "y") + assert sorted(zs) == sorted(ys) == sorted(IDS) + assert {y // 1000 for y in ys.values()} == {1} + for key, y in ys.items(): + assert zs[key] == 2 * y, key + assert _stale([first, joined, last]) == [False, False, False] From 328761c5e67fe83176ca15399a4c699a5599a8a1 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 00:07:51 -0800 Subject: [PATCH 10/12] fix(function_pod): a pod stream stamps itself when its input is read to the end (ITL-1052) A FunctionPodStream stamped its modified time when it opened its input. The input of a chained pod stream is a lazy generator, so the upstream opened and stamped after the downstream had: the downstream ended its first full read older than its upstream, and its next read cleared and recomputed it. The stream now stamps when its input is read to the end, before its reader sees the end, the rule job nodes follow (ITL-1042). A reset of the input during the read (its reset generation, or for any other input its modified time, changed since the first row arrived) clears the modified time instead, and a stale pod stream reports no modified time, so every stream reading it is stale too. The bookkeeping is in memory only. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/function_pod.py | 71 +++++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 2 deletions(-) diff --git a/src/orcapod/core/function_pod.py b/src/orcapod/core/function_pod.py index 07337337b..7040f3677 100644 --- a/src/orcapod/core/function_pod.py +++ b/src/orcapod/core/function_pod.py @@ -983,6 +983,14 @@ def __init__( self._cached_output_table: pa.Table | None = None self._cached_content_hash_column: pa.Array | None = None + # Staleness bookkeeping, in memory only (never hashed, saved or stored). The + # stream stamps its modified time when its input is read to the end, if its + # input did not change while it was read (ITL-1042, ITL-1052): every reset + # bumps the generation, and the state of the input taken when the first row + # arrived is compared at the end. + self._reset_generation = 0 + self._input_state: tuple[object, ...] | None = None + @property def producer(self) -> PodProtocol: return self._function_pod @@ -1040,7 +1048,6 @@ def _ensure_iterator(self) -> None: if self._needs_iterator: self._cached_input_iterator = enumerate(self._input_stream.iter_data()) self._needs_iterator = False - self._update_modified_time() elif self._input_interrupted: self._cached_input_iterator = enumerate(self._input_stream.iter_data()) self._input_interrupted = False @@ -1050,8 +1057,63 @@ def _input_raised(self) -> None: self._cached_input_iterator = None self._input_interrupted = True + @property + def last_modified(self) -> datetime | None: + """When this stream's results were completed; ``None`` while they are not current. + + ``None`` (modification time unknown) while the stream is stale, that is while its + input or its pod changed after its results were completed, or while those results + may mix two states of its input. A stream reading this one then sees an unknown + time and is stale as well, however far down the chain it is. + """ + own_time = self._modified_time + if own_time is None: + return None + for upstream_time in ( + self._input_stream.last_modified, + self._function_pod.last_modified, + ): + if upstream_time is None or upstream_time > own_time: + return None + return own_time + + def _current_input_state(self) -> tuple[object, ...]: + """What tells a change of the input: its reset generation, else its modified time. + + A pod stream's modified time moves when it finishes its own read, which happens + while this stream reads it; only its resets are changes. Any other input + (operator stream, source, node) is computed or fixed by the time it yields. + """ + upstream = self._input_stream + if isinstance(upstream, FunctionPodStream): + return (upstream._reset_generation,) + return (upstream.last_modified,) + + def _note_input_state(self) -> None: + """Remember the input's state when its first row (or its end) arrives.""" + if self._input_state is None: + self._input_state = self._current_input_state() + + def _input_read_to_end(self) -> None: + """Stamp the completed result, or mark it stale if the input changed during the read. + + A stream stamps its modified time when its results are complete, before its + reader sees the end, so it is newer than its input and older than its readers. + If the input was reset while it was read, or is itself not current (no modified + time), the result may mix two states of the input: the modified time is cleared + instead, which makes this stream, and every stream reading it, stale. + """ + self._note_input_state() + changed = self._input_state != self._current_input_state() + if changed or self._input_stream.last_modified is None: + self._set_modified_time(None) + else: + self._update_modified_time() + def clear_cache(self) -> None: """Discard all in-memory cached state.""" + self._reset_generation += 1 + self._input_state = None self._cached_input_iterator = None self._needs_iterator = True self._input_interrupted = False @@ -1113,14 +1175,17 @@ def _iter_data_sequential( try: input_index, (tag, data) = next(input_iter) except StopIteration: - # Mark completion by releasing the iterator + # Mark completion by releasing the iterator, and stamp the result + # before the reader sees the end. self._cached_input_iterator = None + self._input_read_to_end() return except BaseException: # The input itself raised; its iterator is closed. Open it again on # the next read instead of taking it as exhausted. self._input_raised() raise + self._note_input_state() if input_index in self._cached_output_datas: # An input opened again yields the rows this stream already cached. continue @@ -1209,6 +1274,8 @@ async def _gather() -> list[ if i not in self._cached_output_datas ] self._cached_input_iterator = iter(remaining) if remaining else None + # Every row is computed: stamp the result before the reader sees any row. + self._input_read_to_end() # Yield every row in input order, those an earlier iteration computed included. for i in range(len(self._cached_output_datas)): From e002f99bd4867447fdc09bae529423476304de2a Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 00:10:03 -0800 Subject: [PATCH 11/12] docs: one modified-time rule for nodes and pod streams (ITL-1042, ITL-1052) Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1042.md | 10 +++++++++ design_issues/ITL-1042-NS1.md | 3 ++- design_issues/ITL-1052-F18.md | 42 +++++++++++++++++++++++++++++++++++ docs/concepts/streams.md | 7 ++++-- 4 files changed, 59 insertions(+), 3 deletions(-) create mode 100644 design_issues/ITL-1052-F18.md diff --git a/changelog.d/ITL-1042.md b/changelog.d/ITL-1042.md index 11c707e5c..35c1743b0 100644 --- a/changelog.d/ITL-1042.md +++ b/changelog.d/ITL-1042.md @@ -10,3 +10,13 @@ results in memory, and from then on every operator node downstream of it showed `as_table()` and `iter_data()` returned every row. After a run on either orchestrator no node is now out of date: `head()` shows the rows `as_table()` returns, and none of them reads the store. Nothing was ever computed twice. (ITL-1042) + +#### A chain of function pod streams is not computed again on its second read (ITL-1052) + +In a chain such as `g.pod(f.pod(source))`, the last stream was taken to be out of date right +after it was read to the end, so its next `iter_data()`, `as_table()` or `head()` called `g` +again on every row (or, for a cached pod, looked every row up in its store again). A pod +stream now records that its results are complete when it has read its input to the end, so a +second read computes nothing. A change upstream (`clear_cache()`), also one made while the +chain is being read, still makes every stream after it out of date, and its next read +computes it again. (ITL-1052) diff --git a/design_issues/ITL-1042-NS1.md b/design_issues/ITL-1042-NS1.md index ecbdf505b..645152838 100644 --- a/design_issues/ITL-1042-NS1.md +++ b/design_issues/ITL-1042-NS1.md @@ -20,7 +20,8 @@ memory. No function was called again and no operator recomputed. **Rule:** a job node stamps its modified time once per completed run, after it has read all its inputs and before it closes its output, on both paths; a failed run does not stamp. Every node then ends a run at least as new as its upstreams, so nothing is stale after a completed -run, and `head()`, `as_table()` and `iter_data()` serve the run's results from memory. +run, and `head()`, `as_table()` and `iter_data()` serve the run's results from memory. The +same rule governs function pod streams outside a job (F18, ITL-1052). **Fix:** `async_execute` stamps the node after its run completes and before its `finally` closes the output. On the no-database path the pod closes the writer it is given, so it gets a diff --git a/design_issues/ITL-1052-F18.md b/design_issues/ITL-1052-F18.md new file mode 100644 index 000000000..817d5e08f --- /dev/null +++ b/design_issues/ITL-1052-F18.md @@ -0,0 +1,42 @@ +## `src/orcapod/core/function_pod.py` + +### F18 — A chained function pod stream was stale right after its first full read +**Status:** resolved +**Severity:** high +**Issue:** ITL-1052 + +`FunctionPodStream` stamped its modified time when it opened its input +(`_ensure_iterator`). The input of a chained pod stream is a lazy generator, so in +`s2 = g.pod(f.pod(src))` the upstream opened, and stamped, only after `s2` had: `s2` ended its +first full read older than its upstream, and its next read cleared its cache and called `g` +again on every row (6 calls, then 9). Operator streams compute eagerly when they are opened, +so a chain through an operator was not affected; a pod reading a pod directly was. + +**Rule** (the same as for job nodes, NS1 / ITL-1042): a stream stamps its modified time when +its results are complete, that is when its input is read to the end, before its reader sees +the end; never when a read starts. `clear_cache()` stamps (a reset is a change). A failed or +partial read does not stamp. + +**Fix:** the stamp moved from `_ensure_iterator` to the end of the input, on the sequential +and the concurrent read paths. A reset of the input during a read must not be reported as +current (the result may mix two states of the input), so: + +- each pod stream keeps a reset generation, bumped by `clear_cache()`; +- when the first row arrives, the stream notes its input's state: the generation of a pod + stream input (whose modified time legitimately moves during the read, when it finishes its + own read), the modified time of any other input; +- at the end of the input it stamps only if that state is unchanged and the input has a + modified time; otherwise it clears its own modified time, which makes it stale; +- a stale pod stream reports `last_modified` as `None` (unknown), so every stream reading it, + however far down the chain, is stale too. `is_stale` compares only direct upstreams, and a + stale pod stream used to keep its old time: before this fix a reset two links up went + unnoticed by the last stream, hidden by the false staleness above. + +The generation and the noted state are in-memory bookkeeping only: they enter no hash, no +saved file and no store. Regressions: `tests/test_core/function_pod/test_pod_stream_staleness.py` +(chains of two and three, sequential and concurrent, the cached pod stream, a chain through a +join; second reads call no function and read no store; a reset upstream recomputes; a reset +during a read leaves the reader stale and its next read is of one state). + +Not changed: `FunctionPodStream.as_table` still returns its kept table when the stream is stale +(ITL-1045). diff --git a/docs/concepts/streams.md b/docs/concepts/streams.md index 910eedb41..4566c29b4 100644 --- a/docs/concepts/streams.md +++ b/docs/concepts/streams.md @@ -170,9 +170,12 @@ rows because nothing is computed or stored yet shows the stream's declared colum source-info (`_source_*`), `_context_key` and content-hash columns, which are known only from rows, can be missing from it even when `columns` or `all_info` asks for them. -After `job.run()` on either orchestrator, sync or async, every node shows the rows of that +A stream or node records that its results are current when it has read its input to the end. +So after `job.run()` on either orchestrator, sync or async, every node shows the rows of that run: `head()`, `as_table()` and `iter_data()` all serve them from memory, without reading the -store. +store. Likewise, once a chain of streams such as `g.pod(f.pod(source))` has been read to the +end, reading it again computes nothing. A change upstream (`clear_cache()`) makes every stream +after it out of date, and its next read computes it again. In the preview, large cells are summarised (an array as `ndarray float64 (2, 3)`, bytes as `bytes (300 B)`, long text and long lists shortened); `head().table` keeps the full values. From 2f32e7e6479ff029a18fdbbeadf8598cac05717a Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 01:17:59 -0800 Subject: [PATCH 12/12] fix(function_pod): note the input state before the concurrent computation (ITL-1052) A reset of the input made while the concurrent path computes (a post-run hook, or the function itself) became the baseline and the pre-reset results were stamped current (CodeRabbit on PR #355). Also narrows the docs sentence on reads after a run: a function node whose run left no rows in memory reads its stored rows. Co-Authored-By: Claude Opus 5.5 --- design_issues/ITL-1052-F18.md | 6 ++- docs/concepts/streams.md | 4 +- src/orcapod/core/function_pod.py | 3 ++ .../function_pod/test_pod_stream_staleness.py | 46 +++++++++++++++++++ 4 files changed, 55 insertions(+), 4 deletions(-) diff --git a/design_issues/ITL-1052-F18.md b/design_issues/ITL-1052-F18.md index 817d5e08f..943c2803b 100644 --- a/design_issues/ITL-1052-F18.md +++ b/design_issues/ITL-1052-F18.md @@ -22,7 +22,8 @@ and the concurrent read paths. A reset of the input during a read must not be re current (the result may mix two states of the input), so: - each pod stream keeps a reset generation, bumped by `clear_cache()`; -- when the first row arrives, the stream notes its input's state: the generation of a pod +- when the first row arrives (on the concurrent path, once the input has delivered its rows, + before they are computed), the stream notes its input's state: the generation of a pod stream input (whose modified time legitimately moves during the read, when it finishes its own read), the modified time of any other input; - at the end of the input it stamps only if that state is unchanged and the input has a @@ -36,7 +37,8 @@ The generation and the noted state are in-memory bookkeeping only: they enter no saved file and no store. Regressions: `tests/test_core/function_pod/test_pod_stream_staleness.py` (chains of two and three, sequential and concurrent, the cached pod stream, a chain through a join; second reads call no function and read no store; a reset upstream recomputes; a reset -during a read leaves the reader stale and its next read is of one state). +during a read, or made by the function while the stream computes, leaves the reader stale and +its next read is of one state). Not changed: `FunctionPodStream.as_table` still returns its kept table when the stream is stale (ITL-1045). diff --git a/docs/concepts/streams.md b/docs/concepts/streams.md index 4566c29b4..e26973cae 100644 --- a/docs/concepts/streams.md +++ b/docs/concepts/streams.md @@ -172,8 +172,8 @@ rows, can be missing from it even when `columns` or `all_info` asks for them. A stream or node records that its results are current when it has read its input to the end. So after `job.run()` on either orchestrator, sync or async, every node shows the rows of that -run: `head()`, `as_table()` and `iter_data()` all serve them from memory, without reading the -store. Likewise, once a chain of streams such as `g.pod(f.pod(source))` has been read to the +run: `head()`, `as_table()` and `iter_data()` serve them from memory, without reading the +store (a function node whose run left no rows in memory reads its stored rows). Likewise, once a chain of streams such as `g.pod(f.pod(source))` has been read to the end, reading it again computes nothing. A change upstream (`clear_cache()`) makes every stream after it out of date, and its next read computes it again. diff --git a/src/orcapod/core/function_pod.py b/src/orcapod/core/function_pod.py index 7040f3677..9e170ad26 100644 --- a/src/orcapod/core/function_pod.py +++ b/src/orcapod/core/function_pod.py @@ -1225,6 +1225,9 @@ def _iter_data_concurrent( # and reads them anew (an upstream stream yields its cached rows). self._input_raised() raise + # The input has delivered its rows (after its own refresh, if it had one): note + # its state now, before computing, so a reset during the computation is seen. + self._note_input_state() # Submit uncached data concurrently via async_process_data. Every row that # returns is cached; a row whose function raised stays in the input, so the # next read computes it again, and the first error is raised after the others diff --git a/tests/test_core/function_pod/test_pod_stream_staleness.py b/tests/test_core/function_pod/test_pod_stream_staleness.py index 036ec7bc2..523e12ae5 100644 --- a/tests/test_core/function_pod/test_pod_stream_staleness.py +++ b/tests/test_core/function_pod/test_pod_stream_staleness.py @@ -389,3 +389,49 @@ def test_an_upstream_reset_during_a_read_through_an_operator(executor: str) -> N for key, y in ys.items(): assert zs[key] == 2 * y, key assert _stale([first, joined, last]) == [False, False, False] + + +RESETS: list[Callable[[], None]] = [] + + +def double_and_reset(y: int) -> int: + """Return ``2 * y``; the first call runs the pending reset, if any.""" + CALLS["double_and_reset"] += 1 + if RESETS: + RESETS.pop()() + return 2 * y + + +@pytest.mark.parametrize("executor", ["sequential", "concurrent"]) +def test_an_upstream_reset_while_computing_leaves_the_reader_stale( + executor: str, +) -> None: + """A reset of the input made while the stream computes is never reported as current. + + On the concurrent path every input row is read before any is computed, so the reset + comes after the input delivered its rows and before the results are complete. + """ + EPOCH[0] = 0 + first = _stamped_stream(executor, None) + last = function_pod( + output_keys="z", + executor=_ConcurrentExecutor() if executor == "concurrent" else None, + )(double_and_reset).pod.process(first) + + def reset_first() -> None: + """Move to a new epoch and reset the first stream.""" + EPOCH[0] = 1 + clear_cache = getattr(first, "clear_cache") + clear_cache() + + RESETS.append(reset_first) + _ = list(last.iter_data()) + assert RESETS == [] + assert getattr(last, "is_stale") + zs = _values(last, "z") + ys = _values(first, "y") + assert sorted(zs) == sorted(ys) == sorted(IDS) + assert {y // 1000 for y in ys.values()} == {1} + for key, y in ys.items(): + assert zs[key] == 2 * y, key + assert _stale([first, last]) == [False, False]