From b5247136781ac3abc95b7f8b06e8bc0e80874497 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 03:07:05 -0800 Subject: [PATCH 1/5] test(function_pod): interleaved and resumed reads compute each row once (ITL-1065) WIP: the sequential cases with an operator stream at the root fail until the fix. Co-Authored-By: Claude Opus 5.5 --- .../2026-11-13-itl-1065-interleaved-reads.md | 118 +++++++++ .../test_pod_stream_interleaved_reads.py | 240 ++++++++++++++++++ 2 files changed, 358 insertions(+) create mode 100644 superpowers/plans/2026-11-13-itl-1065-interleaved-reads.md create mode 100644 tests/test_core/function_pod/test_pod_stream_interleaved_reads.py diff --git a/superpowers/plans/2026-11-13-itl-1065-interleaved-reads.md b/superpowers/plans/2026-11-13-itl-1065-interleaved-reads.md new file mode 100644 index 000000000..baa6e1559 --- /dev/null +++ b/superpowers/plans/2026-11-13-itl-1065-interleaved-reads.md @@ -0,0 +1,118 @@ +# ITL-1065 Interleaved reads of a pod stream recompute a yielded row — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:executing-plans to implement this plan +> task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Interleaved and resumed reads of a function pod stream call the function once per +input row and yield the same datagrams as a later full read, whatever its upstream kind, while +every staleness guarantee of PR #355 (ITL-1042, ITL-1052) still holds. + +**Architecture:** `FunctionPodStream.is_stale` gets two rules: completed results are judged by +modified-time order (as now); results in progress (a read has noted its input's state and has +not reached the end) are judged by whether the input's noted state changed. In-memory +bookkeeping only. + +**Tech Stack:** Python 3.12, pytest, `uv run`. + +--- + +## Cause (confirmed on 89160c47, probe in the session scratchpad) + +`s = f.pod(DictSource(rows with an extra column))` reads an automatic `SelectDataColumns` +stream (`DynamicPodStream`, last_modified `None` until it runs). + +``` +start s.own=.655051 s.lm=None gen=0 stale=True up.lm=None calls=0 +next(first) s.own=.655087 s.lm=None gen=1 stale=True up.lm=.655445 calls=1 +3 x next(second) s.own=.656093 s.lm=.656093 gen=2 stale=False up.lm=.655445 calls=4 +drain first/second/third: calls=7, first read's row 0 is not the third read's +``` + +1. `first`'s `iter_data` sees the stream stale (input time unknown), `clear_cache()` stamps + `s` at t1 (generation 1). +2. `_ensure_iterator` opens the operator stream, which computes now and takes t2 > t1. +3. Until the read completes, `s` is older than its input, so `last_modified` is `None` and + `is_stale` is True mid-read. `second`'s `iter_data` clears the cache (generation 2) and + recomputes row 0, which `first` had already yielded. + +Before #355, `_ensure_iterator` stamped `s` after opening (and so computing) the operator +input, which hid this. Without the extra column the input is a source, older than `s`. + +## Rule after the fix + +- Completed results (input read to the end and stamped, or nothing read since a reset) are + current while no direct upstream and not the pod is newer: modified-time order, as #355. +- Results in progress (a read has noted its input's state, the end not reached) are current + while the input's state is the noted one and the pod did not change: a pod-stream input's + reset generation; any other input's modified time, noted once it has delivered a row (an + operator stream computes when opened, so its time then moves only when it is recomputed or + invalidated). An upstream completing for the first time during the read is not a change. +- So the operator stream needs no reset generation: its modified time, noted after it has + computed, is one. +- A reset during a read still changes the noted state: the next read of the stream clears it, + and the stream ends its read stale (unchanged from #355). + +## Files + +- Modify: `src/orcapod/core/function_pod.py` (`FunctionPodStream`: `is_stale` override, + `_input_read_to_end` ends the read in progress). +- Create: `tests/test_core/function_pod/test_pod_stream_interleaved_reads.py`. +- Create: `design_issues/ITL-1065-F19.md`; update `design_issues/ITL-1052-F18.md` (rule). +- Create: `changelog.d/ITL-1065.md`. + +### Task 1: Failing tests + +Public interface only: `iter_data`, `is_stale`, `clear_cache`, call counts, `is` identity of +yielded datagrams. + +Upstream kinds: `source` (`pod.process(src)`), `operator` (`pod(src_with_extra_column)`, the +automatic select), `pod_stream` (`g.pod(f.pod(src))`), `pod_over_operator` +(`g.pod(f.pod(src_with_extra))`). Executors: sequential, concurrent. Pods: plain, cached +(`CachedFunctionPod`, fresh in-memory store). + +- [ ] `test_interleaved_reads_compute_each_row_once`: `first`, `second` iterators; one row from + `first`, three from `second`; drain both; a third full read. Every function called once per + row; the three reads yield the same datagram objects; nothing stale. +- [ ] `test_a_read_after_an_abandoned_partial_read_computes_each_row_once`: one row, abandon, + full read: once per row, same row 0 object. +- [ ] `test_a_reset_between_interleaved_reads_gives_one_state`: stamped (epoch) upstream pod, + directly or through a join + automatic select; reset it between the two iterators; drain; + third read is all epoch 1, derived from the upstream's rows, computes nothing, not stale. +- [ ] Run, expect `operator` / `pod_over_operator` sequential cases to fail (7 calls). +- [ ] WIP commit. + +### Task 2: Fix + +```python +@property +def is_stale(self) -> bool: + if self._input_state is None: + return super().is_stale + current = self._current_input_state() + if current != self._input_state or None in current: + return True + own = self._modified_time + pod_time = self._function_pod.last_modified + return own is None or pod_time is None or pod_time > own +``` + +`_input_read_to_end` sets `self._input_state = None` after stamping (the read is no longer in +progress). `clear_cache` already clears it. + +- [ ] Implement; run the new tests, `test_pod_stream_staleness.py`, + `tests/test_pipeline/test_node_staleness_after_run.py`, `tests/test_core/function_pod`, + `tests/test_core/streams` (one pytest process, Python 3.12). +- [ ] Re-run the probe: 6 calls. +- [ ] Commit. + +### Task 3: Docs and records + +- [ ] `design_issues/ITL-1065-F19.md` (cause, rule, fix); F18 rule paragraph points to F19. +- [ ] `changelog.d/ITL-1065.md` under `### Fixed`. +- [ ] `uv run python scripts/design_issues.py --check`, `scripts/assemble_changelog.py --check`, + ruff, basedpyright. +- [ ] Commit, merge origin/main, push, PR (Fixes ITL-1065), Copilot review. + +## Notes for a successor + +- Status: (update as work proceeds) diff --git a/tests/test_core/function_pod/test_pod_stream_interleaved_reads.py b/tests/test_core/function_pod/test_pod_stream_interleaved_reads.py new file mode 100644 index 000000000..dc1e73666 --- /dev/null +++ b/tests/test_core/function_pod/test_pod_stream_interleaved_reads.py @@ -0,0 +1,240 @@ +"""Interleaved and resumed reads of a function pod stream compute each row once (ITL-1065). + +Two iterators open on one pod stream share its results: a row one of them computed is the +row the other yields, and a later full read yields the same datagrams. An upstream that +completes (or, for an operator stream, computes) for the first time while the stream is +being read is not a change of the stream's input, whatever the upstream's kind: a source, an +operator stream (the automatic column selection of ``pod(stream)``), a pod stream, or a pod +stream over an operator stream. A real reset of the input between two reads is still a +change: the stream's next read starts again, and every later read is of one state. +""" + +from __future__ import annotations + +from collections import Counter +from collections.abc import Iterator +from typing import cast + +import pyarrow as pa +import pytest + +from orcapod import function_pod +from orcapod.core.cached_function_pod import CachedFunctionPod +from orcapod.core.function_pod import FunctionPod +from orcapod.core.operators import Join +from orcapod.core.sources import ArrowTableSource +from orcapod.databases import InMemoryArrowDatabase +from orcapod.protocols.core_protocols import DataProtocol, StreamProtocol, TagProtocol + +from .test_pod_stream_staleness import _ConcurrentExecutor + +# Rows of the source, not in tag order. +IDS = [3, 1, 4, 0, 2, 5] + +CALLS: Counter[str] = Counter() +EPOCH = [0] + +UPSTREAMS = ["source", "operator", "pod_stream", "pod_over_operator"] + + +def f(x: int) -> int: + """Return ``x + 1``; counted.""" + CALLS["f"] += 1 + return x + 1 + + +def g(y: int) -> int: + """Return ``2 * y``; counted.""" + CALLS["g"] += 1 + return 2 * y + + +def stamped(x: int) -> int: + """Return ``x`` tagged with the current epoch: a function whose values change.""" + CALLS["stamped"] += 1 + return 1000 * EPOCH[0] + x + + +@pytest.fixture(autouse=True) +def _reset() -> Iterator[None]: + """Clear the call record and the epoch around each test.""" + CALLS.clear() + EPOCH[0] = 0 + yield + CALLS.clear() + EPOCH[0] = 0 + + +def _source(extra: bool) -> ArrowTableSource: + """A source with tag ``id`` and data ``x``; with ``extra``, a data column ``e`` too. + + A pod taking only ``x`` called on the source with ``e`` puts an automatic + ``SelectDataColumns`` operator stream in front of itself. + """ + fields = [ + pa.field("id", pa.int64(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] + arrays = [pa.array(IDS, pa.int64()), pa.array([10 * i for i in IDS], pa.int64())] + if extra: + fields.append(pa.field("e", pa.int64(), nullable=False)) + arrays.append(pa.array([-i for i in IDS], pa.int64())) + table = pa.Table.from_arrays(arrays, schema=pa.schema(fields)) + return ArrowTableSource(table, tag_columns=["id"], source_id="src") + + +def _other() -> ArrowTableSource: + """A second source with tag ``id`` and data ``q``, in another row order.""" + schema = pa.schema( + [ + pa.field("id", pa.int64(), nullable=False), + pa.field("q", pa.int64(), nullable=False), + ] + ) + ids = sorted(IDS) + table = pa.Table.from_arrays( + [pa.array(ids, pa.int64()), pa.array([7 * i for i in ids], pa.int64())], + schema=schema, + ) + return ArrowTableSource(table, tag_columns=["id"], source_id="other") + + +def _pod( + function: object, output: str, executor: str, cached: bool +) -> FunctionPod | CachedFunctionPod: + """The pod of ``function``; when ``cached``, storing its results in a fresh store.""" + pod = function_pod( + output_keys=output, + executor=_ConcurrentExecutor() if executor == "concurrent" else None, + )(function).pod + assert isinstance(pod, FunctionPod) + if not cached: + return pod + return CachedFunctionPod(pod, result_database=InMemoryArrowDatabase()) + + +def _stream(upstream: str, executor: str, cached: bool) -> StreamProtocol: + """The pod stream read by the tests, over an input of kind ``upstream``.""" + if upstream == "source": + return _pod(f, "y", executor, cached)(_source(extra=False)) + if upstream == "operator": + return _pod(f, "y", executor, cached)(_source(extra=True)) + first = _pod(f, "y", executor, cached)( + _source(extra=upstream == "pod_over_operator") + ) + return _pod(g, "z", executor, cached)(first) + + +def _expected_calls(upstream: str) -> Counter[str]: + """Each function of the stream called once per input row.""" + if upstream in ("source", "operator"): + return Counter({"f": len(IDS)}) + return Counter({"f": len(IDS), "g": len(IDS)}) + + +Row = tuple[TagProtocol, DataProtocol] + + +def _same_datagrams(left: list[Row], right: list[Row]) -> bool: + """Whether two reads yielded the very same data datagrams, in the same order.""" + return len(left) == len(right) and all( + a is b for (_, a), (_, b) in zip(left, right, strict=True) + ) + + +def _stale(stream: StreamProtocol) -> bool: + """Whether ``stream`` is stale (``is_stale`` is not on ``StreamProtocol``).""" + return bool(getattr(stream, "is_stale")) + + +@pytest.mark.parametrize("cached", [False, True], ids=["plain", "cached"]) +@pytest.mark.parametrize("executor", ["sequential", "concurrent"]) +@pytest.mark.parametrize("upstream", UPSTREAMS) +def test_interleaved_reads_compute_each_row_once( + upstream: str, executor: str, cached: bool +) -> None: + """Two interleaved reads and a later full read share one computation of every row.""" + stream = _stream(upstream, executor, cached) + first = iter(stream.iter_data()) + second = iter(stream.iter_data()) + first_rows = [next(first)] + second_rows = [next(second) for _ in range(3)] + first_rows += list(first) + second_rows += list(second) + assert CALLS == _expected_calls(upstream) + assert not _stale(stream) + + third_rows = list(stream.iter_data()) + assert CALLS == _expected_calls(upstream) + assert len(third_rows) == len(IDS) + assert _same_datagrams(first_rows, third_rows) + assert _same_datagrams(second_rows, third_rows) + + +@pytest.mark.parametrize("cached", [False, True], ids=["plain", "cached"]) +@pytest.mark.parametrize("executor", ["sequential", "concurrent"]) +@pytest.mark.parametrize("upstream", UPSTREAMS) +def test_a_read_after_an_abandoned_partial_read_computes_each_row_once( + upstream: str, executor: str, cached: bool +) -> None: + """A read left after one row is resumed by the next read, not started again.""" + stream = _stream(upstream, executor, cached) + partial = iter(stream.iter_data()) + first_row = next(partial) + del partial + rows = list(stream.iter_data()) + assert CALLS == _expected_calls(upstream) + assert len(rows) == len(IDS) + assert rows[0][1] is first_row[1] + assert not _stale(stream) + + +def _values(stream: StreamProtocol, column: str) -> dict[int, int]: + """Each row's ``column`` value, keyed by its ``id`` tag, read with ``iter_data``.""" + return { + cast(int, tag.as_dict()["id"]): cast(int, data.as_dict()[column]) + for tag, data in stream.iter_data() + } + + +@pytest.mark.parametrize("executor", ["sequential", "concurrent"]) +@pytest.mark.parametrize("through", ["pod_stream", "operator"]) +def test_a_reset_between_interleaved_reads_gives_one_state( + through: str, executor: str +) -> None: + """A reset of the input between two interleaved reads is a change of content. + + The stamped stream is reset after the first read took one row. The second read then + starts again from the new state, and once both reads are drained the stream is current + and every read of it is of the new state only. + """ + stamped_stream = _pod(stamped, "y", executor, cached=False)(_source(extra=False)) + if through == "pod_stream": + input_stream: StreamProtocol = stamped_stream + else: + # A join adds data column ``q``, so the pod call adds an automatic selection. + input_stream = Join()(stamped_stream, _other()) + last = _pod(g, "z", executor, cached=False)(input_stream) + + first = iter(last.iter_data()) + second = iter(last.iter_data()) + _ = next(first) + EPOCH[0] = 1 + clear_cache = getattr(stamped_stream, "clear_cache") + clear_cache() + second_rows = [next(second) for _ in range(3)] + _ = list(first) + second_rows += list(second) + assert not _stale(last) + + CALLS.clear() + third_rows = list(last.iter_data()) + assert CALLS == Counter() + assert _same_datagrams(second_rows, third_rows) + zs = _values(last, "z") + ys = _values(stamped_stream, "y") + assert CALLS == Counter() + assert sorted(zs) == sorted(ys) == sorted(IDS) + assert {y // 1000 for y in ys.values()} == {1} + for key, y in ys.items(): + assert zs[key] == 2 * y, key From 2c270f1568ecfdb546365d7a819a9bd7b127f258 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 03:10:48 -0800 Subject: [PATCH 2/5] fix(function_pod): a read in progress is judged by its input's noted state (ITL-1065) Two iterators open on one pod stream fed by an operator stream computed a row twice: the first read's clear_cache() stamped the stream before the operator stream computed, so the stream was older than its input until the end of the read, and the second iter_data() cleared it. While a read is in progress the stream is now stale only if its input's noted state (a pod stream's reset generation, any other input's modified time noted after it delivered a row) changed or its pod changed; completed results keep the modified-time rule. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1065.md | 12 +++++ design_issues/ITL-1052-F18.md | 3 ++ design_issues/ITL-1065-F19.md | 46 +++++++++++++++++++ docs/concepts/streams.md | 4 ++ src/orcapod/core/function_pod.py | 22 +++++++++ .../2026-11-13-itl-1065-interleaved-reads.md | 7 ++- 6 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 changelog.d/ITL-1065.md create mode 100644 design_issues/ITL-1065-F19.md diff --git a/changelog.d/ITL-1065.md b/changelog.d/ITL-1065.md new file mode 100644 index 000000000..79a13e69c --- /dev/null +++ b/changelog.d/ITL-1065.md @@ -0,0 +1,12 @@ +### Fixed + +#### Two reads of a pod stream open at once compute each row once (ITL-1065) + +With two `iter_data()` iterators open on one function pod stream fed by an operator stream +(for example the column selection a pod call adds when its input has a column the function +does not take), the second iterator threw away the rows the first had computed and computed +them again: one call too many, and a row the first iterator had already handed out was no +longer the stream's row. A read left partway and read again did the same. Such reads now share +the stream's results, whatever feeds it: each row is computed once and every read yields the +same rows. An upstream changed with `clear_cache()` between the reads still makes the next +read start again. (ITL-1065) diff --git a/design_issues/ITL-1052-F18.md b/design_issues/ITL-1052-F18.md index 943c2803b..60bca5704 100644 --- a/design_issues/ITL-1052-F18.md +++ b/design_issues/ITL-1052-F18.md @@ -16,6 +16,9 @@ so a chain through an operator was not affected; a pod reading a pod directly wa its results are complete, that is when its input is read to the end, before its reader sees the end; never when a read starts. `clear_cache()` stamps (a reset is a change). A failed or partial read does not stamp. +Between a read's first row and its end, the stream is judged by its input's noted state, not +by modified-time order: an upstream completing for the first time during the read is not a +change (F19, ITL-1065). **Fix:** the stamp moved from `_ensure_iterator` to the end of the input, on the sequential and the concurrent read paths. A reset of the input during a read must not be reported as diff --git a/design_issues/ITL-1065-F19.md b/design_issues/ITL-1065-F19.md new file mode 100644 index 000000000..d579e5a93 --- /dev/null +++ b/design_issues/ITL-1065-F19.md @@ -0,0 +1,46 @@ +## `src/orcapod/core/function_pod.py` + +### F19 — Interleaved reads of a pod stream fed by an operator stream recomputed a yielded row +**Status:** resolved +**Severity:** high +**Issue:** ITL-1065 + +A regression from F18 (PR #355). With two iterators open on one function pod stream whose +input is an operator stream (for example the automatic `SelectDataColumns` that `pod(src)` +inserts when `src` has a data column the function does not take), the second iterator dropped +the stream's results and computed them again: 7 calls of the function for 6 rows, and the row +the first iterator had already yielded was no longer the stream's row. A read left after one +row and then read again did the same. + +**Cause** (modified times printed at each step): the first read found the stream stale (its +input, the operator stream, had no modified time before it ran), so `clear_cache()` stamped +the stream at t1; then opening the input made the operator stream compute and take t2 > t1. +The stream stamps again only when its input is read to the end (F18), so until then it was +older than its input: `last_modified` was `None` and `is_stale` True in the middle of the read, +and the next `iter_data()` cleared it. Before F18 the stream stamped right after opening (and +so computing) the operator input, which hid this. A source is older than the stream, and a +pod-stream input stamps only at its end, so neither showed it directly; a pod stream over an +operator-fed pod stream did. + +**Rule:** a pod stream's results are judged in one of two ways. + +- Completed results (the input read to the end, or nothing read since a reset) are current + while neither a direct upstream nor the pod is newer or without a modified time (F18). +- Results in progress (a read has noted its input's state and not reached the input's end) are + current while that state is unchanged and the pod did not change. The state is the reset + generation of a pod-stream input, and the modified time of any other input, noted when its + first row (or its end) arrives. An upstream completing, or computing, for the first time + during the read is not a change of the stream's input. + +An operator stream needs no reset generation of its own: it computes when it is opened, so by +the time the pod stream notes its modified time it has computed, and that time moves only when +it is invalidated or recomputed, which are real changes. + +**Fix:** `FunctionPodStream.is_stale` applies the in-progress rule while a noted input state +exists; `_input_read_to_end` ends the read in progress after stamping (or clearing) the +modified time. A reset upstream during a read still changes the noted state: the next read of +the stream starts again, and the stream ends a read that saw the reset stale (F18 unchanged). +In-memory bookkeeping only: no hash, saved file or store sees it. Regressions: +`tests/test_core/function_pod/test_pod_stream_interleaved_reads.py` (source, operator, pod +stream and pod-over-operator inputs; sequential and concurrent paths; plain and cached pods; +interleaved reads, an abandoned partial read, and a reset between interleaved reads). diff --git a/docs/concepts/streams.md b/docs/concepts/streams.md index e26973cae..abb2e2d97 100644 --- a/docs/concepts/streams.md +++ b/docs/concepts/streams.md @@ -176,6 +176,10 @@ run: `head()`, `as_table()` and `iter_data()` serve them from memory, without re store (a function node whose run left no rows in memory reads its stored rows). Likewise, once a chain of streams such as `g.pod(f.pod(source))` has been read to the end, reading it again computes nothing. A change upstream (`clear_cache()`) makes every stream after it out of date, and its next read computes it again. +Two reads of one stream open at the same time, or a read that picks up one left partway, share +the stream's results: each row is computed once and every read yields the same rows, whatever +feeds the stream. An upstream finishing its first computation during the read is not a change; +a `clear_cache()` upstream between the reads is, and the next read starts again. In the preview, large cells are summarised (an array as `ndarray float64 (2, 3)`, bytes as `bytes (300 B)`, long text and long lists shortened); `head().table` keeps the full values. diff --git a/src/orcapod/core/function_pod.py b/src/orcapod/core/function_pod.py index 9e170ad26..d8a0f06ab 100644 --- a/src/orcapod/core/function_pod.py +++ b/src/orcapod/core/function_pod.py @@ -1109,6 +1109,28 @@ def _input_read_to_end(self) -> None: self._set_modified_time(None) else: self._update_modified_time() + # The read is complete: from now on the stream is judged by its modified time. + self._input_state = None + + @property + def is_stale(self) -> bool: + """Whether the stream's results must be discarded before it is read again. + + Results in progress (a read has noted its input's state and not reached the + input's end) stay current while that state is unchanged and the pod did not + change: an upstream completing or computing for the first time while the stream + is read is not a change of its input (ITL-1065). Otherwise the stream is stale + when it, an upstream or its pod has no modified time or an upstream or its pod + is newer (``StreamBase.is_stale``). + """ + if self._input_state is None: + return super().is_stale + current = self._current_input_state() + if current != self._input_state or None in current: + return True + own_time = self._modified_time + pod_time = self._function_pod.last_modified + return own_time is None or pod_time is None or pod_time > own_time def clear_cache(self) -> None: """Discard all in-memory cached state.""" diff --git a/superpowers/plans/2026-11-13-itl-1065-interleaved-reads.md b/superpowers/plans/2026-11-13-itl-1065-interleaved-reads.md index baa6e1559..f672a89e7 100644 --- a/superpowers/plans/2026-11-13-itl-1065-interleaved-reads.md +++ b/superpowers/plans/2026-11-13-itl-1065-interleaved-reads.md @@ -115,4 +115,9 @@ progress). `clear_cache` already clears it. ## Notes for a successor -- Status: (update as work proceeds) +- Status (2026-11-13): tests (b5247136) failed 8 sequential cases before the fix; fix in + `FunctionPodStream.is_stale` / `_input_read_to_end`; targeted suites pass (1472); repro and + the ITL-1034 suite's interleaved cases (`--runxfail`, 117) pass. Docs, F19, F18 pointer and + changelog fragment written. Next: merge origin/main, push, PR, Copilot. +- The ITL-1034 branch's strict `ITL_1065` xfail marks will XPASS once this merges: that branch + must drop them. From b87f2d2318cb5787457e9db3a8f6baa61fd092ae Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 03:15:18 -0800 Subject: [PATCH 3/5] fix(function_pod): a read in progress also sees a reset further up the chain (ITL-1065) A pod stream left in progress noted only its pod-stream input's reset generation, which does not move when a stream further up is reset: after an abandoned read of s <- s1 <- p and a reset of p, the next read resumed s with a truncated input. The in-progress rule now also asks a pod-stream input whether it is stale. Adds the coordinator's cases: as_table() or head() while an iterator is open, and a reset after an abandoned read, one or two links up or through an operator. Co-Authored-By: Claude Opus 5.5 --- design_issues/ITL-1065-F19.md | 21 +++-- src/orcapod/core/function_pod.py | 16 ++-- .../2026-11-13-itl-1065-interleaved-reads.md | 4 + .../test_pod_stream_interleaved_reads.py | 84 +++++++++++++++++++ 4 files changed, 115 insertions(+), 10 deletions(-) diff --git a/design_issues/ITL-1065-F19.md b/design_issues/ITL-1065-F19.md index d579e5a93..0cbb864e4 100644 --- a/design_issues/ITL-1065-F19.md +++ b/design_issues/ITL-1065-F19.md @@ -27,10 +27,19 @@ operator-fed pod stream did. - Completed results (the input read to the end, or nothing read since a reset) are current while neither a direct upstream nor the pod is newer or without a modified time (F18). - Results in progress (a read has noted its input's state and not reached the input's end) are - current while that state is unchanged and the pod did not change. The state is the reset - generation of a pod-stream input, and the modified time of any other input, noted when its - first row (or its end) arrives. An upstream completing, or computing, for the first time - during the read is not a change of the stream's input. + current while that state is unchanged, the input is not stale itself and the pod did not + change. The state is the reset generation of a pod-stream input, and the modified time of + any other input, noted when its first row (or its end) arrives. An upstream completing, or + computing, for the first time during the read is not a change of the stream's input. A + pod-stream input is also asked whether it is stale, because its generation does not move + when a stream further up is reset; any other input's modified time already reflects its + upstreams. +- A read in progress ends when any read reaches the input's end (the stream stamps, or clears + its modified time if the input changed) or when the stream is reset (`clear_cache()`, also + the one a stale stream's next read makes). An abandoned iterator does not end it: the + stream keeps its partial results and its open input, and the next read resumes it + (ITL-1039) unless the input changed, in which case that read finds the stream stale and + starts again. An operator stream needs no reset generation of its own: it computes when it is opened, so by the time the pod stream notes its modified time it has computed, and that time moves only when @@ -43,4 +52,6 @@ the stream starts again, and the stream ends a read that saw the reset stale (F1 In-memory bookkeeping only: no hash, saved file or store sees it. Regressions: `tests/test_core/function_pod/test_pod_stream_interleaved_reads.py` (source, operator, pod stream and pod-over-operator inputs; sequential and concurrent paths; plain and cached pods; -interleaved reads, an abandoned partial read, and a reset between interleaved reads). +interleaved reads, an abandoned partial read, a reset between interleaved reads, a reset +after an abandoned read one or two links up, and `as_table()` or `head()` while an iterator is +open). diff --git a/src/orcapod/core/function_pod.py b/src/orcapod/core/function_pod.py index d8a0f06ab..fa16f472b 100644 --- a/src/orcapod/core/function_pod.py +++ b/src/orcapod/core/function_pod.py @@ -1117,17 +1117,23 @@ def is_stale(self) -> bool: """Whether the stream's results must be discarded before it is read again. Results in progress (a read has noted its input's state and not reached the - input's end) stay current while that state is unchanged and the pod did not - change: an upstream completing or computing for the first time while the stream - is read is not a change of its input (ITL-1065). Otherwise the stream is stale - when it, an upstream or its pod has no modified time or an upstream or its pod - is newer (``StreamBase.is_stale``). + input's end) stay current while that state is unchanged, the input is not + stale itself and the pod did not change: an upstream completing or computing for + the first time while the stream is read is not a change of its input (ITL-1065). + A pod-stream input is asked whether it is stale because its noted state, the + reset generation, does not move when a stream further up is reset; any other + input's modified time already reflects its upstreams. Otherwise the stream is + stale when it, an upstream or its pod has no modified time or an upstream or its + pod is newer (``StreamBase.is_stale``). """ if self._input_state is None: return super().is_stale current = self._current_input_state() if current != self._input_state or None in current: return True + upstream = self._input_stream + if isinstance(upstream, FunctionPodStream) and upstream.is_stale: + return True own_time = self._modified_time pod_time = self._function_pod.last_modified return own_time is None or pod_time is None or pod_time > own_time diff --git a/superpowers/plans/2026-11-13-itl-1065-interleaved-reads.md b/superpowers/plans/2026-11-13-itl-1065-interleaved-reads.md index f672a89e7..00c91489f 100644 --- a/superpowers/plans/2026-11-13-itl-1065-interleaved-reads.md +++ b/superpowers/plans/2026-11-13-itl-1065-interleaved-reads.md @@ -119,5 +119,9 @@ progress). `clear_cache` already clears it. `FunctionPodStream.is_stale` / `_input_read_to_end`; targeted suites pass (1472); repro and the ITL-1034 suite's interleaved cases (`--runxfail`, 117) pass. Docs, F19, F18 pointer and changelog fragment written. Next: merge origin/main, push, PR, Copilot. +- Coordinator follow-up (msg_7bf21fb00ea3): added `as_table`/`head` during an open read and + an abandoned read followed by a reset. The latter found a gap two links up (`s <- s1 <- p`, + only `s1`'s generation noted): the in-progress rule now also asks a pod-stream input whether + it is stale. Merge origin/main after #358 (as_table clears a stale cache) and rerun. - The ITL-1034 branch's strict `ITL_1065` xfail marks will XPASS once this merges: that branch must drop them. diff --git a/tests/test_core/function_pod/test_pod_stream_interleaved_reads.py b/tests/test_core/function_pod/test_pod_stream_interleaved_reads.py index dc1e73666..b683543e0 100644 --- a/tests/test_core/function_pod/test_pod_stream_interleaved_reads.py +++ b/tests/test_core/function_pod/test_pod_stream_interleaved_reads.py @@ -238,3 +238,87 @@ def test_a_reset_between_interleaved_reads_gives_one_state( assert {y // 1000 for y in ys.values()} == {1} for key, y in ys.items(): assert zs[key] == 2 * y, key + + +def h(z: int) -> int: + """Return ``z - 3``; counted.""" + CALLS["h"] += 1 + return z - 3 + + +def _over_stamped(through: str, executor: str) -> tuple[StreamProtocol, StreamProtocol]: + """The stamped stream and the last stream over it, ``through`` the given links. + + ``pod_stream``: a pod reading the stamped stream. ``operator``: a pod over a join of + it (the pod call adds an automatic selection). ``pod_chain``: two pods, so the reset + is two links up from the last stream. + """ + stamped_stream = _pod(stamped, "y", executor, cached=False)(_source(extra=False)) + if through == "operator": + # A join adds data column ``q``, so the pod call adds an automatic selection. + last = _pod(g, "z", executor, cached=False)(Join()(stamped_stream, _other())) + else: + last = _pod(g, "z", executor, cached=False)(stamped_stream) + if through == "pod_chain": + last = _pod(h, "w", executor, cached=False)(last) + return stamped_stream, last + + +def _derived(through: str, y: int) -> int: + """The last stream's value for a stamped value ``y``.""" + return 2 * y - 3 if through == "pod_chain" else 2 * y + + +@pytest.mark.parametrize("executor", ["sequential", "concurrent"]) +@pytest.mark.parametrize("through", ["pod_stream", "operator", "pod_chain"]) +def test_an_abandoned_read_does_not_hide_a_later_reset( + through: str, executor: str +) -> None: + """A read left after one row stays resumable, and a reset after it is still seen. + + The abandoned read keeps the stream's partial results and its open input; the next + read resumes it unless the input changed. Here the stamped stream is reset after the + read was abandoned, however far up it is, so the stream is stale and the next read + starts again from the new state. + """ + stamped_stream, last = _over_stamped(through, executor) + output = "w" if through == "pod_chain" else "z" + abandoned = iter(last.iter_data()) + _ = next(abandoned) + del abandoned + EPOCH[0] = 1 + clear_cache = getattr(stamped_stream, "clear_cache") + clear_cache() + assert _stale(last) + + values = _values(last, output) + assert not _stale(last) + ys = _values(stamped_stream, "y") + assert sorted(values) == sorted(ys) == sorted(IDS) + assert {y // 1000 for y in ys.values()} == {1} + for key, y in ys.items(): + assert values[key] == _derived(through, y), key + + +@pytest.mark.parametrize("executor", ["sequential", "concurrent"]) +@pytest.mark.parametrize("read", ["as_table", "head"]) +@pytest.mark.parametrize("upstream", UPSTREAMS) +def test_a_table_or_head_during_an_open_read_computes_each_row_once( + upstream: str, read: str, executor: str +) -> None: + """``as_table()`` or ``head()`` while an iterator is open shares the stream's results.""" + stream = _stream(upstream, executor, cached=False) + first = iter(stream.iter_data()) + first_rows = [next(first)] + if read == "as_table": + assert stream.as_table().num_rows == len(IDS) + else: + assert stream.head(100).table.num_rows >= 1 + first_rows += list(first) + assert CALLS == _expected_calls(upstream) + assert not _stale(stream) + third_rows = list(stream.iter_data()) + assert CALLS == _expected_calls(upstream) + assert _same_datagrams(first_rows, third_rows) + assert stream.as_table().num_rows == len(IDS) + assert CALLS == _expected_calls(upstream) From d1229b6097fd72142ef8ee294524a374e6217b9a Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 04:08:23 -0800 Subject: [PATCH 4/5] test(function_pod): a reset two links up between interleaved reads restarts the read (ITL-1065) Review follow-up (Copilot, CodeRabbit on 2c270f15): the case is fixed by b87f2d23; this covers it with two open iterators as well as after an abandoned read. Co-Authored-By: Claude Opus 5.5 --- design_issues/ITL-1065-F19.md | 4 +- .../test_pod_stream_interleaved_reads.py | 81 +++++++++---------- 2 files changed, 40 insertions(+), 45 deletions(-) diff --git a/design_issues/ITL-1065-F19.md b/design_issues/ITL-1065-F19.md index 0cbb864e4..3ae181cfa 100644 --- a/design_issues/ITL-1065-F19.md +++ b/design_issues/ITL-1065-F19.md @@ -52,6 +52,6 @@ the stream starts again, and the stream ends a read that saw the reset stale (F1 In-memory bookkeeping only: no hash, saved file or store sees it. Regressions: `tests/test_core/function_pod/test_pod_stream_interleaved_reads.py` (source, operator, pod stream and pod-over-operator inputs; sequential and concurrent paths; plain and cached pods; -interleaved reads, an abandoned partial read, a reset between interleaved reads, a reset -after an abandoned read one or two links up, and `as_table()` or `head()` while an iterator is +interleaved reads, an abandoned partial read, a reset one or two links up between +interleaved reads or after an abandoned read, and `as_table()` or `head()` while an iterator is open). diff --git a/tests/test_core/function_pod/test_pod_stream_interleaved_reads.py b/tests/test_core/function_pod/test_pod_stream_interleaved_reads.py index b683543e0..4f8f57ad7 100644 --- a/tests/test_core/function_pod/test_pod_stream_interleaved_reads.py +++ b/tests/test_core/function_pod/test_pod_stream_interleaved_reads.py @@ -197,49 +197,6 @@ def _values(stream: StreamProtocol, column: str) -> dict[int, int]: } -@pytest.mark.parametrize("executor", ["sequential", "concurrent"]) -@pytest.mark.parametrize("through", ["pod_stream", "operator"]) -def test_a_reset_between_interleaved_reads_gives_one_state( - through: str, executor: str -) -> None: - """A reset of the input between two interleaved reads is a change of content. - - The stamped stream is reset after the first read took one row. The second read then - starts again from the new state, and once both reads are drained the stream is current - and every read of it is of the new state only. - """ - stamped_stream = _pod(stamped, "y", executor, cached=False)(_source(extra=False)) - if through == "pod_stream": - input_stream: StreamProtocol = stamped_stream - else: - # A join adds data column ``q``, so the pod call adds an automatic selection. - input_stream = Join()(stamped_stream, _other()) - last = _pod(g, "z", executor, cached=False)(input_stream) - - first = iter(last.iter_data()) - second = iter(last.iter_data()) - _ = next(first) - EPOCH[0] = 1 - clear_cache = getattr(stamped_stream, "clear_cache") - clear_cache() - second_rows = [next(second) for _ in range(3)] - _ = list(first) - second_rows += list(second) - assert not _stale(last) - - CALLS.clear() - third_rows = list(last.iter_data()) - assert CALLS == Counter() - assert _same_datagrams(second_rows, third_rows) - zs = _values(last, "z") - ys = _values(stamped_stream, "y") - assert CALLS == Counter() - assert sorted(zs) == sorted(ys) == sorted(IDS) - assert {y // 1000 for y in ys.values()} == {1} - for key, y in ys.items(): - assert zs[key] == 2 * y, key - - def h(z: int) -> int: """Return ``z - 3``; counted.""" CALLS["h"] += 1 @@ -269,6 +226,44 @@ def _derived(through: str, y: int) -> int: return 2 * y - 3 if through == "pod_chain" else 2 * y +@pytest.mark.parametrize("executor", ["sequential", "concurrent"]) +@pytest.mark.parametrize("through", ["pod_stream", "operator", "pod_chain"]) +def test_a_reset_between_interleaved_reads_gives_one_state( + through: str, executor: str +) -> None: + """A reset of the input between two interleaved reads is a change of content. + + The stamped stream, one or two links up, is reset after the first read took one row. + The second read then starts again from the new state, and once both reads are drained + the stream is current and every read of it is of the new state only. + """ + stamped_stream, last = _over_stamped(through, executor) + output = "w" if through == "pod_chain" else "z" + first = iter(last.iter_data()) + second = iter(last.iter_data()) + _ = next(first) + EPOCH[0] = 1 + clear_cache = getattr(stamped_stream, "clear_cache") + clear_cache() + assert _stale(last) + second_rows = [next(second) for _ in range(3)] + _ = list(first) + second_rows += list(second) + assert not _stale(last) + + CALLS.clear() + third_rows = list(last.iter_data()) + assert CALLS == Counter() + assert _same_datagrams(second_rows, third_rows) + values = _values(last, output) + ys = _values(stamped_stream, "y") + assert CALLS == Counter() + assert sorted(values) == sorted(ys) == sorted(IDS) + assert {y // 1000 for y in ys.values()} == {1} + for key, y in ys.items(): + assert values[key] == _derived(through, y), key + + @pytest.mark.parametrize("executor", ["sequential", "concurrent"]) @pytest.mark.parametrize("through", ["pod_stream", "operator", "pod_chain"]) def test_an_abandoned_read_does_not_hide_a_later_reset( From f1d5bef83f597b2e2f145fc2360f55982684fcfa Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 04:11:20 -0800 Subject: [PATCH 5/5] test(function_pod): the head after an abandoned read shows its rows (ITL-1065) A read of an operator-fed pod stream left after three rows made the stream stale on main, and head() showed no rows as not computed; with the in-progress rule it shows the computed rows (partial) and is not stale. Reported by the stream-consistency suite. Co-Authored-By: Claude Opus 5.5 --- design_issues/ITL-1065-F19.md | 3 ++- .../test_pod_stream_interleaved_reads.py | 26 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/design_issues/ITL-1065-F19.md b/design_issues/ITL-1065-F19.md index 3ae181cfa..e17003b74 100644 --- a/design_issues/ITL-1065-F19.md +++ b/design_issues/ITL-1065-F19.md @@ -53,5 +53,6 @@ In-memory bookkeeping only: no hash, saved file or store sees it. Regressions: `tests/test_core/function_pod/test_pod_stream_interleaved_reads.py` (source, operator, pod stream and pod-over-operator inputs; sequential and concurrent paths; plain and cached pods; interleaved reads, an abandoned partial read, a reset one or two links up between -interleaved reads or after an abandoned read, and `as_table()` or `head()` while an iterator is +interleaved reads or after an abandoned read, `head()` after an abandoned read (partial, not +stale), and `as_table()` or `head()` while an iterator is open). diff --git a/tests/test_core/function_pod/test_pod_stream_interleaved_reads.py b/tests/test_core/function_pod/test_pod_stream_interleaved_reads.py index 4f8f57ad7..87c21c23f 100644 --- a/tests/test_core/function_pod/test_pod_stream_interleaved_reads.py +++ b/tests/test_core/function_pod/test_pod_stream_interleaved_reads.py @@ -25,6 +25,7 @@ from orcapod.core.sources import ArrowTableSource from orcapod.databases import InMemoryArrowDatabase from orcapod.protocols.core_protocols import DataProtocol, StreamProtocol, TagProtocol +from orcapod.streams import HeadStatus from .test_pod_stream_staleness import _ConcurrentExecutor @@ -317,3 +318,28 @@ def test_a_table_or_head_during_an_open_read_computes_each_row_once( assert _same_datagrams(first_rows, third_rows) assert stream.as_table().num_rows == len(IDS) assert CALLS == _expected_calls(upstream) + + +@pytest.mark.parametrize("executor", ["sequential", "concurrent"]) +@pytest.mark.parametrize("upstream", UPSTREAMS) +def test_the_head_after_an_abandoned_read_shows_its_rows( + upstream: str, executor: str +) -> None: + """A read left after three rows is not stale: ``head()`` shows the rows it computed. + + On the sequential path three rows are computed and the head is partial; the concurrent + path computes every row on the first one, so its head is complete. No head computes. + """ + stream = _stream(upstream, executor, cached=False) + abandoned = iter(stream.iter_data()) + for _ in range(3): + _ = next(abandoned) + del abandoned + calls = Counter(CALLS) + assert not _stale(stream) + head = stream.head(2) + assert head.status is ( + HeadStatus.PARTIAL if executor == "sequential" else HeadStatus.ROWS + ), head.header() + assert head.table.num_rows == 2 + assert CALLS == calls